diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c874e1b361..932a3bc3c3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -231,7 +231,11 @@ jobs: run: rustup show - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - name: "doc --workspace --no-deps" - run: cargo doc --locked --workspace --no-deps --features linalg,flatbuffers,experimental_diversity_search + run: cargo doc --locked --workspace --no-deps --features linalg,flatbuffers,experimental_diversity_search,integration-test + env: + RUSTDOCFLAGS: -D rustdoc::all + - name: "diskann-inmem private docs" + run: cargo doc --locked --package diskann-inmem --no-deps --all-features --document-private-items env: RUSTDOCFLAGS: -D rustdoc::all diff --git a/Cargo.lock b/Cargo.lock index 0c862c6a08..d4a3b6df70 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -634,6 +634,7 @@ dependencies = [ "diskann-vector", "diskann-wide", "half", + "hashbrown 0.16.1", "parking_lot", "rand", "serde", diff --git a/diskann-benchmark/src/index/inmem2.rs b/diskann-benchmark/src/index/inmem2.rs index 93c100c902..60b776d635 100644 --- a/diskann-benchmark/src/index/inmem2.rs +++ b/diskann-benchmark/src/index/inmem2.rs @@ -29,7 +29,8 @@ use diskann_benchmark_runner::{ Benchmark, Checker, Checkpoint, Input, Registry, }; use diskann_inmem::{ - layers::{Full, FullPrecision}, + num::{Capacity, MaxDegree}, + repr::{Full, FullPrecision}, Provider, Strategy, }; use diskann_utils::views::{Matrix, MatrixView}; @@ -47,6 +48,8 @@ use crate::{ pub(crate) fn register_benchmarks(registry: &mut Registry) -> anyhow::Result<()> { registry.register("inmem2-f32", Build::::new())?; + registry.register("inmem2-u8", Build::::new())?; + // registry.register("inmem2-f16", Build::::new())?; registry.register("inmem2-f32-stream", StreamingBenchmark::::new())?; Ok(()) @@ -420,7 +423,7 @@ impl Build { impl Benchmark for Build where - T: diskann_inmem::layers::FullPrecision + diskann::graph::SampleableForStart + AsDataType, + T: diskann_inmem::repr::FullPrecision + diskann::graph::SampleableForStart + AsDataType, { type Input = StaticBuild; type Output = (); @@ -472,11 +475,14 @@ where // Compute the medoid of the dataset as the single start point. let start = StartPointStrategy::Medoid.compute(data.as_view())?; - let layer = Full::::new(dim, input.data.distance); - let config = - diskann_inmem::provider::Config::new(num_points, input.build.config.max_degree().get()); - let provider = Provider::<_, u32>::new(layer, config, start.row_iter())?; + let config = Full::config( + Capacity::new(num_points), + MaxDegree::new(input.build.config.max_degree().get()), + input.data.distance, + start, + )?; + let provider = Provider::<_, u32>::new(config)?; let index = Arc::new(DiskANNIndex::new( input.build.config.clone(), provider, @@ -837,17 +843,18 @@ where let queries: Arc> = Arc::new(datafiles::load_dataset(datafiles::BinFile( &input.search.queries, ))?); - let dim = dataset.ncols(); // Compute the medoid of the dataset as the single start point. let start = StartPointStrategy::Medoid.compute(dataset.as_view())?; let index_config = input.build.config.clone(); - let layer = Full::::new(dim, input.data.distance); - - let config = - diskann_inmem::provider::Config::new(max_points, index_config.max_degree().get()); - let provider = Provider::<_, u32>::new(layer, config, start.row_iter())?; + let config = Full::config( + Capacity::new(max_points), + MaxDegree::new(index_config.max_degree().get()), + input.data.distance, + start, + )?; + let provider = Provider::<_, u32>::new(config)?; let index = Arc::new(DiskANNIndex::new(index_config, provider, None)); let num_threads = input.build.num_threads; diff --git a/diskann-inmem/Cargo.toml b/diskann-inmem/Cargo.toml index 00e02dbf26..72dec19bf5 100644 --- a/diskann-inmem/Cargo.toml +++ b/diskann-inmem/Cargo.toml @@ -16,8 +16,9 @@ diskann-utils = { workspace = true, default-features = false } diskann-vector = { workspace = true } diskann-wide = { workspace = true } parking_lot = "0.12.5" -thiserror = { workspace = true } half = { workspace = true } +hashbrown = { workspace = true } +thiserror = { workspace = true } # Integration Test Dependencies diskann-benchmark-runner = { workspace = true, optional = true, features = ["ux-tools"] } diff --git a/diskann-inmem/integration/index/object.rs b/diskann-inmem/integration/index/object.rs index cc5f4dc6f9..32e357d69d 100644 --- a/diskann-inmem/integration/index/object.rs +++ b/diskann-inmem/integration/index/object.rs @@ -14,7 +14,7 @@ use diskann_benchmark_runner::utils::fmt::KeyValue; use serde::{Deserialize, Serialize}; use thiserror::Error; -use diskann_inmem::{Context, Provider, Strategy, integration, layers}; +use diskann_inmem::{Context, Provider, Strategy, integration, repr}; use crate::support::{ check::{CheckMatch, Match, check_all_fields}, @@ -186,9 +186,9 @@ impl CheckMatch for Counters { // Impls // /////////// -impl Index for DiskANNIndex, u64>> +impl Index for DiskANNIndex, u64>> where - T: layers::FullPrecision + FromSlice + AsDataType, + T: repr::FullPrecision + FromSlice + AsDataType, { fn search<'a>( &'a self, diff --git a/diskann-inmem/integration/index/runner.rs b/diskann-inmem/integration/index/runner.rs index 9b767f41cd..97e711a5d1 100644 --- a/diskann-inmem/integration/index/runner.rs +++ b/diskann-inmem/integration/index/runner.rs @@ -15,10 +15,13 @@ use diskann_benchmark_runner::{ }; use diskann_utils::views::Matrix; use diskann_vector::distance::Metric; -use half::f16; use serde::{Deserialize, Serialize}; -use diskann_inmem::{Provider, layers}; +use diskann_inmem::{ + Provider, + num::{Capacity, MaxDegree}, + repr::Full, +}; use crate::{ index::{Counters, Index}, @@ -106,7 +109,7 @@ mod dto { } #[derive(Debug, Serialize, Deserialize)] - pub(super) enum Layer { + pub(super) enum Representation { FullPrecision { data_type: DataType }, } @@ -134,7 +137,7 @@ mod dto { #[derive(Debug, Serialize, Deserialize)] pub(super) struct Test { pub(super) data: Data, - pub(super) layer: Layer, + pub(super) representation: Representation, pub(super) build: Build, pub(super) search: Search, } @@ -227,20 +230,20 @@ struct Bundle { } #[derive(Debug)] -enum Layer { +enum Representation { FullPrecision { data_type: DataType }, } -impl Layer { - fn from_raw(raw: dto::Layer) -> Self { +impl Representation { + fn from_raw(raw: dto::Representation) -> Self { match raw { - dto::Layer::FullPrecision { data_type } => Self::FullPrecision { data_type }, + dto::Representation::FullPrecision { data_type } => Self::FullPrecision { data_type }, } } - fn as_raw(&self) -> dto::Layer { + fn as_raw(&self) -> dto::Representation { match self { - Self::FullPrecision { data_type } => dto::Layer::FullPrecision { + Self::FullPrecision { data_type } => dto::Representation::FullPrecision { data_type: *data_type, }, } @@ -323,7 +326,7 @@ impl Search { #[derive(Debug)] struct Test { data: Data, - layer: Layer, + representation: Representation, build: Build, search: Search, } @@ -331,13 +334,13 @@ struct Test { impl Test { fn from_raw(raw: dto::Test, checker: Option<&mut Checker>) -> anyhow::Result { let data = Data::from_raw(raw.data, checker)?; - let layer = Layer::from_raw(raw.layer); + let representation = Representation::from_raw(raw.representation); let build = Build::from_raw(raw.build, data.metric)?; let search = Search::from_raw(raw.search)?; Ok(Self { data, - layer, + representation, build, search, }) @@ -346,7 +349,7 @@ impl Test { fn as_raw(&self) -> anyhow::Result { Ok(dto::Test { data: self.data.as_raw()?, - layer: self.layer.as_raw(), + representation: self.representation.as_raw(), build: self.build.as_raw(), search: self.search.as_raw(), }) @@ -357,8 +360,8 @@ impl Test { capacity: usize, start_points: DatasetView<'_>, ) -> anyhow::Result> { - match self.layer { - Layer::FullPrecision { data_type } => { + match self.representation { + Representation::FullPrecision { data_type } => { if start_points.data_type() != data_type { anyhow::bail!( "mismatched data types for start point - expected {}, got {}", @@ -367,30 +370,45 @@ impl Test { ); } - let dim = start_points.ncols(); let metric = self.data.metric; - let config = diskann_inmem::provider::Config::new( - capacity, - self.build.config.max_degree().get(), - ); - + let max_degree = self.build.config.max_degree().get(); let index_config = self.build.config.clone(); let index = match start_points { DatasetView::F32(v) => finish( - Provider::new(layers::Full::::new(dim, metric), config, v.row_iter())?, + Provider::new(Full::config( + Capacity::new(capacity), + MaxDegree::new(max_degree), + metric, + v.to_owned(), + )?)?, index_config, ), DatasetView::F16(v) => finish( - Provider::new(layers::Full::::new(dim, metric), config, v.row_iter())?, + Provider::new(Full::config( + Capacity::new(capacity), + MaxDegree::new(max_degree), + metric, + v.to_owned(), + )?)?, index_config, ), DatasetView::U8(v) => finish( - Provider::new(layers::Full::::new(dim, metric), config, v.row_iter())?, + Provider::new(Full::config( + Capacity::new(capacity), + MaxDegree::new(max_degree), + metric, + v.to_owned(), + )?)?, index_config, ), DatasetView::I8(v) => finish( - Provider::new(layers::Full::::new(dim, metric), config, v.row_iter())?, + Provider::new(Full::config( + Capacity::new(capacity), + MaxDegree::new(max_degree), + metric, + v.to_owned(), + )?)?, index_config, ), }; @@ -439,7 +457,7 @@ impl diskann_benchmark_runner::Input for Test { data_type: DataType::F32, preprocess: vec![], }, - layer: dto::Layer::FullPrecision { + representation: dto::Representation::FullPrecision { data_type: DataType::F32, }, build: dto::Build { @@ -483,8 +501,8 @@ impl diskann_benchmark_runner::Benchmark for FullPrecision { type Output = BuildAndSearch; fn try_match(&self, input: &Test, context: &MatchContext) -> Score { - // Future-proof against additional enums in `input.layer`. - let Layer::FullPrecision { .. } = input.layer; + // Future-proof against additional enums in `input.representation`. + let Representation::FullPrecision { .. } = input.representation; context.success(0) } @@ -498,8 +516,8 @@ impl diskann_benchmark_runner::Benchmark for FullPrecision { _checkpoint: Checkpoint<'_>, mut output: &mut dyn Output, ) -> anyhow::Result { - // Future-proof against additional enums in `input.layer`. - let Layer::FullPrecision { data_type } = input.layer; + // Future-proof against additional enums in `input.representation`. + let Representation::FullPrecision { data_type } = input.representation; // Load the data and perform any necessary data conversions. let Bundle { diff --git a/diskann-inmem/integration/jsons/integration-baseline.json b/diskann-inmem/integration/jsons/integration-baseline.json index 39bd0485d5..6433eee856 100644 --- a/diskann-inmem/integration/jsons/integration-baseline.json +++ b/diskann-inmem/integration/jsons/integration-baseline.json @@ -16,7 +16,7 @@ "preprocess": [], "queries": "/yfcc/yfcc_query_100.fbin" }, - "layer": { + "representation": { "FullPrecision": { "data_type": "f32" } @@ -48,7 +48,7 @@ "append_neighbors": 96949, "distance": 2867876, "get_neighbors": 352139, - "get_vector": 3067092, + "get_vector": 2781513, "query_distance": 2240744, "set_neighbors": 23599, "set_vector": 10000 @@ -137,7 +137,7 @@ "preprocess": [], "queries": "/yfcc/yfcc_query_100.fbin" }, - "layer": { + "representation": { "FullPrecision": { "data_type": "f16" } @@ -169,7 +169,7 @@ "append_neighbors": 96949, "distance": 2867876, "get_neighbors": 352139, - "get_vector": 3067092, + "get_vector": 2781513, "query_distance": 2240744, "set_neighbors": 23599, "set_vector": 10000 @@ -258,7 +258,7 @@ "preprocess": [], "queries": "/yfcc/yfcc_query_100.fbin" }, - "layer": { + "representation": { "FullPrecision": { "data_type": "u8" } @@ -290,7 +290,7 @@ "append_neighbors": 96949, "distance": 2867876, "get_neighbors": 352139, - "get_vector": 3067092, + "get_vector": 2781513, "query_distance": 2240744, "set_neighbors": 23599, "set_vector": 10000 @@ -382,7 +382,7 @@ ], "queries": "/yfcc/yfcc_query_100.fbin" }, - "layer": { + "representation": { "FullPrecision": { "data_type": "i8" } @@ -414,7 +414,7 @@ "append_neighbors": 97055, "distance": 2867292, "get_neighbors": 352087, - "get_vector": 3064106, + "get_vector": 2778779, "query_distance": 2238420, "set_neighbors": 23587, "set_vector": 10000 @@ -486,4 +486,4 @@ ] } } -] \ No newline at end of file +] diff --git a/diskann-inmem/integration/jsons/integration.json b/diskann-inmem/integration/jsons/integration.json index 57f2a338ce..c00dbae441 100644 --- a/diskann-inmem/integration/jsons/integration.json +++ b/diskann-inmem/integration/jsons/integration.json @@ -21,7 +21,7 @@ "queries": "yfcc_query_100.fbin", "preprocess": [] }, - "layer": { + "representation": { "FullPrecision": { "data_type": "f32" } @@ -64,7 +64,7 @@ "queries": "yfcc_query_100.fbin", "preprocess": [] }, - "layer": { + "representation": { "FullPrecision": { "data_type": "f16" } @@ -107,7 +107,7 @@ "queries": "yfcc_query_100.fbin", "preprocess": [] }, - "layer": { + "representation": { "FullPrecision": { "data_type": "u8" } @@ -153,7 +153,7 @@ "floor" ] }, - "layer": { + "representation": { "FullPrecision": { "data_type": "i8" } diff --git a/diskann-inmem/integration/jsons/store-stress-test.json b/diskann-inmem/integration/jsons/store-stress-test.json index 5a1e8351f4..3404133d8d 100644 --- a/diskann-inmem/integration/jsons/store-stress-test.json +++ b/diskann-inmem/integration/jsons/store-stress-test.json @@ -2,19 +2,38 @@ "search_directories": [], "jobs": [ { - "type": "store-stress", + "type": "store-stress-invasive", "content": { - "readers": 4, - "writers": 2, - "retirers": 1, - "capacity": 512, "entry_bytes": 64, - "epoch_guard_slots": 64, - "freelist_recycle_capacity": 32, - "low_watermark": 128, - "duration_secs": 2, - "max_ops": 2000000, - "seed": 11939873485092837375 + "setup": { + "readers": 4, + "writers": 2, + "retirers": 1, + "capacity": 512, + "epoch_guard_slots": 64, + "freelist_recycle_capacity": 32, + "low_watermark": 128, + "duration_secs": 2, + "max_ops": 2000000, + "seed": 11939873485092837375 + } + } + }, + { + "type": "store-stress-checked", + "content": { + "setup": { + "readers": 4, + "writers": 2, + "retirers": 1, + "capacity": 512, + "epoch_guard_slots": 64, + "freelist_recycle_capacity": 32, + "low_watermark": 128, + "duration_secs": 2, + "max_ops": 2000000, + "seed": 11939873485092837375 + } } } ] diff --git a/diskann-inmem/integration/jsons/store-stress.json b/diskann-inmem/integration/jsons/store-stress.json index 89584764b5..4fdce96604 100644 --- a/diskann-inmem/integration/jsons/store-stress.json +++ b/diskann-inmem/integration/jsons/store-stress.json @@ -2,19 +2,38 @@ "search_directories": [], "jobs": [ { - "type": "store-stress", + "type": "store-stress-checked", + "content": { + "setup": { + "readers": 8, + "writers": 4, + "retirers": 2, + "capacity": 4096, + "epoch_guard_slots": 64, + "freelist_recycle_capacity": 512, + "low_watermark": 1024, + "duration_secs": 10, + "max_ops": 50000000, + "seed": 11939873485092837375 + } + } + }, + { + "type": "store-stress-invasive", "content": { - "readers": 8, - "writers": 4, - "retirers": 2, - "capacity": 4096, "entry_bytes": 128, - "epoch_guard_slots": 64, - "freelist_recycle_capacity": 512, - "low_watermark": 1024, - "duration_secs": 10, - "max_ops": 50000000, - "seed": 11939873485092837375 + "setup": { + "readers": 8, + "writers": 4, + "retirers": 2, + "capacity": 4096, + "epoch_guard_slots": 64, + "freelist_recycle_capacity": 512, + "low_watermark": 1024, + "duration_secs": 10, + "max_ops": 50000000, + "seed": 11939873485092837375 + } } } ] diff --git a/diskann-inmem/integration/main.rs b/diskann-inmem/integration/main.rs index d3667b6842..af16d69f17 100644 --- a/diskann-inmem/integration/main.rs +++ b/diskann-inmem/integration/main.rs @@ -3,6 +3,8 @@ * Licensed under the MIT license. */ +//! Integration test runner for [`diskann_inmem`]. + mod index; mod store; mod support; @@ -12,7 +14,7 @@ use diskann_benchmark_runner::{App, Registry, output}; /// Build a [`Registry`] with all integration benchmarks registered. fn registry() -> anyhow::Result { let mut registry = Registry::new(); - registry.register("store-stress", store::StoreStress)?; + store::register(&mut registry)?; index::register(&mut registry)?; Ok(registry) } diff --git a/diskann-inmem/integration/store.rs b/diskann-inmem/integration/store.rs deleted file mode 100644 index ec4e31f4c8..0000000000 --- a/diskann-inmem/integration/store.rs +++ /dev/null @@ -1,597 +0,0 @@ -/* - * Copyright (c) Microsoft Corporation. - * Licensed under the MIT license. - */ - -//! Concurrency stress test for the in-memory [`Store`](diskann_inmem::integration::store::Store). -//! -//! Reader, writer, and retirer threads hammer the epoch-based store concurrently while a -//! per-guard invariant checker verifies the store's safety guarantees: -//! -//! 1. Reads are never torn. -//! 2. A readable value is stable for the lifetime of a single reader guard. -//! 3. A slot never resurrects (`readable -> unreadable -> readable`) within one guard. - -#![expect( - clippy::unwrap_used, - reason = "this code works mainly as an integration test" -)] - -use std::{ - collections::HashMap, - io::Write, - sync::{ - Mutex, - atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering::Relaxed}, - }, - time::{Duration, Instant}, -}; - -use diskann_benchmark_runner::{ - Benchmark, Checker, Checkpoint, Input, Output, - benchmark::{MatchContext, Score}, - utils::fmt::KeyValue, -}; -use rand::{Rng, SeedableRng, distr::Uniform, rngs::StdRng}; -use serde::{Deserialize, Serialize}; - -use diskann_inmem::integration::store::{Config, Store}; - -/// Maximum number of concurrent reader guards supported by the epoch registry. -const GUARD_CAPACITY: usize = 256; - -/// Number of slots a reader inspects per guard. Kept small so guards are short-lived, -/// allowing the epoch to advance and reclamation to make progress. -const READER_WINDOW: usize = 64; - -/// Number of times a reader re-reads its window within a single guard. Re-reading is what -/// exercises the value-stability and no-resurrection invariants. -const READER_PASSES: usize = 4; - -/// How often (in retirer iterations) a retirer attempts to reclaim retired slots. -const RECLAIM_EVERY: u64 = 16; - -/////////// -// Input // -/////////// - -/// Configuration for a [`StoreStress`] run. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct StoreStressInput { - /// Number of reader threads. Must be below [`GUARD_CAPACITY`]. - readers: usize, - /// Number of writer threads. - writers: usize, - /// Number of retirer threads. - retirers: usize, - /// Number of writable (non-frozen) slots. - capacity: usize, - /// Bytes per entry. Must be a non-zero multiple of 8 (the stamp lane width). - entry_bytes: usize, - /// The number of epoch guard slots. - epoch_guard_slots: usize, - /// The capacity of the freelist recycle queue capacity. - freelist_recycle_capacity: usize, - /// Retirers only retire while the live published population exceeds this watermark. - low_watermark: usize, - /// Wall-clock cap for the run, in seconds. Zero means unbounded (rely on `max_ops`). - duration_secs: u64, - /// Total-operation cap across all worker threads. Zero means unbounded (rely on - /// `duration_secs`). - max_ops: u64, - /// Seed for the worker pseudo-random number generators. - seed: u64, -} - -impl StoreStressInput { - fn check(self) -> anyhow::Result { - if self.readers == 0 || self.writers == 0 { - anyhow::bail!("`readers` and `writers` must be non-zero"); - } - if self.readers >= GUARD_CAPACITY { - anyhow::bail!( - "`readers` ({}) must be below the epoch guard capacity ({GUARD_CAPACITY})", - self.readers, - ); - } - if self.capacity == 0 { - anyhow::bail!("`capacity` must be non-zero"); - } - if self.entry_bytes == 0 || !self.entry_bytes.is_multiple_of(8) { - anyhow::bail!( - "`entry_bytes` ({}) must be a non-zero multiple of 8", - self.entry_bytes, - ); - } - if self.low_watermark > self.capacity { - anyhow::bail!( - "`low_watermark` ({}) must not exceed `capacity` ({})", - self.low_watermark, - self.capacity, - ); - } - if self.duration_secs == 0 && self.max_ops == 0 { - anyhow::bail!("at least one of `duration_secs` or `max_ops` must be non-zero"); - } - Ok(self) - } -} - -impl Input for StoreStressInput { - type Raw = Self; - - fn tag() -> &'static str { - "store-stress" - } - - fn from_raw(raw: Self::Raw, _checker: &mut Checker) -> anyhow::Result { - Self::check(raw) - } - - fn serialize(&self) -> anyhow::Result { - Ok(serde_json::to_value(self)?) - } - - fn example() -> Self::Raw { - StoreStressInput { - readers: 8, - writers: 4, - retirers: 2, - capacity: 4096, - entry_bytes: 128, - epoch_guard_slots: 256, - freelist_recycle_capacity: 1024, - low_watermark: 1024, - duration_secs: 5, - max_ops: 50_000_000, - seed: 0xA5A5_1234_DEAD_BEEF, - } - } -} - -impl std::fmt::Display for StoreStressInput { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let mut kv = KeyValue::new(); - kv.push("readers", &self.readers); - kv.push("writers", &self.writers); - kv.push("retirers", &self.retirers); - kv.push("capacity", &self.capacity); - kv.push("entry_bytes", &self.entry_bytes); - kv.push("epoch_guard_slots", &self.epoch_guard_slots); - kv.push("freelist_recycle_capacity", &self.freelist_recycle_capacity); - kv.push("low_watermark", &self.low_watermark); - kv.push("duration_secs", &self.duration_secs); - kv.push("max_ops", &self.max_ops); - kv.push("seed", &self.seed); - write!(f, "{}", kv) - } -} - -//////////// -// Output // -//////////// - -/// Summary statistics produced by a [`StoreStress`] run. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct StoreStressStats { - elapsed_secs: f64, - reads: u64, - acquires_ok: u64, - acquires_fail: u64, - retires_ok: u64, - retires_fail: u64, - reclaims: u64, - /// Observed `readable -> unreadable` transitions across all reader guards. - transitions: u64, - /// Peak observed live (published, not-yet-retired) population. - peak_live: usize, -} - -impl std::fmt::Display for StoreStressStats { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let mut kv = KeyValue::new(); - kv.push("elapsed_secs", &self.elapsed_secs); - kv.push("reads", &self.reads); - kv.push("acquires_ok", &self.acquires_ok); - kv.push("acquires_fail", &self.acquires_fail); - kv.push("retires_ok", &self.retires_ok); - kv.push("retires_fail", &self.retires_fail); - kv.push("reclaims", &self.reclaims); - kv.push("transitions", &self.transitions); - kv.push("peak_live", &self.peak_live); - write!(f, "{}", kv) - } -} - -///////////// -// Payload // -///////////// - -/// Fill `buf` with `stamp` replicated across every 8-byte lane. -fn write_stamp(buf: &mut [u8], stamp: u64) { - let bytes = stamp.to_ne_bytes(); - for lane in buf.chunks_exact_mut(8) { - lane.copy_from_slice(&bytes); - } -} - -/// Read the stamp from `buf`, returning `Err` if any 8-byte lane disagrees (a torn read). -fn read_stamp(buf: &[u8]) -> Result { - let (lanes, _) = buf.as_chunks::<8>(); - let mut lanes = lanes.iter(); - let first = u64::from_ne_bytes(*lanes.next().ok_or(())?); - for lane in lanes { - if u64::from_ne_bytes(*lane) != first { - return Err(()); - } - } - Ok(first) -} - -//////////////// -// Invariants // -//////////////// - -/// Per-guard observation of a single slot. -#[derive(Debug, Clone, Copy)] -enum SlotObservations { - /// The slot was observed readable with the given stamp. - Readable(u64), - /// The slot was observed readable and then became unreadable (retired). - Retired, -} - -/// Feed a single observation of slot `i` into the per-guard checker, recording a violation -/// on the shared state if a safety invariant is broken. -fn observe( - shared: &Shared, - observed: &mut HashMap, - i: usize, - read: Option<&[u8]>, -) { - match (observed.get(&i).copied(), read) { - // Not yet observed readable; an unreadable slot tells us nothing actionable. - (None, None) => {} - // First readable observation: record the stamp (after a tearing check). - (None, Some(bytes)) => match read_stamp(bytes) { - Ok(stamp) => { - observed.insert(i, SlotObservations::Readable(stamp)); - } - Err(()) => record_violation(shared, format!("torn read at slot {i}")), - }, - // Still readable: the value must be identical and untorn. - (Some(SlotObservations::Readable(prev)), Some(bytes)) => match read_stamp(bytes) { - Ok(stamp) if stamp != prev => record_violation( - shared, - format!("slot {i} value changed within guard: {prev} -> {stamp}"), - ), - Ok(_) => {} - Err(()) => record_violation(shared, format!("torn read at slot {i}")), - }, - // Readable -> unreadable: an allowed, terminal transition. - (Some(SlotObservations::Readable(_)), None) => { - observed.insert(i, SlotObservations::Retired); - shared.transitions.fetch_add(1, Relaxed); - } - // Resurrection: a slot that retired came back to life within the same guard. - (Some(SlotObservations::Retired), Some(_)) => record_violation( - shared, - format!("resurrection at slot {i}: unreadable -> readable within one guard"), - ), - (Some(SlotObservations::Retired), None) => {} - } -} - -//////////// -// Shared // -//////////// - -struct Local<'a> { - counter: u64, - parent: &'a AtomicU64, -} - -impl<'a> Local<'a> { - fn new(parent: &'a AtomicU64) -> Self { - Self { counter: 0, parent } - } - - fn add(&mut self, by: u64) { - self.counter += by; - - if self.counter >= 2048 { - self.parent.fetch_add(self.counter, Relaxed); - self.counter = 0; - } - } -} - -impl Drop for Local<'_> { - fn drop(&mut self) { - self.parent.fetch_add(self.counter, Relaxed); - } -} - -struct LocalMax<'a> { - max: usize, - parent: &'a AtomicUsize, -} - -impl<'a> LocalMax<'a> { - fn new(parent: &'a AtomicUsize) -> Self { - Self { max: 0, parent } - } - - fn max(&mut self, m: usize) { - self.max = self.max.max(m); - } -} - -impl Drop for LocalMax<'_> { - fn drop(&mut self) { - self.parent.fetch_max(self.max, Relaxed); - } -} - -/// State shared by all worker threads for the duration of a run. -struct Shared { - store: Store, - slots: usize, - readable: Uniform, - writable: Uniform, - low_watermark: usize, - max_ops: u64, - deadline: Instant, - - stop: AtomicBool, - violation: Mutex>, - - stamp: AtomicU64, - live: AtomicUsize, - peak_live: AtomicUsize, - - ops: AtomicU64, - reads: AtomicU64, - acquires_ok: AtomicU64, - acquires_fail: AtomicU64, - retires_ok: AtomicU64, - retires_fail: AtomicU64, - reclaims: AtomicU64, - transitions: AtomicU64, -} - -/// Record an observed invariant violation and signal all workers to stop. -fn record_violation(shared: &Shared, message: String) { - let mut slot = shared.violation.lock().unwrap(); - slot.push(message); - shared.stop.store(true, Relaxed); -} - -/// Return `true` once any termination condition is met. -fn should_stop(shared: &Shared) -> bool { - shared.stop.load(Relaxed) - || shared.ops.load(Relaxed) >= shared.max_ops - || Instant::now() >= shared.deadline -} - -///////////// -// Workers // -///////////// - -fn writer(shared: &Shared) { - let mut ops = Local::new(&shared.ops); - let mut acquires_ok = Local::new(&shared.acquires_ok); - let mut acquires_fail = Local::new(&shared.acquires_fail); - - let mut peak_live = LocalMax::new(&shared.peak_live); - - while !should_stop(shared) { - ops.add(1); - match shared.store.acquire() { - Some(mut writer) => { - let stamp = shared.stamp.fetch_add(1, Relaxed); - write_stamp(writer.as_mut_slice(), stamp); - writer.publish(); - - let live = shared.live.fetch_add(1, Relaxed) + 1; - peak_live.max(live); - acquires_ok.add(1); - } - None => { - acquires_fail.add(1); - std::thread::yield_now(); - } - } - } -} - -fn retirer(shared: &Shared, seed: u64) { - let mut rng = StdRng::seed_from_u64(seed); - let mut iteration: u64 = 0; - - let mut ops = Local::new(&shared.ops); - let mut retires_ok = Local::new(&shared.retires_ok); - let mut retires_fail = Local::new(&shared.retires_fail); - let mut reclaims = Local::new(&shared.reclaims); - - while !should_stop(shared) { - ops.add(1); - iteration += 1; - - // Flow control: keep a steady readable population. - if shared.live.load(Relaxed) > shared.low_watermark { - let i = rng.sample(shared.writable); - if shared.store.retire(i) { - shared.live.fetch_sub(1, Relaxed); - retires_ok.add(1); - } else { - retires_fail.add(1); - } - } - - if iteration.is_multiple_of(RECLAIM_EVERY) - && let Some(reclaimed) = shared.store.reclaim() - { - reclaims.add(reclaimed as u64); - } - - std::thread::yield_now(); - } -} - -fn reader(shared: &Shared, seed: u64) { - let mut rng = StdRng::seed_from_u64(seed); - let slots = shared.slots; - let window = READER_WINDOW.min(slots); - let mut observations = HashMap::with_capacity(window); - - let mut ops = Local::new(&shared.ops); - let mut reads = Local::new(&shared.reads); - - while !should_stop(shared) { - ops.add(1); - let Some(guard) = shared.store.reader() else { - // All guard slots are occupied; back off and retry. - std::thread::yield_now(); - continue; - }; - - observations.clear(); - let start = rng.sample(shared.readable); - for _ in 0..READER_PASSES { - for k in 0..window { - let i = (start + k) % slots; - observe(shared, &mut observations, i, guard.read(i)); - reads.add(1); - } - } - } -} - -/////////////// -// Benchmark // -/////////////// - -/// The store concurrency stress benchmark. -#[derive(Debug)] -pub struct StoreStress; - -impl Benchmark for StoreStress { - type Input = StoreStressInput; - type Output = StoreStressStats; - - fn try_match(&self, _input: &StoreStressInput, context: &MatchContext) -> Score { - context.success(0) - } - - fn description(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - f, - "concurrency stress test for the in-memory store (readers/writers/retirers)" - ) - } - - fn run( - &self, - input: &StoreStressInput, - _checkpoint: Checkpoint<'_>, - mut output: &mut dyn Output, - ) -> anyhow::Result { - let config = Config { - capacity: input.capacity, - entry_bytes: input.entry_bytes, - epoch_guard_slots: input.epoch_guard_slots, - freelist_recycle_capacity: input.freelist_recycle_capacity, - }; - - let store = Store::new(config); - let writable = store.writable(); - let slots = store.slots(); - let start = Instant::now(); - - let shared = Shared { - store, - slots, - readable: Uniform::new(0, slots)?, - writable: Uniform::try_from(writable)?, - low_watermark: input.low_watermark, - max_ops: if input.max_ops == 0 { - u64::MAX - } else { - input.max_ops - }, - deadline: if input.duration_secs == 0 { - // Effectively unbounded; the op cap terminates the run. - start + Duration::from_secs(u64::from(u32::MAX)) - } else { - start + Duration::from_secs(input.duration_secs) - }, - stop: AtomicBool::new(false), - violation: Mutex::new(Vec::new()), - // Stamp 0 is reserved for the zeroed frozen point. - stamp: AtomicU64::new(1), - live: AtomicUsize::new(0), - peak_live: AtomicUsize::new(0), - ops: AtomicU64::new(0), - reads: AtomicU64::new(0), - acquires_ok: AtomicU64::new(0), - acquires_fail: AtomicU64::new(0), - retires_ok: AtomicU64::new(0), - retires_fail: AtomicU64::new(0), - reclaims: AtomicU64::new(0), - transitions: AtomicU64::new(0), - }; - - writeln!(output, "{}", input)?; - - std::thread::scope(|scope| { - let shared = &shared; - for _ in 0..input.writers { - scope.spawn(move || writer(shared)); - } - for t in 0..input.retirers { - let seed = input.seed ^ (0x2000_0000 + t as u64); - scope.spawn(move || retirer(shared, seed)); - } - for t in 0..input.readers { - let seed = input.seed ^ (0x4000_0000 + t as u64); - scope.spawn(move || reader(shared, seed)); - } - }); - - let errors: Vec<_> = std::mem::take(&mut *shared.violation.lock().unwrap()); - if !errors.is_empty() { - anyhow::bail!("invariants violated: {:?}", errors); - } - - let elapsed = start.elapsed(); - let stats = StoreStressStats { - elapsed_secs: elapsed.as_secs_f64(), - reads: shared.reads.load(Relaxed), - acquires_ok: shared.acquires_ok.load(Relaxed), - acquires_fail: shared.acquires_fail.load(Relaxed), - retires_ok: shared.retires_ok.load(Relaxed), - retires_fail: shared.retires_fail.load(Relaxed), - reclaims: shared.reclaims.load(Relaxed), - transitions: shared.transitions.load(Relaxed), - peak_live: shared.peak_live.load(Relaxed), - }; - - writeln!(output, "{}", stats)?; - Ok(stats) - } -} - -/////////// -// Tests // -/////////// - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn make_sure_example_parses() { - let _ = StoreStressInput::check(StoreStressInput::example()).unwrap(); - } -} diff --git a/diskann-inmem/integration/store/checked.rs b/diskann-inmem/integration/store/checked.rs new file mode 100644 index 0000000000..9a139349ad --- /dev/null +++ b/diskann-inmem/integration/store/checked.rs @@ -0,0 +1,224 @@ +/* + * Copyright (c) Microsoft Corporation. + * Licensed under the MIT license. + */ + +use std::{collections::HashMap, io::Write, sync::atomic::Ordering::Relaxed}; + +use diskann_benchmark_runner as dbr; +use diskann_inmem::integration::store::checked; +use serde::{Deserialize, Serialize}; + +pub(super) fn register(registry: &mut dbr::Registry) -> Result<(), dbr::RegistryError> { + registry.register("store-stress-test-checked", Stress) +} + +/// Configuration for a [`Stress`] run. +#[derive(Debug, Clone, Serialize, Deserialize)] +struct Input { + /// Shared stress test setup. + setup: super::Setup, +} + +impl Input { + fn check(self) -> anyhow::Result { + self.setup.check()?; + Ok(self) + } +} + +impl dbr::Input for Input { + type Raw = Self; + + fn tag() -> &'static str { + "store-stress-checked" + } + + fn from_raw(raw: Self::Raw, _checker: &mut dbr::Checker) -> anyhow::Result { + raw.check() + } + + fn serialize(&self) -> anyhow::Result { + Ok(serde_json::to_value(self)?) + } + + fn example() -> Self::Raw { + Input { + setup: super::Setup::example(), + } + } +} + +impl std::fmt::Display for Input { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let Input { setup } = self; + + let mut kv = dbr::utils::fmt::KeyValue::new(); + kv.push("setup", &setup); + write!(f, "{}", kv) + } +} + +#[derive(Debug)] +struct Stress; + +impl dbr::Benchmark for Stress { + type Input = Input; + type Output = super::Stats; + + fn try_match( + &self, + _input: &Input, + context: &dbr::benchmark::MatchContext, + ) -> dbr::benchmark::Score { + context.success(0) + } + + fn description(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "concurrency stress test for the checked in-memory store") + } + + fn run( + &self, + input: &Input, + _checkpoint: dbr::Checkpoint<'_>, + mut output: &mut dyn dbr::Output, + ) -> anyhow::Result { + let config = checked::Config { + capacity: input.setup.capacity, + epoch_guard_slots: input.setup.epoch_guard_slots, + freelist_recycle_capacity: input.setup.freelist_recycle_capacity, + }; + + writeln!(output, "{}", input)?; + let stats = super::run_benchmark(checked::Store::new(config), &input.setup)?; + writeln!(output, "{}", stats)?; + Ok(stats) + } +} + +impl super::Testable for checked::Store { + type Writer<'a> = checked::Writer<'a>; + type ReaderState<'a> = ReaderState<'a>; + + fn writer(&self) -> Option> { + ::acquire(self) + } + + fn reader_state<'a>( + &'a self, + capacity_hint: usize, + shared: &'a super::Shared, + ) -> Self::ReaderState<'a> { + ReaderState { + store: self, + shared, + capacity_hint, + } + } + + fn retire(&self, i: usize) -> bool { + ::retire(self, i) + } + + fn reclaim(&self) -> Option { + ::reclaim(self) + } + + fn readable_slots(&self) -> usize { + ::readable_slots(self) + } + + fn writable_slots(&self) -> usize { + ::writable_slots(self) + } +} + +impl super::Writer for checked::Writer<'_> { + fn write(mut self, stamp: u64) { + self.set(stamp); + self.publish(); + } +} + +#[derive(Debug)] +pub(super) struct ReaderState<'a> { + store: &'a checked::Store, + shared: &'a super::Shared, + capacity_hint: usize, +} + +impl super::ReaderState for ReaderState<'_> { + type Reader<'a> = Reader<'a>; + + fn try_with_reader(&mut self, f: F) -> bool + where + F: FnOnce(Self::Reader<'_>), + { + let Some(reader) = self.store.reader() else { + return false; + }; + let observed = HashMap::with_capacity(self.capacity_hint); + + f(Reader { + reader: &reader, + observed, + shared: self.shared, + }); + + true + } +} + +#[derive(Debug)] +pub(super) struct Reader<'a> { + observed: HashMap>, + reader: &'a checked::Reader<'a>, + shared: &'a super::Shared, +} + +impl super::Reader for Reader<'_> { + /// Feed a single observation of slot `i` into the per-guard checker, recording a + /// violation on the shared state if a safety invariant is broken. + fn observe(&mut self, i: usize) { + let read = self.reader.read(i); + let observed = self.observed.get(&i).map(|v| v.get()); + + match (observed, read) { + // Not yet observed readable; an unreadable slot tells us nothing actionable. + (None, None) => {} + // First readable observation: record the stamp (after a tearing check). + (None, Some(value)) => { + self.observed.insert(i, value); + } + // Still readable: the value must be identical and untorn. + (Some(previous), Some(value)) => { + if previous != value.get() { + self.shared.record_violation(format!( + "slot {i} value changed within guard: {} -> {}", + previous, + value.get(), + )) + } + } + // Readable -> unreadable: an allowed, terminal transition. + (Some(_), None) => { + self.shared.transitions.fetch_add(1, Relaxed); + } + } + } +} + +/////////// +// Tests // +/////////// + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn make_sure_example_parses() { + let _ = Input::check(::example()).unwrap(); + } +} diff --git a/diskann-inmem/integration/store/invasive.rs b/diskann-inmem/integration/store/invasive.rs new file mode 100644 index 0000000000..ede89d8111 --- /dev/null +++ b/diskann-inmem/integration/store/invasive.rs @@ -0,0 +1,283 @@ +/* + * Copyright (c) Microsoft Corporation. + * Licensed under the MIT license. + */ + +use std::{collections::HashMap, io::Write, sync::atomic::Ordering::Relaxed}; + +use diskann_benchmark_runner as dbr; +use diskann_inmem::integration::store::invasive; +use serde::{Deserialize, Serialize}; + +pub(super) fn register(registry: &mut dbr::Registry) -> Result<(), dbr::RegistryError> { + registry.register("invasive-store-stress-test", Stress) +} + +/// Configuration for a [`Stress`] run. +#[derive(Debug, Clone, Serialize, Deserialize)] +struct Input { + /// Shared stress test setup. + setup: super::Setup, + + /// Bytes per entry. Must be a non-zero multiple of 8 (the stamp lane width). + entry_bytes: usize, +} + +impl Input { + fn check(self) -> anyhow::Result { + self.setup.check()?; + + if self.entry_bytes == 0 || !self.entry_bytes.is_multiple_of(8) { + anyhow::bail!( + "`entry_bytes` ({}) must be a non-zero multiple of 8", + self.entry_bytes, + ); + } + + Ok(self) + } +} + +impl dbr::Input for Input { + type Raw = Self; + + fn tag() -> &'static str { + "store-stress-invasive" + } + + fn from_raw(raw: Self::Raw, _checker: &mut dbr::Checker) -> anyhow::Result { + raw.check() + } + + fn serialize(&self) -> anyhow::Result { + Ok(serde_json::to_value(self)?) + } + + fn example() -> Self::Raw { + Input { + setup: super::Setup::example(), + entry_bytes: 128, + } + } +} + +impl std::fmt::Display for Input { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let Input { setup, entry_bytes } = self; + + let mut kv = dbr::utils::fmt::KeyValue::new(); + kv.push("setup", &setup); + kv.push("entry_bytes", &entry_bytes); + write!(f, "{}", kv) + } +} + +#[derive(Debug)] +struct Stress; + +impl dbr::Benchmark for Stress { + type Input = Input; + type Output = super::Stats; + + fn try_match( + &self, + _input: &Input, + context: &dbr::benchmark::MatchContext, + ) -> dbr::benchmark::Score { + context.success(0) + } + + fn description(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "concurrency stress test for the invasive in-memory store" + ) + } + + fn run( + &self, + input: &Input, + _checkpoint: dbr::Checkpoint<'_>, + mut output: &mut dyn dbr::Output, + ) -> anyhow::Result { + let config = invasive::Config { + capacity: input.setup.capacity, + entry_bytes: input.entry_bytes, + epoch_guard_slots: input.setup.epoch_guard_slots, + freelist_recycle_capacity: input.setup.freelist_recycle_capacity, + }; + + writeln!(output, "{}", input)?; + let stats = super::run_benchmark(invasive::Store::new(config), &input.setup)?; + writeln!(output, "{}", stats)?; + Ok(stats) + } +} + +/// Per-guard observation of a single slot. +#[derive(Debug, Clone, Copy)] +enum SlotObservations { + /// The slot was observed readable with the given stamp. + Readable(u64), + /// The slot was observed readable and then became unreadable (retired). + Retired, +} + +/// Fill `buf` with `stamp` replicated across every 8-byte lane. +fn write_stamp(buf: &mut [u8], stamp: u64) { + let bytes = stamp.to_ne_bytes(); + for lane in buf.chunks_exact_mut(8) { + lane.copy_from_slice(&bytes); + } +} + +/// Read the stamp from `buf`, returning `Err` if any 8-byte lane disagrees (a torn read). +fn read_stamp(buf: &[u8]) -> Result { + let (lanes, _) = buf.as_chunks::<8>(); + let mut lanes = lanes.iter(); + let first = u64::from_ne_bytes(*lanes.next().ok_or(())?); + for lane in lanes { + if u64::from_ne_bytes(*lane) != first { + return Err(()); + } + } + Ok(first) +} + +impl super::Testable for invasive::Store { + type Writer<'a> = invasive::Writer<'a>; + type ReaderState<'a> = ReaderState<'a>; + + fn writer(&self) -> Option> { + ::acquire(self) + } + + fn reader_state<'a>( + &'a self, + capacity_hint: usize, + shared: &'a super::Shared, + ) -> Self::ReaderState<'a> { + let observed = HashMap::with_capacity(capacity_hint); + ReaderState { + store: self, + observed, + shared, + } + } + + fn retire(&self, i: usize) -> bool { + ::retire(self, i) + } + + fn reclaim(&self) -> Option { + ::reclaim(self) + } + + fn readable_slots(&self) -> usize { + ::readable_slots(self) + } + + fn writable_slots(&self) -> usize { + ::writable_slots(self) + } +} + +impl super::Writer for invasive::Writer<'_> { + fn write(mut self, stamp: u64) { + write_stamp(self.as_mut_slice(), stamp); + self.publish(); + } +} + +#[derive(Debug)] +pub(super) struct ReaderState<'a> { + store: &'a invasive::Store, + observed: HashMap, + shared: &'a super::Shared, +} + +impl super::ReaderState for ReaderState<'_> { + type Reader<'a> = Reader<'a>; + + fn try_with_reader(&mut self, f: F) -> bool + where + F: FnOnce(Self::Reader<'_>), + { + let Some(reader) = self.store.reader() else { + return false; + }; + self.observed.clear(); + + f(Reader { + reader, + observed: &mut self.observed, + shared: self.shared, + }); + + true + } +} + +#[derive(Debug)] +pub(super) struct Reader<'a> { + reader: invasive::Reader<'a>, + observed: &'a mut HashMap, + shared: &'a super::Shared, +} + +impl super::Reader for Reader<'_> { + /// Feed a single observation of slot `i` into the per-guard checker, recording a + /// violation on the shared state if a safety invariant is broken. + fn observe(&mut self, i: usize) { + let read = self.reader.read(i); + let observed = self.observed.get(&i).copied(); + + match (observed, read) { + // Not yet observed readable; an unreadable slot tells us nothing actionable. + (None, None) => {} + // First readable observation: record the stamp (after a tearing check). + (None, Some(bytes)) => match read_stamp(bytes) { + Ok(stamp) => { + self.observed.insert(i, SlotObservations::Readable(stamp)); + } + Err(()) => self + .shared + .record_violation(format!("torn read at slot {i}")), + }, + // Still readable: the value must be identical and untorn. + (Some(SlotObservations::Readable(prev)), Some(bytes)) => match read_stamp(bytes) { + Ok(stamp) if stamp != prev => self.shared.record_violation(format!( + "slot {i} value changed within guard: {prev} -> {stamp}" + )), + Ok(_) => {} + Err(()) => self + .shared + .record_violation(format!("torn read at slot {i}")), + }, + // Readable -> unreadable: an allowed, terminal transition. + (Some(SlotObservations::Readable(_)), None) => { + self.observed.insert(i, SlotObservations::Retired); + self.shared.transitions.fetch_add(1, Relaxed); + } + // Resurrection: a slot that retired came back to life within the same guard. + (Some(SlotObservations::Retired), Some(_)) => self.shared.record_violation(format!( + "resurrection at slot {i}: unreadable -> readable within one guard" + )), + (Some(SlotObservations::Retired), None) => {} + } + } +} + +/////////// +// Tests // +/////////// + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn make_sure_example_parses() { + let _ = Input::check(::example()).unwrap(); + } +} diff --git a/diskann-inmem/integration/store/mod.rs b/diskann-inmem/integration/store/mod.rs new file mode 100644 index 0000000000..8e6991f709 --- /dev/null +++ b/diskann-inmem/integration/store/mod.rs @@ -0,0 +1,538 @@ +/* + * Copyright (c) Microsoft Corporation. + * Licensed under the MIT license. + */ + +//! Concurrency stress test for the in-memory [stores](diskann_inmem::integration::store). +//! +//! Reader, writer, and retirer threads hammer the epoch-based store concurrently while a +//! per-guard invariant checker verifies the store's safety guarantees: +//! +//! 1. Reads are never torn. +//! 2. A readable value is stable for the lifetime of a single reader guard. +//! 3. A slot never resurrects (`readable -> unreadable -> readable`) within one guard. +//! +//! This module exposes shared functionality that is instantiated by different implementations. + +#![expect( + clippy::unwrap_used, + reason = "this code works mainly as an integration test" +)] + +use std::{ + sync::{ + Mutex, + atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering::Relaxed}, + }, + time::{Duration, Instant}, +}; + +use diskann_benchmark_runner::{Registry, RegistryError, utils::fmt::KeyValue}; +use rand::{Rng, SeedableRng, distr::Uniform, rngs::StdRng}; +use serde::{Deserialize, Serialize}; + +/// Number of slots a reader inspects per guard. Kept small so guards are short-lived, +/// allowing the epoch to advance and reclamation to make progress. +const READER_WINDOW: usize = 64; + +/// Number of times a reader re-reads its window within a single guard. Re-reading is what +/// exercises the value-stability and no-resurrection invariants. +const READER_PASSES: usize = 4; + +/// How often (in retirer iterations) a retirer attempts to reclaim retired slots. +const RECLAIM_EVERY: u64 = 16; + +mod checked; +mod invasive; + +pub(super) fn register(registry: &mut Registry) -> Result<(), RegistryError> { + invasive::register(registry)?; + checked::register(registry)?; + + Ok(()) +} + +/////////// +// Input // +/////////// + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Setup { + /// Number of reader threads. Must be below `epoch_guard_slots`. + readers: usize, + /// Number of writer threads. + writers: usize, + /// Number of retirer threads. + retirers: usize, + /// Number of writable (non-frozen) slots. + capacity: usize, + /// The number of epoch guard slots. + epoch_guard_slots: usize, + /// The capacity of the freelist recycle queue capacity. + freelist_recycle_capacity: usize, + /// Retirers only retire while the live published population exceeds this watermark. + low_watermark: usize, + /// Wall-clock cap for the run, in seconds. Zero means unbounded (rely on `max_ops`). + duration_secs: u64, + /// Total-operation cap across all worker threads. Zero means unbounded (rely on + /// `duration_secs`). + max_ops: u64, + /// Seed for the worker pseudo-random number generators. + seed: u64, +} + +impl Setup { + fn check(&self) -> anyhow::Result<()> { + if self.readers == 0 || self.writers == 0 { + anyhow::bail!("`readers` and `writers` must be non-zero"); + } + if self.readers >= self.epoch_guard_slots { + anyhow::bail!( + "`readers` ({}) must be below the epoch guard capacity ({})", + self.readers, + self.epoch_guard_slots, + ); + } + if self.capacity == 0 { + anyhow::bail!("`capacity` must be non-zero"); + } + if self.low_watermark > self.capacity { + anyhow::bail!( + "`low_watermark` ({}) must not exceed `capacity` ({})", + self.low_watermark, + self.capacity, + ); + } + if self.duration_secs == 0 && self.max_ops == 0 { + anyhow::bail!("at least one of `duration_secs` or `max_ops` must be non-zero"); + } + + Ok(()) + } + + fn example() -> Self { + Setup { + readers: 8, + writers: 4, + retirers: 2, + capacity: 4096, + epoch_guard_slots: 256, + freelist_recycle_capacity: 1024, + low_watermark: 1024, + duration_secs: 5, + max_ops: 50_000_000, + seed: 0xA5A5_1234_DEAD_BEEF, + } + } +} + +impl std::fmt::Display for Setup { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut kv = KeyValue::new(); + kv.push("readers", &self.readers); + kv.push("writers", &self.writers); + kv.push("retirers", &self.retirers); + kv.push("capacity", &self.capacity); + kv.push("epoch_guard_slots", &self.epoch_guard_slots); + kv.push("freelist_recycle_capacity", &self.freelist_recycle_capacity); + kv.push("low_watermark", &self.low_watermark); + kv.push("duration_secs", &self.duration_secs); + kv.push("max_ops", &self.max_ops); + kv.push("seed", &self.seed); + write!(f, "{}", kv) + } +} + +/// A testable store. +/// +/// Readers are split into a [`ReaderState`], which is used to produce a shorter lived [`Reader`]. +/// +/// The reason for this is two fold: +/// +/// 1. We rely on [`Reader`]s being dropped to allow epochs to advance. +/// 2. A separate [`ReaderState`] allows correctness checking data structures (e.g. hash maps) +/// to be allocated once and used for the duration of the test. +/// +/// Since the goal is to hammer the underlying store as hard as possible, amortizing +/// allocations makes a non-negligible difference. +trait Testable: std::fmt::Debug + Sized + Sync { + /// The writer. + type Writer<'a>: Writer + where + Self: 'a; + + /// Shared reader-state to enable allocation amortization. + type ReaderState<'a>: ReaderState + where + Self: 'a; + + /// Construct a [`Writer`] into a slot. Returns `None` is an available slot could not + /// be found. + fn writer(&self) -> Option>; + + /// Create the [`ReaderState`]. This will be called once per reader thread. + fn reader_state<'a>( + &'a self, + capacity_hint: usize, + shared: &'a Shared, + ) -> Self::ReaderState<'a>; + + /// Attempt to retire slot `i`. Return `true` on success, otherwise return `false`. + fn retire(&self, i: usize) -> bool; + + /// Attempt to advance the epoch and reclaim retired slots. + /// + /// Return `None` if we failed to advance the epoch. Otherwise, return the number of + /// slots reclaimed. + fn reclaim(&self) -> Option; + + /// Return the number of readable slots. Assume indices `[0..readable_slots)` are valid + /// for reading. + fn readable_slots(&self) -> usize; + + /// Return the number of writable slots. Assume indices `[0..writable_slots)` are valid + /// for writing. + fn writable_slots(&self) -> usize; +} + +/// A writable slot. +trait Writer: std::fmt::Debug { + /// Perform any writes, using `stamp` as a unique tag. + fn write(self, stamp: u64); +} + +/// Amortized shared state for reader tasks. +trait ReaderState: std::fmt::Debug { + /// The type of the [`Reader`]. + type Reader<'a>: Reader; + + /// Attempt to run the closure `f` on a [`Reader`] into the state's parent store. + /// + /// Return `true` if a reader was obtained and `f` was called. Otherwise return `false`. + /// + /// The callback style mechanism is used to allow implementations to stack-allocate some + /// variables. This allows validation implementations to hold onto references into the + /// parent store by doing the following: + /// + /// 1. Stack allocate the internal "reader" to the store. The reader will have an epoch + /// guard. + /// + /// 2. Borrow from that reader to construct [`Self::Reader`], allowing [`Self::Reader`] + /// to borrow items directly from the internal reader. + #[must_use] + fn try_with_reader(&mut self, f: F) -> bool + where + F: FnOnce(Self::Reader<'_>); +} + +/// A read validator for a [`Testable`]. +trait Reader: std::fmt::Debug { + /// Observe the state of `i`, panicking if an invalid transition has been observed. + fn observe(&mut self, i: usize); +} + +//////////// +// Output // +//////////// + +/// Summary statistics produced by a [`Shared`] run. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Stats { + elapsed_secs: f64, + reads: u64, + acquires_ok: u64, + acquires_fail: u64, + retires_ok: u64, + retires_fail: u64, + reclaims: u64, + /// Observed `readable -> unreadable` transitions across all reader guards. + transitions: u64, + /// Peak observed live (published, not-yet-retired) population. + peak_live: usize, +} + +impl std::fmt::Display for Stats { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut kv = KeyValue::new(); + kv.push("elapsed_secs", &self.elapsed_secs); + kv.push("reads", &self.reads); + kv.push("acquires_ok", &self.acquires_ok); + kv.push("acquires_fail", &self.acquires_fail); + kv.push("retires_ok", &self.retires_ok); + kv.push("retires_fail", &self.retires_fail); + kv.push("reclaims", &self.reclaims); + kv.push("transitions", &self.transitions); + kv.push("peak_live", &self.peak_live); + write!(f, "{}", kv) + } +} + +//////////// +// Shared // +//////////// + +struct Local<'a> { + counter: u64, + parent: &'a AtomicU64, +} + +impl<'a> Local<'a> { + fn new(parent: &'a AtomicU64) -> Self { + Self { counter: 0, parent } + } + + fn add(&mut self, by: u64) { + self.counter += by; + + if self.counter >= 2048 { + self.parent.fetch_add(self.counter, Relaxed); + self.counter = 0; + } + } +} + +impl Drop for Local<'_> { + fn drop(&mut self) { + self.parent.fetch_add(self.counter, Relaxed); + } +} + +struct LocalMax<'a> { + max: usize, + parent: &'a AtomicUsize, +} + +impl<'a> LocalMax<'a> { + fn new(parent: &'a AtomicUsize) -> Self { + Self { max: 0, parent } + } + + fn max(&mut self, m: usize) { + self.max = self.max.max(m); + } +} + +impl Drop for LocalMax<'_> { + fn drop(&mut self) { + self.parent.fetch_max(self.max, Relaxed); + } +} + +fn run_benchmark(store: T, setup: &Setup) -> anyhow::Result +where + T: Testable, +{ + let writable = store.writable_slots(); + let readable = store.readable_slots(); + let start = Instant::now(); + + let shared = Shared { + store, + slots: readable, + readable: Uniform::new(0, readable)?, + writable: Uniform::new(0, writable)?, + low_watermark: setup.low_watermark, + max_ops: if setup.max_ops == 0 { + u64::MAX + } else { + setup.max_ops + }, + deadline: if setup.duration_secs == 0 { + // Effectively unbounded; the op cap terminates the run. + start + Duration::from_secs(u64::from(u32::MAX)) + } else { + start + Duration::from_secs(setup.duration_secs) + }, + stop: AtomicBool::new(false), + violation: Mutex::new(Vec::new()), + // Stamp 0 is reserved for the zeroed frozen point. + stamp: AtomicU64::new(1), + live: AtomicUsize::new(0), + peak_live: AtomicUsize::new(0), + ops: AtomicU64::new(0), + reads: AtomicU64::new(0), + acquires_ok: AtomicU64::new(0), + acquires_fail: AtomicU64::new(0), + retires_ok: AtomicU64::new(0), + retires_fail: AtomicU64::new(0), + reclaims: AtomicU64::new(0), + transitions: AtomicU64::new(0), + }; + + std::thread::scope(|scope| { + let shared = &shared; + for _ in 0..setup.writers { + scope.spawn(move || shared.writer()); + } + for t in 0..setup.retirers { + let seed = setup.seed ^ (0x2000_0000 + t as u64); + scope.spawn(move || shared.retirer(seed)); + } + for t in 0..setup.readers { + let seed = setup.seed ^ (0x4000_0000 + t as u64); + scope.spawn(move || shared.reader(seed)); + } + }); + + let errors: Vec<_> = std::mem::take(&mut *shared.violation.lock().unwrap()); + if !errors.is_empty() { + anyhow::bail!("invariants violated: {:?}", errors); + } + + let elapsed = start.elapsed(); + let stats = Stats { + elapsed_secs: elapsed.as_secs_f64(), + reads: shared.reads.load(Relaxed), + acquires_ok: shared.acquires_ok.load(Relaxed), + acquires_fail: shared.acquires_fail.load(Relaxed), + retires_ok: shared.retires_ok.load(Relaxed), + retires_fail: shared.retires_fail.load(Relaxed), + reclaims: shared.reclaims.load(Relaxed), + transitions: shared.transitions.load(Relaxed), + peak_live: shared.peak_live.load(Relaxed), + }; + + Ok(stats) +} + +/// State shared by all worker threads for the duration of a run. +#[derive(Debug)] +struct Shared { + store: T, + slots: usize, + readable: Uniform, + writable: Uniform, + low_watermark: usize, + max_ops: u64, + deadline: Instant, + + stop: AtomicBool, + violation: Mutex>, + + stamp: AtomicU64, + live: AtomicUsize, + peak_live: AtomicUsize, + + ops: AtomicU64, + reads: AtomicU64, + acquires_ok: AtomicU64, + acquires_fail: AtomicU64, + retires_ok: AtomicU64, + retires_fail: AtomicU64, + reclaims: AtomicU64, + transitions: AtomicU64, +} + +impl Shared +where + T: Testable, +{ + /// Record an observed invariant violation and signal all workers to stop. + fn record_violation(&self, message: String) { + let mut slot = self.violation.lock().unwrap(); + slot.push(message); + self.stop.store(true, Relaxed); + } + + /// Return `true` once any termination condition is met. + fn should_stop(&self) -> bool { + self.stop.load(Relaxed) + || self.ops.load(Relaxed) >= self.max_ops + || Instant::now() >= self.deadline + } + + //---------// + // Workers // + //---------// + + fn writer(&self) { + let mut ops = Local::new(&self.ops); + let mut acquires_ok = Local::new(&self.acquires_ok); + let mut acquires_fail = Local::new(&self.acquires_fail); + + let mut peak_live = LocalMax::new(&self.peak_live); + + while !self.should_stop() { + ops.add(1); + match self.store.writer() { + Some(writer) => { + let stamp = self.stamp.fetch_add(1, Relaxed); + writer.write(stamp); + + let live = self.live.fetch_add(1, Relaxed) + 1; + peak_live.max(live); + acquires_ok.add(1); + } + None => { + acquires_fail.add(1); + std::thread::yield_now(); + } + } + } + } + + fn retirer(&self, seed: u64) { + let mut rng = StdRng::seed_from_u64(seed); + let mut iteration: u64 = 0; + + let mut ops = Local::new(&self.ops); + let mut retires_ok = Local::new(&self.retires_ok); + let mut retires_fail = Local::new(&self.retires_fail); + let mut reclaims = Local::new(&self.reclaims); + + while !self.should_stop() { + ops.add(1); + iteration += 1; + + // Flow control: keep a steady readable population. + if self.live.load(Relaxed) > self.low_watermark { + let i = rng.sample(self.writable); + if self.store.retire(i) { + self.live.fetch_sub(1, Relaxed); + retires_ok.add(1); + } else { + retires_fail.add(1); + } + } + + if iteration.is_multiple_of(RECLAIM_EVERY) + && let Some(reclaimed) = self.store.reclaim() + { + reclaims.add(reclaimed as u64); + } + + std::thread::yield_now(); + } + } + + fn reader(&self, seed: u64) { + let mut rng = StdRng::seed_from_u64(seed); + let slots = self.slots; + let window = READER_WINDOW.min(slots); + + let mut ops = Local::new(&self.ops); + let mut reads = Local::new(&self.reads); + + let mut reader_state = self.store.reader_state(window, self); + + while !self.should_stop() { + ops.add(1); + + let succeeded = reader_state.try_with_reader(|mut reader| { + let start = rng.sample(self.readable); + for _ in 0..READER_PASSES { + for k in 0..window { + let i = (start + k) % slots; + reader.observe(i); + reads.add(1); + } + } + }); + + // All guard slots are occupied; back off and retry. + if !succeeded { + std::thread::yield_now(); + }; + } + } +} diff --git a/diskann-inmem/integration/support/datatype.rs b/diskann-inmem/integration/support/datatype.rs index fe61de5398..34a60190aa 100644 --- a/diskann-inmem/integration/support/datatype.rs +++ b/diskann-inmem/integration/support/datatype.rs @@ -7,6 +7,7 @@ use diskann_utils::{ sampling::medoid::ComputeMedoid, views::{Matrix, MatrixView, MutMatrixView}, }; +use diskann_wide::{cast_f16_to_f32, cast_f32_to_f16}; use half::f16; use serde::{Deserialize, Serialize}; use thiserror::Error; @@ -146,8 +147,8 @@ where } fn f32_to_f16(x: f32) -> Result { - let y = f16::from_f32(x); - let z = f32::from(y); + let y = cast_f32_to_f16(x); + let z = cast_f16_to_f32(y); if z != x { Err(()) } else { Ok(y) } } @@ -164,11 +165,11 @@ fn f32_to_i8(x: f32) -> Result { } fn f16_to_u8(x: f16) -> Result { - f32_to_u8(x.into()) + f32_to_u8(cast_f16_to_f32(x)) } fn f16_to_i8(x: f16) -> Result { - f32_to_i8(x.into()) + f32_to_i8(cast_f16_to_f32(x)) } impl<'a> SliceMut<'a> { @@ -192,14 +193,14 @@ impl<'a> SliceMut<'a> { match (self, rhs) { (SliceMut::F32(dst), Slice::F32(src)) => dst.copy_from_slice(src), - (SliceMut::F32(dst), Slice::F16(src)) => map(dst, src, |x| x.into()), + (SliceMut::F32(dst), Slice::F16(src)) => map(dst, src, cast_f16_to_f32), (SliceMut::F32(dst), Slice::U8(src)) => map(dst, src, |x| x.into()), (SliceMut::F32(dst), Slice::I8(src)) => map(dst, src, |x| x.into()), (SliceMut::F16(dst), Slice::F32(src)) => try_map(dst, src, f32_to_f16)?, (SliceMut::F16(dst), Slice::F16(src)) => dst.copy_from_slice(src), - (SliceMut::F16(dst), Slice::U8(src)) => map(dst, src, |x| x.into()), - (SliceMut::F16(dst), Slice::I8(src)) => map(dst, src, |x| x.into()), + (SliceMut::F16(dst), Slice::U8(src)) => map(dst, src, |x| cast_f32_to_f16(x.into())), + (SliceMut::F16(dst), Slice::I8(src)) => map(dst, src, |x| cast_f32_to_f16(x.into())), (SliceMut::U8(dst), Slice::F32(src)) => try_map(dst, src, f32_to_u8)?, (SliceMut::U8(dst), Slice::F16(src)) => try_map(dst, src, f16_to_u8)?, @@ -544,7 +545,7 @@ mod tests { SliceMut::from(dst.as_mut_slice()) .convert_lossless(Slice::from(src)) .unwrap(); - assert_eq!(dst, [f16::from_f32(-5.0), f16::from_f32(7.0)]); + assert_eq!(dst, [cast_f32_to_f16(-5.0), cast_f32_to_f16(7.0)]); } #[test] diff --git a/diskann-inmem/src/counters.rs b/diskann-inmem/src/counters.rs index b53940dc2c..5c10954268 100644 --- a/diskann-inmem/src/counters.rs +++ b/diskann-inmem/src/counters.rs @@ -22,8 +22,10 @@ mod inner { } } + // Must be public because it shows up in public APIs - but is unconstructable by + // downstream code. #[derive(Debug)] - pub(crate) struct LocalCounters<'a> { + pub struct LocalCounters<'a> { _marker: PhantomData<&'a ()>, } @@ -41,7 +43,6 @@ mod inner { pub(crate) fn query_distance(&mut self, _i: u64) {} pub(crate) fn distance_ref(&self, _i: u64) {} pub(crate) fn get_vector(&mut self, _i: u64) {} - pub(crate) fn get_vector_ref(&self, _i: u64) {} pub(crate) fn set_vector(&mut self, _i: u64) {} pub(crate) fn get_neighbors(&mut self, _i: u64) {} pub(crate) fn set_neighbors(&mut self, _i: u64) {} @@ -86,8 +87,10 @@ mod inner { } } + // Must be public because it shows up in public APIs - but is unconstructable by + // downstream code. #[derive(Debug)] - pub(crate) struct LocalCounters<'a> { + pub struct LocalCounters<'a> { query_distance: u64, // This fields needs to be `AtomicU64` because we increment in some loops where we // have to increment it behind a shared reference. @@ -132,10 +135,6 @@ mod inner { *self.get_vector.get_mut() += i; } - pub(crate) fn get_vector_ref(&self, i: u64) { - self.get_vector.fetch_add(i, Relaxed); - } - pub(crate) fn set_vector(&mut self, i: u64) { self.set_vector += i; } diff --git a/diskann-inmem/src/freelist.rs b/diskann-inmem/src/freelist.rs index 33e5521b6a..3f196473b8 100644 --- a/diskann-inmem/src/freelist.rs +++ b/diskann-inmem/src/freelist.rs @@ -17,6 +17,7 @@ //! bounded to conserve memory. //! //! ## Minted +//! //! If no slots live in the recycled queue, new slots can be "minted" up to the configured //! maximum. This simply tracks the maximum slot ID that has been yielded so far and returns //! the next one. diff --git a/diskann-inmem/src/ids.rs b/diskann-inmem/src/ids.rs index dcf41424a8..7fe6dee4be 100644 --- a/diskann-inmem/src/ids.rs +++ b/diskann-inmem/src/ids.rs @@ -13,6 +13,8 @@ use diskann::utils::IntoUsize; use parking_lot::{RwLock, RwLockWriteGuard}; use thiserror::Error; +use crate::num::Capacity; + const SHARD_SIZE: usize = 1024; /// Bidirectional mapping between an external id `I` and a dense internal `u32` id. @@ -27,19 +29,19 @@ where // Since we know the internal IDs are contiguous from `[0..self.capacity)`, we can // use a lighter-weight `backward` ID map than a full `DashMap`. backward: Vec]>>>, - capacity: usize, + capacity: Capacity, } impl IdMap where I: Hash + Eq, { - pub(crate) fn new(capacity: usize) -> Self { + pub(crate) fn new(capacity: Capacity) -> Self { let backward = std::iter::repeat_with(|| { let shard = std::iter::repeat_with(|| None).take(SHARD_SIZE).collect(); RwLock::new(shard) }) - .take(capacity.div_ceil(SHARD_SIZE)) + .take(capacity.value().div_ceil(SHARD_SIZE)) .collect(); Self { @@ -60,7 +62,7 @@ where where I: Eq + Hash + Clone, { - if internal.into_usize() >= self.capacity { + if internal.into_usize() >= self.capacity.value() { return Err(InsertError::OutOfBounds); } @@ -108,7 +110,7 @@ where where I: Clone, { - if internal.into_usize() >= self.capacity { + if internal.into_usize() >= self.capacity.value() { return None; } @@ -154,7 +156,7 @@ where } #[cfg(test)] - fn capacity(&self) -> usize { + fn capacity(&self) -> Capacity { self.capacity } } @@ -221,7 +223,9 @@ mod tests { SHARD_SIZE, SHARD_SIZE + 1, 3 * SHARD_SIZE, - ] { + ] + .map(Capacity::new) + { let map = IdMap::::new(capacity); assert_eq!(map.capacity(), capacity); } @@ -229,7 +233,7 @@ mod tests { #[test] fn insert_round_trips() { - let map = IdMap::::new(16); + let map = IdMap::::new(Capacity::new(16)); assert!(map.insert(100, 3).is_ok()); assert_eq!(map.to_internal(&100), Some(3)); @@ -244,7 +248,7 @@ mod tests { #[test] fn insert_rejects_out_of_bounds_internal() { - let map = IdMap::::new(16); + let map = IdMap::::new(Capacity::new(16)); assert!(matches!(map.insert(0, 16), Err(InsertError::OutOfBounds))); assert!(matches!( map.insert(0, u32::MAX), @@ -257,7 +261,7 @@ mod tests { #[test] fn insert_rejects_duplicate_external_and_preserves_state() { - let map = IdMap::::new(16); + let map = IdMap::::new(Capacity::new(16)); map.insert(7, 5).unwrap(); assert!(matches!(map.insert(7, 6), Err(InsertError::ExternalExists))); @@ -270,7 +274,7 @@ mod tests { #[test] fn insert_rejects_duplicate_internal_and_preserves_state() { - let map = IdMap::::new(16); + let map = IdMap::::new(Capacity::new(16)); map.insert(7, 5).unwrap(); assert!(matches!(map.insert(8, 5), Err(InsertError::InternalExists))); @@ -283,7 +287,7 @@ mod tests { #[test] fn to_external_handles_bounds_and_empty_slots() { - let map = IdMap::::new(16); + let map = IdMap::::new(Capacity::new(16)); // In-bounds but unmapped slot. assert_eq!(map.to_external(5), None); // Out-of-bounds slot. @@ -292,7 +296,7 @@ mod tests { #[test] fn mappings_span_shard_boundaries() { - let capacity = 3 * SHARD_SIZE; + let capacity = Capacity::new(3 * SHARD_SIZE); let map = IdMap::::new(capacity); // Ids straddling every internal shard boundary. @@ -302,7 +306,7 @@ mod tests { SHARD_SIZE as u32, (2 * SHARD_SIZE - 1) as u32, (2 * SHARD_SIZE) as u32, - (capacity - 1) as u32, + (capacity.value() - 1) as u32, ]; for (external, &internal) in ids.iter().enumerate() { @@ -317,7 +321,7 @@ mod tests { #[test] fn lookup_supports_borrowed_query() { - let map = IdMap::::new(16); + let map = IdMap::::new(Capacity::new(16)); map.insert("alpha".to_string(), 1).unwrap(); // Borrowed `&str` lookups against `String` keys. @@ -329,7 +333,7 @@ mod tests { #[test] fn occupied_entry_exposes_mapping() { - let map = IdMap::::new(16); + let map = IdMap::::new(Capacity::new(16)); map.insert(42, 9).unwrap(); let entry = map.occupied_entry(42).expect("entry should exist"); @@ -339,13 +343,13 @@ mod tests { #[test] fn occupied_entry_absent_for_unmapped() { - let map = IdMap::::new(16); + let map = IdMap::::new(Capacity::new(16)); assert!(map.occupied_entry(42).is_none()); } #[test] fn entry_delete_clears_both_directions() { - let map = IdMap::::new(16); + let map = IdMap::::new(Capacity::new(16)); map.insert(42, 9).unwrap(); // Just creating and dropping an `occupied_entry` does not clear it. diff --git a/diskann-inmem/src/integration/counters.rs b/diskann-inmem/src/integration/counters.rs index fcb16d6cde..b6af15ef4b 100644 --- a/diskann-inmem/src/integration/counters.rs +++ b/diskann-inmem/src/integration/counters.rs @@ -3,7 +3,7 @@ * Licensed under the MIT license. */ -/// A snapshot of global [`Counters`](crate::counters::Counters). +/// A snapshot of global counters. #[derive(Debug, Clone)] #[non_exhaustive] pub struct CounterSnapshot { diff --git a/diskann-inmem/src/integration/store.rs b/diskann-inmem/src/integration/store.rs deleted file mode 100644 index 352e36572b..0000000000 --- a/diskann-inmem/src/integration/store.rs +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Copyright (c) Microsoft Corporation. - * Licensed under the MIT license. - */ - -#![expect( - clippy::expect_used, - reason = "integration test tools are not production code" -)] - -use std::num::{NonZeroU32, NonZeroUsize}; - -use diskann_utils::views::Matrix; - -use crate::{num::Bytes, store}; - -#[derive(Debug)] -pub struct Config { - pub capacity: usize, - pub entry_bytes: usize, - pub epoch_guard_slots: usize, - pub freelist_recycle_capacity: usize, -} - -#[derive(Debug)] -pub struct Store { - store: store::Store, -} - -impl Store { - /// Construct a store with `config.capacity` writable slots, each holding - /// `config.entry_bytes` bytes. - /// - /// A single zeroed frozen point is created internally to satisfy the underlying - /// store's requirement of at least one frozen entry; it occupies the highest slot - /// index and is always readable. - /// - /// # Panics - /// - /// Panics if the underlying store could not be constructed (e.g. `config.capacity` plus - /// the frozen point exceeds `u32::MAX`) or if other configuration parameters such as - /// the number of epoch guard slots are invalid (e.g. zero). - pub fn new(config: Config) -> Self { - let mut store_config = - store::Config::new(config.capacity, Bytes::new(config.entry_bytes), 0); - - store_config - .epoch_guard_slots( - NonZeroUsize::new(config.epoch_guard_slots) - .expect("`epoch_guard_slots` must be non-zero"), - ) - .freelist_recycle_capacity( - NonZeroU32::new( - config - .freelist_recycle_capacity - .try_into() - .expect("`freelist_recycle_capacity` must fit within 32-bits"), - ) - .expect("`freelist_recycle_capacity` must be non-zero"), - ); - - let data = Matrix::new(0u8, 1, config.entry_bytes); - let store = - store::Store::new(store_config, data.as_view()).expect("failed to construct store"); - Self { store } - } - - /// Return the total number of slots, including the frozen point. - pub fn slots(&self) -> usize { - self.store.frozen().end as usize - } - - /// Return the range of writable (non-frozen) slot indices. - pub fn writable(&self) -> std::ops::Range { - 0..(self.store.frozen().start as usize) - } - - /// Attempt to reclaim retired slots, returning the number reclaimed if any. - pub fn reclaim(&self) -> Option { - self.store.try_drain() - } - - pub fn acquire(&self) -> Option> { - self.store.acquire().map(Writer::new) - } - - #[must_use = "result indicates success or failure"] - pub fn retire(&self, i: usize) -> bool { - self.store.retire(i).is_ok() - } - - pub fn reader(&self) -> Option> { - match self.store.reader() { - Ok(reader) => Some(Reader::new(reader)), - Err(crate::epoch::Unavailable) => None, - } - } -} - -pub struct Reader<'a> { - reader: store::Reader<'a>, -} - -impl<'a> Reader<'a> { - fn new(reader: store::Reader<'a>) -> Self { - Self { reader } - } - - pub fn read(&self, i: usize) -> Option<&[u8]> { - self.reader.read(i) - } -} - -pub struct Writer<'a> { - slot: store::Slot<'a>, -} - -impl<'a> Writer<'a> { - fn new(slot: store::Slot<'a>) -> Self { - Self { slot } - } - - pub fn publish(self) { - self.slot.publish(); - } - - pub fn as_mut_slice(&mut self) -> &mut [u8] { - self.slot.as_mut_slice() - } -} diff --git a/diskann-inmem/src/integration/store/checked.rs b/diskann-inmem/src/integration/store/checked.rs new file mode 100644 index 0000000000..8aee490b9b --- /dev/null +++ b/diskann-inmem/src/integration/store/checked.rs @@ -0,0 +1,90 @@ +/* + * Copyright (c) Microsoft Corporation. + * Licensed under the MIT license. + */ + +#![expect( + clippy::expect_used, + clippy::unwrap_used, + reason = "integration test tools are not production code" +)] + +use std::num::{NonZeroU32, NonZeroUsize}; + +use crate::{ + num::{Capacity, MaxDegree}, + store::{self, checked}, +}; + +use super::boilerplate; + +#[derive(Debug)] +pub struct Config { + pub capacity: usize, + pub epoch_guard_slots: usize, + pub freelist_recycle_capacity: usize, +} + +boilerplate!( + checked::Checked => Store, + for<'a> checked::Reader<'a> => Reader, + for<'a> checked::Slot<'a> => Writer, +); + +impl Store { + /// Construct a store with `config.capacity` writable slots and no frozen points. + /// + /// # Panics + /// + /// Panics if the underlying store could not be constructed (e.g. `config.capacity` + /// exceeds `u32::MAX`) or if other configuration parameters such as the number of + /// epoch guard slots are invalid (e.g. zero). + pub fn new(config: Config) -> Self { + let store_layout = store::Layout::new(Capacity::new(config.capacity), MaxDegree::new(0), 0); + + let store_config = store::Config::__exhaustive( + NonZeroUsize::new(config.epoch_guard_slots) + .expect("`epoch_guard_slots` must be non-zero"), + NonZeroU32::new( + config + .freelist_recycle_capacity + .try_into() + .expect("`freelist_recycle_capacity` must fit within 32-bits"), + ) + .expect("`freelist_recycle_capacity` must be non-zero"), + ); + + let slots_config = checked::Checked::config(); + let store = store::Store::new(store_layout, store_config, slots_config) + .expect("failed to construct store"); + + Self { store } + } +} + +#[derive(Debug)] +pub struct Value<'a> { + value: checked::Value<'a>, +} + +impl<'a> Value<'a> { + fn new(value: checked::Value<'a>) -> Self { + Self { value } + } + + pub fn get(&self) -> u64 { + self.value.get() + } +} + +impl Reader<'_> { + pub fn read(&self, i: usize) -> Option> { + self.reader.read(i.try_into().unwrap()).map(Value::new) + } +} + +impl<'a> Writer<'a> { + pub fn set(&mut self, v: u64) { + self.slot.data().set(v) + } +} diff --git a/diskann-inmem/src/integration/store/invasive.rs b/diskann-inmem/src/integration/store/invasive.rs new file mode 100644 index 0000000000..274e80b77f --- /dev/null +++ b/diskann-inmem/src/integration/store/invasive.rs @@ -0,0 +1,76 @@ +/* + * Copyright (c) Microsoft Corporation. + * Licensed under the MIT license. + */ + +#![expect( + clippy::expect_used, + reason = "integration test tools are not production code" +)] + +use std::num::{NonZeroU32, NonZeroUsize}; + +use crate::{ + num::{Bytes, Capacity, MaxDegree}, + store, +}; + +use super::boilerplate; + +#[derive(Debug)] +pub struct Config { + pub capacity: usize, + pub entry_bytes: usize, + pub epoch_guard_slots: usize, + pub freelist_recycle_capacity: usize, +} + +boilerplate!( + store::invasive::Invasive => Store, + for<'a> store::invasive::Reader<'a> => Reader, + for<'a> store::invasive::Slot<'a> => Writer, +); + +impl Store { + /// Construct a store with `config.capacity` writable slots, each holding + /// `config.entry_bytes` bytes. No frozen points are created. + /// + /// # Panics + /// + /// Panics if the underlying store could not be constructed (e.g. `config.capacity` plus + /// the frozen point exceeds `u32::MAX`) or if other configuration parameters such as + /// the number of epoch guard slots are invalid (e.g. zero). + pub fn new(config: Config) -> Self { + let store_layout = store::Layout::new(Capacity::new(config.capacity), MaxDegree::new(0), 0); + + let store_config = store::Config::__exhaustive( + NonZeroUsize::new(config.epoch_guard_slots) + .expect("`epoch_guard_slots` must be non-zero"), + NonZeroU32::new( + config + .freelist_recycle_capacity + .try_into() + .expect("`freelist_recycle_capacity` must fit within 32-bits"), + ) + .expect("`freelist_recycle_capacity` must be non-zero"), + ); + + let slots_config = store::invasive::Invasive::config(Bytes::new(config.entry_bytes)); + let store = store::Store::new(store_layout, store_config, slots_config) + .expect("failed to construct store"); + + Self { store } + } +} + +impl<'a> Reader<'a> { + pub fn read(&self, i: usize) -> Option<&[u8]> { + self.reader.read(i) + } +} + +impl<'a> Writer<'a> { + pub fn as_mut_slice(&mut self) -> &mut [u8] { + self.slot.data().as_mut_slice() + } +} diff --git a/diskann-inmem/src/integration/store/mod.rs b/diskann-inmem/src/integration/store/mod.rs new file mode 100644 index 0000000000..ef7b32cf52 --- /dev/null +++ b/diskann-inmem/src/integration/store/mod.rs @@ -0,0 +1,96 @@ +/* + * Copyright (c) Microsoft Corporation. + * Licensed under the MIT license. + */ + +//! This module exposes "public" integration-test wrappers for the various internal store +//! mechanisms to drive larger concurrency tests. + +pub mod checked; +pub mod invasive; + +/// These implementations have a similar structure. A [`boilerplate`] macro is used to ensure +/// the capabilities exposed are mostly the same. +macro_rules! boilerplate { + ( + $slots:ty => $store:ident, + for<$read_lt:lifetime> $read:ty => $reader:ident, + for<$slot_lt:lifetime> $slot:ty => $writer:ident, + ) => { + /// A test store wraper. + #[derive(Debug)] + pub struct $store { + store: $crate::store::Store<$slots>, + } + + impl $store { + /// Return the total number of slots, including the frozen point. + pub fn readable_slots(&self) -> usize { + self.store.frozen().end as usize + } + + /// Return the range of writable (non-frozen) slot indices. + pub fn writable_slots(&self) -> usize { + self.store.frozen().start as usize + } + + /// Attempt to reclaim retired slots, returning the number reclaimed if any. + pub fn reclaim(&self) -> Option { + self.store.try_drain() + } + + /// Acquire a slot, returning a [`Writer`]. Returns `None` if no slot is + /// available for writing. + pub fn acquire(&self) -> Option<$writer<'_>> { + self.store.acquire().map(Writer::new) + } + + /// Attempt to retire slot `i`. Returns `true` only if the slot was successfully + /// retired. + #[must_use = "result indicates success or failure"] + pub fn retire(&self, i: usize) -> bool { + self.store.retire(i).is_ok() + } + + /// Attain a reader into the store. Returns `None` if all epoch guard slots + /// are used. + pub fn reader(&self) -> Option<$reader<'_>> { + match <$slots>::reader(&self.store) { + Ok(reader) => Some($reader::new(reader)), + Err($crate::epoch::Unavailable) => None, + } + } + } + + /// A reader for the test store. + #[derive(Debug)] + pub struct $reader<$read_lt> { + reader: $read, + } + + impl<$read_lt> $reader<$read_lt> { + fn new(reader: $read) -> Self { + Self { reader } + } + } + + /// A writer for the test store. + #[derive(Debug)] + pub struct $writer<$slot_lt> { + slot: $crate::store::Slot<$slot_lt, $slot>, + } + + impl<$slot_lt> $writer<$slot_lt> { + fn new(slot: $crate::store::Slot<$slot_lt, $slot>) -> Self { + Self { slot } + } + + /// Publish the slot - making it accessible to readers. + pub fn publish(self) { + self.slot.publish(); + } + } + }; +} + +use boilerplate; diff --git a/diskann-inmem/src/layers/full.rs b/diskann-inmem/src/layers/full.rs deleted file mode 100644 index 9cdd94c529..0000000000 --- a/diskann-inmem/src/layers/full.rs +++ /dev/null @@ -1,703 +0,0 @@ -/* - * Copyright (c) Microsoft Corporation. - * Licensed under the MIT license. - */ - -use std::{fmt::Debug, marker::PhantomData}; - -use diskann::{ANNError, ANNResult}; -use diskann_vector::{ - UnalignedSlice, - conversion::SliceCast, - distance::{ - self, Cosine, CosineNormalized, DistanceProvider, InnerProduct, Metric, Specialize, - SquaredL2, - }, -}; -use diskann_wide::{ - ARCH, - arch::{Current, FTarget2}, -}; -use half::f16; -use thiserror::Error; - -use crate::{Hidden, layers, num::Bytes}; - -/// A useful trait bound for types compatible with [`Full`]. -/// -/// This encompasses *everything* required for `Full: layers::Insert` and can be used as -/// a single bound. -pub trait FullPrecision: bytemuck::Pod + std::fmt::Debug + Send + Sync { - #[doc(hidden)] - fn __new(_: Hidden, dim: usize, metric: Metric) -> Full; - - #[doc(hidden)] - fn __query_distance<'a, V>( - _: Hidden, - full: &'a Full, - query: &'a [Self], - visitor: V, - ) -> ANNResult - where - V: layers::QueryVisitor<'a>; -} - -/// Full-precision data layer. -#[derive(Debug)] -pub struct Full -where - T: 'static, -{ - distance: Distance, - metric: Metric, -} - -impl Full -where - T: 'static, -{ - /// Create a new full-precision layer for data with the given `dim` and `metric`. - pub fn new(dim: usize, metric: Metric) -> Self - where - T: FullPrecision, - { - T::__new(Hidden::new(), dim, metric) - } - - fn from_distance_provider(dim: usize, metric: Metric) -> Self - where - T: DistanceProvider, - { - let distance = Distance { - f: T::distance_comparer(metric, Some(dim)), - dim, - }; - - Self { distance, metric } - } - - /// Return the logical dimension of the data handled by this [`layers::Layer`]. - pub fn dim(&self) -> usize { - self.distance.dim - } - - /// Return the number of bytes of the data handles by this [`layers::Layer`]. - pub fn bytes(&self) -> Bytes { - Bytes::new(self.dim() * std::mem::size_of::()) - } - - fn check_dim(&self, dim: usize) -> Result<(), QueryDistanceError> { - if self.dim() != dim { - Err(QueryDistanceError { - expected: self.dim(), - xlen: dim, - }) - } else { - Ok(()) - } - } -} - -impl layers::Layer for Full -where - T: FullPrecision, -{ - fn bytes(&self) -> Bytes { - >::bytes(self) - } -} - -impl layers::Set<&[T]> for Full -where - T: FullPrecision, -{ - fn set(&self, v: &[T], bytes: &mut [u8]) -> ANNResult<()> { - if v.len() != self.dim() { - Err(ANNError::from(SetError::Dim { - got: v.len(), - expected: self.dim(), - })) - } else if bytes.len() != self.bytes().value() { - Err(ANNError::from(SetError::Bytes { - got: bytes.len(), - expected: self.bytes().value(), - })) - } else { - bytes.copy_from_slice(bytemuck::must_cast_slice::(v)); - Ok(()) - } - } -} - -#[derive(Debug, Error)] -enum SetError { - #[error( - "data of dimension {} does not match full precision layer's dimension {}", - got, - expected - )] - Dim { got: usize, expected: usize }, - #[error( - "raw byte slice of length {} does not match expected length {}", - got, - expected - )] - Bytes { got: usize, expected: usize }, -} - -diskann::convert_error!(SetError); - -impl layers::AsDistance for Full -where - T: FullPrecision, -{ - fn as_distance(&self) -> &dyn layers::Distance { - &self.distance - } -} - -impl layers::Search for Full -where - T: FullPrecision, -{ - type Query<'a> = &'a [T]; - - fn query_distance<'a, V>(&'a self, query: &'a [T], visitor: V) -> ANNResult - where - V: layers::QueryVisitor<'a>, - { - T::__query_distance(Hidden::new(), self, query, visitor) - } -} - -impl layers::Insert for Full where T: FullPrecision {} - -////////////// -// Distance // -////////////// - -#[derive(Debug)] -#[doc(hidden)] -pub struct Distance -where - T: 'static, - U: 'static, -{ - f: distance::Distance, - dim: usize, -} - -impl Clone for Distance { - fn clone(&self) -> Self { - *self - } -} - -impl Copy for Distance {} - -impl Distance -where - T: 'static, - U: 'static, -{ - #[cold] - #[inline(never)] - fn error(&self, x: &[u8], y: &[u8]) -> ANNResult { - let error = DistanceError { - expected: self.bytes(), - xlen: x.len(), - ylen: y.len(), - }; - - Err(ANNError::new(error)) - } - - fn dim(&self) -> usize { - self.dim - } - - fn bytes(&self) -> usize { - self.dim() * std::mem::size_of::() - } -} - -impl layers::Distance for Distance -where - T: Debug + 'static, -{ - fn evaluate(&self, x: &[u8], y: &[u8]) -> ANNResult { - let bytes = self.bytes(); - if x.len() != bytes || y.len() != bytes { - self.error(x, y) - } else { - // SAFETY: We've checked that both `x` and `y` are valid for - // `size_of::() * self.dim` bytes. - let ux = unsafe { UnalignedSlice::new(x.as_ptr().cast::(), self.dim) }; - - // SAFETY: Same as above - let uy = unsafe { UnalignedSlice::new(y.as_ptr().cast::(), self.dim) }; - Ok(self.f.call_unaligned(ux, uy)) - } - } -} - -#[derive(Debug, Error)] -#[error( - "expected slices of length {} - instead got {} and {}", - self.expected, - self.xlen, - self.ylen -)] -struct DistanceError { - expected: usize, - xlen: usize, - ylen: usize, -} - -/////////////////// -// QueryDistance // -/////////////////// - -// A baby [`std::borrow::Cow`]. -#[derive(Debug)] -enum Calf<'a, T> { - Borrowed(&'a [T]), - Owned(Box<[T]>), -} - -impl std::ops::Deref for Calf<'_, T> { - type Target = [T]; - fn deref(&self) -> &Self::Target { - match self { - Self::Borrowed(slice) => slice, - Self::Owned(boxed) => boxed, - } - } -} - -/// A fused query distance based on [`diskann_vector::PureDistanceFunction`] to enable -/// inlining of the final distance function (`D`). -/// -/// The type of the embedded query (`T`) is distinct from the expected data-set (`U`) to -/// allow `f16` queries to be pre-converted to `f32`, saving on-the-fly conversion that -/// would otherwise be needed. -#[derive(Debug)] -struct QueryDistance<'a, T, U, D> { - query: Calf<'a, T>, - // The type of the data in the original dataset. - _data: PhantomData, - // The type of the `PureDistanceFunction` used for the implementation. - _distance: PhantomData, -} - -impl<'a, T, U, D> QueryDistance<'a, T, U, D> { - fn new(query: Calf<'a, T>) -> Self { - Self { - query, - _data: PhantomData, - _distance: PhantomData, - } - } - - fn bytes(&self) -> usize { - std::mem::size_of::() * self.query.len() - } - - #[inline(never)] - fn error(&self, len: usize) -> ANNResult { - let error = QueryDistanceError { - expected: self.bytes(), - xlen: len, - }; - - Err(ANNError::new(error)) - } -} - -impl layers::QueryDistance for QueryDistance<'_, T, U, D> -where - T: Send + Sync + 'static + Debug, - U: Send + Sync + 'static + Debug, - D: for<'a> FTarget2, UnalignedSlice<'a, U>> - + Send - + Sync - + Debug, -{ - #[inline(always)] - fn evaluate(&self, x: &[u8]) -> ANNResult { - if x.len() != self.bytes() { - self.error(x.len()) - } else { - // SAFETY: We've validated that `x` has the correct length. - let x = unsafe { UnalignedSlice::new(x.as_ptr().cast::(), self.query.len()) }; - Ok(D::run(ARCH, (*self.query).into(), x)) - } - } -} - -#[derive(Debug, Error)] -#[error( - "expected slice of length {} - instead got {}", - self.expected, - self.xlen, -)] -struct QueryDistanceError { - expected: usize, - xlen: usize, -} - -diskann::convert_error!(QueryDistanceError); - -macro_rules! mint { - ($query:ident, $visitor:ident, $T:ty => { $N:literal, $f:ident }) => {{ - mint!($query, $visitor, { $T, $T } => { $N, $f }) - }}; - ($query:ident, $visitor:ident, { $T:ty, $U:ty } => { $N:literal, $f:ident }) => {{ - let inner = QueryDistance::<$T, $U, Specialize<$N, $f>>::new($query); - $visitor.visit_sized::<{ $N * std::mem::size_of::<$U>() }, _>(inner) - }}; - ($query:ident, $visitor:ident, $T:ty => $f:ident) => {{ - mint!($query, $visitor, { $T, $T } => $f) - }}; - ($query:ident, $visitor:ident, { $T:ty, $U:ty } => $f:ident) => {{ - let inner = QueryDistance::<$T, $U, $f>::new($query); - $visitor.visit(inner) - }}; -} - -impl FullPrecision for f32 { - fn __new(_: Hidden, dim: usize, metric: Metric) -> Full { - Full::from_distance_provider(dim, metric) - } - - fn __query_distance<'a, V>( - _: Hidden, - full: &'a Full, - query: &'a [f32], - visitor: V, - ) -> ANNResult - where - V: layers::QueryVisitor<'a>, - { - full.check_dim(query.len())?; - - let query = Calf::Borrowed(query); - - let output = match full.metric { - Metric::L2 => { - if full.dim() == 100 { - mint!(query, visitor, f32 => { 100, SquaredL2 }) - } else { - mint!(query, visitor, f32 => SquaredL2) - } - } - Metric::InnerProduct => { - mint!(query, visitor, f32 => InnerProduct) - } - Metric::Cosine => mint!(query, visitor, f32 => Cosine), - Metric::CosineNormalized => mint!(query, visitor, f32 => CosineNormalized), - }; - - Ok(output) - } -} - -impl FullPrecision for f16 { - fn __new(_: Hidden, dim: usize, metric: Metric) -> Full { - Full::from_distance_provider(dim, metric) - } - - fn __query_distance<'a, V>( - _: Hidden, - full: &'a Full, - query: &'a [f16], - visitor: V, - ) -> ANNResult - where - V: layers::QueryVisitor<'a>, - { - full.check_dim(query.len())?; - - let mut as_f32: Box<[f32]> = std::iter::repeat_n(0.0, full.dim()).collect(); - diskann_wide::arch::dispatch2(SliceCast::new(), &mut *as_f32, query); - let query = Calf::Owned(as_f32); - - let output = match full.metric { - Metric::L2 => { - if full.dim() == 100 { - mint!(query, visitor, { f32, f16 } => { 100, SquaredL2 }) - } else { - mint!(query, visitor, { f32, f16 } => SquaredL2) - } - } - Metric::InnerProduct => mint!(query, visitor, { f32, f16 } => InnerProduct), - Metric::Cosine => mint!(query, visitor, { f32, f16 } => Cosine), - Metric::CosineNormalized => mint!(query, visitor, { f32, f16 } => CosineNormalized), - }; - - Ok(output) - } -} - -impl FullPrecision for u8 { - fn __new(_: Hidden, dim: usize, metric: Metric) -> Full { - Full::from_distance_provider(dim, metric) - } - - fn __query_distance<'a, V>( - _: Hidden, - full: &'a Full, - query: &'a [u8], - visitor: V, - ) -> ANNResult - where - V: layers::QueryVisitor<'a>, - { - full.check_dim(query.len())?; - - let query = Calf::Borrowed(query); - - let output = match full.metric { - Metric::L2 => { - if full.dim() == 128 { - mint!(query, visitor, u8 => { 128, SquaredL2 }) - } else { - mint!(query, visitor, u8 => SquaredL2) - } - } - Metric::InnerProduct => mint!(query, visitor, u8 => InnerProduct), - Metric::Cosine => mint!(query, visitor, u8 => Cosine), - Metric::CosineNormalized => mint!(query, visitor, u8 => Cosine), - }; - - Ok(output) - } -} - -impl FullPrecision for i8 { - fn __new(_: Hidden, dim: usize, metric: Metric) -> Full { - Full::from_distance_provider(dim, metric) - } - - fn __query_distance<'a, V>( - _: Hidden, - full: &'a Full, - query: &'a [i8], - visitor: V, - ) -> ANNResult - where - V: layers::QueryVisitor<'a>, - { - full.check_dim(query.len())?; - - let query = Calf::Borrowed(query); - - let output = match full.metric { - Metric::L2 => mint!(query, visitor, i8 => SquaredL2), - Metric::InnerProduct => mint!(query, visitor, i8 => InnerProduct), - Metric::Cosine => mint!(query, visitor, i8 => Cosine), - Metric::CosineNormalized => mint!(query, visitor, i8 => Cosine), - }; - - Ok(output) - } -} - -/////////// -// Tests // -/////////// - -#[cfg(test)] -#[cfg(not(miri))] -mod tests { - use std::fmt::Display; - - use rand::{Rng, SeedableRng, rngs::StdRng}; - - use super::*; - // Bring the inherent-call traits into method scope. The `Distance` / `QueryDistance` - // traits are not imported: their methods are reached through `&dyn _` trait objects, - // which does not require the trait to be in scope. - use crate::layers::{AsDistance as _, QueryVisitor, Search as _, Set as _}; - - /// Generate random elements of a layer's data type from a seeded RNG. - trait Sample: bytemuck::Pod { - fn sample(rng: &mut R) -> Self; - } - - impl Sample for f32 { - fn sample(rng: &mut R) -> Self { - rng.random_range(-1.0f32..1.0f32) - } - } - - impl Sample for f16 { - fn sample(rng: &mut R) -> Self { - f16::from_f32(rng.random_range(-1.0f32..1.0f32)) - } - } - - impl Sample for u8 { - fn sample(rng: &mut R) -> Self { - rng.random() - } - } - - impl Sample for i8 { - fn sample(rng: &mut R) -> Self { - rng.random() - } - } - - fn gen_vec(rng: &mut R, dim: usize) -> Vec { - (0..dim).map(|_| T::sample(rng)).collect() - } - - /// A [`QueryVisitor`] that simply boxes the minted kernel so the test can probe it - /// directly. Exercises both `visit` (dynamic) and `visit_sized` (specialized) paths. - struct Collect; - - impl<'a> QueryVisitor<'a> for Collect { - type Output = Box; - - fn visit(self, distance: Q) -> Self::Output - where - Q: layers::QueryDistance + 'a, - { - Box::new(distance) - } - } - - /// Compare two distances allowing for floating-point reassociation between the - /// specialized / converted kernels and the dynamic reference. - fn approx_eq(got: f32, want: f32) -> bool { - (got - want).abs() <= 1e-3 + 1e-4 * want.abs() - } - - /// Exercise every `Full` API across dimensions `1..=max_dim`. - /// - /// For each dimension we check that `bytes`/`set` agree, that `distance` and - /// `query_distance` are consistent with `DistanceProvider`, and that all of these - /// reject byte slices that are too long or too short. - fn test_impl(max_dim: usize, ctx: &dyn Display) - where - T: FullPrecision + Sample + DistanceProvider, - { - let mut rng = StdRng::seed_from_u64(0x0D15_0ACE ^ max_dim as u64); - let metrics = [ - Metric::L2, - Metric::InnerProduct, - Metric::Cosine, - Metric::CosineNormalized, - ]; - - for dim in 1..=max_dim { - let a = gen_vec::(&mut rng, dim); - let b = gen_vec::(&mut rng, dim); - - // `bytes` and `set` agree: the encoded buffer equals the raw cast bytes. - let layer = Full::::new(dim, Metric::L2); - assert_eq!( - layer.bytes().value(), - dim * std::mem::size_of::(), - "{ctx}: dim {dim}: unexpected byte length", - ); - - let mut a_bytes = vec![0u8; layer.bytes().value()]; - layer.set(&a, &mut a_bytes).unwrap(); - assert_eq!( - a_bytes.as_slice(), - bytemuck::cast_slice::(&a), - "{ctx}: dim {dim}: set mismatch", - ); - - let mut b_bytes = vec![0u8; layer.bytes().value()]; - layer.set(&b, &mut b_bytes).unwrap(); - - for metric in metrics { - let full = Full::::new(dim, metric); - - // Reference value straight from `DistanceProvider`. - let reference = - >::distance_comparer(metric, Some(dim)).call(&a, &b); - - // `distance` is built from the same comparer, so it must match exactly. - let distance = full.as_distance(); - let via_distance = distance.evaluate(&a_bytes, &b_bytes).unwrap(); - assert_eq!( - via_distance, reference, - "{ctx}: dim {dim}, metric {metric:?}: distance != DistanceProvider", - ); - - // `query_distance` computes the same geometry. Specialized and f16-converted - // kernels may reassociate the summation, so compare approximately. - let query = full.query_distance(a.as_slice(), Collect).unwrap(); - let via_query = query.evaluate(&b_bytes).unwrap(); - assert!( - approx_eq(via_query, via_distance), - "{ctx}: dim {dim}, metric {metric:?}: query {via_query} != distance {via_distance}", - ); - - // Every distance API rejects byte slices that are too long or too short. - let short = &a_bytes[..a_bytes.len() - 1]; - let mut long = a_bytes.clone(); - long.push(0); - - assert!(distance.evaluate(short, &b_bytes).is_err()); - assert!(distance.evaluate(&long, &b_bytes).is_err()); - assert!(distance.evaluate(&a_bytes, short).is_err()); - assert!(distance.evaluate(&a_bytes, &long).is_err()); - - assert!(query.evaluate(short).is_err()); - assert!(query.evaluate(&long).is_err()); - } - - // `set` rejects mis-sized element and buffer slices. - let mut buf = vec![0u8; layer.bytes().value()]; - let too_many = gen_vec::(&mut rng, dim + 1); - assert!( - layer.set(&too_many, &mut buf).is_err(), - "{ctx}: dim {dim}: set accepted an over-long element slice", - ); - - assert!( - layer.query_distance(&too_many, Collect).is_err(), - "{ctx}: dim {dim}: incorrect query lengths should be rejected" - ); - - let mut short_buf = vec![0u8; layer.bytes().value().saturating_sub(1)]; - assert!( - layer.set(&a, &mut short_buf).is_err(), - "{ctx}: dim {dim}: set accepted an under-sized buffer", - ); - - let too_few = gen_vec::(&mut rng, dim - 1); - assert!( - layer.query_distance(&too_few, Collect).is_err(), - "{ctx}: dim {dim}: incorrect query lengths should be rejected" - ); - } - } - - // `max_dim` must exceed the largest specialized dimension for each type so the - // const-generic (`visit_sized`) paths are covered alongside the dynamic ones. - #[test] - fn full_f32() { - test_impl::(256, &"f32"); - } - - #[test] - fn full_f16() { - test_impl::(256, &"f16"); - } - - #[test] - fn full_u8() { - test_impl::(160, &"u8"); - } - - #[test] - fn full_i8() { - test_impl::(160, &"i8"); - } -} diff --git a/diskann-inmem/src/layers/mod.rs b/diskann-inmem/src/layers/mod.rs deleted file mode 100644 index 13179d9bc5..0000000000 --- a/diskann-inmem/src/layers/mod.rs +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Copyright (c) Microsoft Corporation. - * Licensed under the MIT license. - */ - -//! Distance layers indexing. -//! -//! An important assumption made by this module is that the data within each layer is -//! uniformly sized: each entry occupies the same number of bytes. Furthermore, the data -//! to be stored may not assume any particular alignment. Implementations will strive to -//! achieve a reasonable alignment, but this may not be relied on. -//! -//! # Query Distance Specialization -//! -//! The design of this module allows aggressive optimization of graph search kernels via -//! the [`Search`] and [`QueryVisitor`] pairs of traits. -//! -//! Implementations of [`Search`] can pass a [`QueryDistance`] kernel specialized to -//! a specific geometry (dimensionality or metric type) which upstream [`QueryVisitor`] -//! will fuse into larger kernels. While this allows for high performance graph kernels, -//! some considerations should be taken into account: -//! -//! 1. For correctness purposes, upstream callers cannot do any kind of caching. As such, -//! the dispatch layer used to select the kernel passed to the [`QueryVisitor`] should -//! be relatively efficient. -//! -//! 2. Keep the number of specializations bounded for compile time reasons. - -use diskann::ANNResult; - -use crate::num::Bytes; - -mod full; -pub use full::{Full, FullPrecision}; - -/// Base layer for data representations. -pub trait Layer: Send + Sync + 'static { - /// Return the number of bytes needed by this layer representation. - /// - /// To be well-behaved, this function must be idempotent. - fn bytes(&self) -> Bytes; -} - -/// Store an element of type `T` into a raw byte buffer. -/// -/// Implementations may assume that `bytes.len()` is equal to [`Layer::bytes`]. -pub trait Set: Layer { - /// Write into the stored representation. - fn set(&self, element: T, bytes: &mut [u8]) -> ANNResult<()>; -} - -/// A distance computation on raw byte slices. -/// -/// When paired with [`Layer`] via helpers like [`AsDistance`], implementations may assume -/// that `x` and `y` have length [`Layer::bytes`]. -/// -/// No alignment guarantees are made for `x` and `y`, though in practice they are likely -/// to be aligned to 32 or 64 bytes. -pub trait Distance: Send + Sync + std::fmt::Debug { - fn evaluate(&self, x: &[u8], y: &[u8]) -> ANNResult; -} - -/// Return a [`Distance`] function for a [`Layer`]. -pub trait AsDistance: Send + Sync + std::fmt::Debug { - fn as_distance(&self) -> &dyn Distance; -} - -/// A unary query distance on raw byte slices. -/// -/// When paired with [`Layer`] via helpers like [`Search`], implementations may assume -/// that `x` has length [`Layer::bytes`]. -/// -/// No alignment guarantees are made for `x`, though in practice it is likely to be -/// aligned to 32 or 64 bytes. -pub trait QueryDistance: Send + Sync + std::fmt::Debug { - fn evaluate(&self, x: &[u8]) -> ANNResult; -} - -/// Enable search over vectors defined by a [`Layer`]. -pub trait Search: Send + Sync + 'static { - /// The type of the query. This should be equivalent to the generic parameter in - /// [`Set`], but needs to be replicated here due to limitations in the current trait - /// design. - type Query<'a>; - - /// Create a distance computer specialized for `query` and provide it to `visitor`. - fn query_distance<'a, V>(&'a self, query: Self::Query<'a>, visitor: V) -> ANNResult - where - V: QueryVisitor<'a>; -} - -/// Specialize a kernel around a [`QueryDistance`] implementation. -pub trait QueryVisitor<'a>: Sized { - /// The type of the type-erased output. - type Output; - - /// Specialize [`Self::Output`] for `distance`. - fn visit(self, distance: T) -> Self::Output - where - T: QueryDistance + 'a; - - /// Specialize [`Self::Output`] for `distance` accepting a hint that `distance` has been - /// specialized to work on data elements of exactly `BYTES` bytes long. - /// - /// This can be used to tailor surrounding code (e.g. software prefetches) for exactly - /// the length of the data being processed. - fn visit_sized(self, distance: T) -> Self::Output - where - T: QueryDistance + 'a, - { - self.visit(distance) - } -} - -/// A insert-specific specialization of [`Search`]. -/// -/// Note that the bounds for this trait are unnecessarily complicated, but rely on changes -/// to `diskann` to full resolve. -pub trait Insert: Search + for<'a> Set> + AsDistance { - /// A specialization of [`Search::query_distance`] targeting vector insert specifically. - fn insert_distance<'a, V>(&'a self, query: Self::Query<'a>, visitor: V) -> ANNResult - where - V: QueryVisitor<'a>, - { - self.query_distance(query, visitor) - } -} diff --git a/diskann-inmem/src/lib.rs b/diskann-inmem/src/lib.rs index 8f0570dc30..eb250bf518 100644 --- a/diskann-inmem/src/lib.rs +++ b/diskann-inmem/src/lib.rs @@ -15,12 +15,12 @@ mod epoch; mod freelist; mod ids; mod neighbors; +mod prefetch; mod tag; -mod store; - -pub mod layers; pub mod provider; +pub mod repr; +pub mod store; pub use provider::{Context, Provider, Strategy}; @@ -30,20 +30,3 @@ mod test; #[cfg(feature = "integration-test")] #[doc(hidden)] pub mod integration; - -//----------------// -// Internal Tools // -//----------------// - -/// A "public" type that can only be constructed by this crate. -/// -/// This helps with public traits with internal methods that we don't want users to call. -#[doc(hidden)] -#[derive(Debug)] -pub struct Hidden(()); - -impl Hidden { - const fn new() -> Self { - Self(()) - } -} diff --git a/diskann-inmem/src/neighbors.rs b/diskann-inmem/src/neighbors.rs index 88f9b786aa..f8ad42b0f1 100644 --- a/diskann-inmem/src/neighbors.rs +++ b/diskann-inmem/src/neighbors.rs @@ -31,7 +31,7 @@ use thiserror::Error; use crate::{ buffer::{Buffer, BufferError}, - num::{Align, Bytes}, + num::{Align, Bytes, IdLimit, MaxDegree}, }; type Id = u32; @@ -63,21 +63,21 @@ pub(crate) struct Neighbors { } impl Neighbors { - /// Construct a new [`Neighbors`] capable of holding `entries` adjacency lists with a - /// maximum length of `max_length`. + /// Construct a new [`Neighbors`] capable of holding `id_limit` adjacency lists with a + /// maximum length of `max_degree`. /// /// # Errors /// - /// Returns an error if `(max_length + 1) * size_of::()` overflows `usize` + /// Returns an error if `(max_degree + 1) * size_of::()` overflows `usize` /// (unreachable on 64-bit targets) or the resulting allocation would exceed /// `isize::MAX` bytes. - pub(crate) fn new(entries: u32, max_length: u32) -> Result { - let bytes = max_length + pub(crate) fn new(id_limit: IdLimit, max_degree: u32) -> Result { + let bytes = max_degree .into_usize() .checked_add(1) .and_then(|len| len.checked_mul(std::mem::size_of::())) .map(Bytes::new) - .ok_or(NeighborsError::Overflow(max_length))?; + .ok_or(NeighborsError::Overflow(max_degree))?; // We materialize slices of `Id` into the raw byte buffers. // @@ -91,25 +91,28 @@ impl Neighbors { ); } - let neighbors = Buffer::new(entries.into_usize(), bytes, ALIGN)?; + let neighbors = Buffer::new(id_limit.as_usize(), bytes, ALIGN)?; let locks = std::iter::repeat_with(|| RwLock::new(())) - .take(entries.into_usize().div_ceil(LOCK_GRANULARITY)) + .take(id_limit.as_usize().div_ceil(LOCK_GRANULARITY)) .collect(); Ok(Self { neighbors, locks }) } /// Return the maximum length for any adjacency list. - pub(crate) fn max_length(&self) -> usize { + pub(crate) fn max_degree(&self) -> MaxDegree { // We reserve 4 bytes at the beginning for the length of the adjacency list. - (self.neighbors.stride().value() - std::mem::size_of::()) / std::mem::size_of::() + MaxDegree::new( + (self.neighbors.stride().value() - std::mem::size_of::()) + / std::mem::size_of::(), + ) } /// Return the maximum length for any adjacency list as a 32-bit integer. - pub(crate) fn max_length_u32(&self) -> u32 { + pub(crate) fn max_degree_u32(&self) -> u32 { // Lossless by the invariants on `Self::new`. - self.max_length() as u32 + self.max_degree().value() as u32 } /// Return the number of adjacency lists contained by this graph. @@ -144,7 +147,7 @@ impl Neighbors { // SAFETY: We hold the read-lock, so reading is safe. From our bounds checks, we // know that this pointer is valid. let len: usize = unsafe { prefix.as_ptr().cast::().read() } - .min(self.max_length_u32()) + .min(self.max_degree_u32()) .into_usize(); let mut resizer = neighbors.resize(len); @@ -189,7 +192,7 @@ impl Neighbors { Lock { ptr: slice.as_non_null().cast::(), - capacity: self.max_length().into_usize(), + capacity: self.max_degree().value(), _lock: lock, } } @@ -201,24 +204,24 @@ impl Neighbors { /// Returns an error if: /// /// * `i` exceeds [`Self::entries`]. - /// * `neighbors.len()` exceeds [`Self::max_length_u32`]. + /// * `neighbors.len()` exceeds [`Self::max_degree_u32`]. /// /// If an error is returned, the graph is left unmodified. pub(crate) fn set(&self, i: u32, neighbors: &[u32]) -> Result<(), SetError> { self.check(i).map_err(SetError::OutOfBounds)?; // We can check the length of `neighbors` before acquiring any locks as an early exit. - if neighbors.len() > self.max_length().into_usize() { + if neighbors.len() > self.max_degree().value() { return Err(SetError::TooLong(TooLong { got: neighbors.len(), - max: self.max_length_u32(), + max: self.max_degree_u32(), })); } // SAFETY: We've checked `i` is in-bounds. let lock = unsafe { self.lock_unchecked(i) }; - // SAFETY: `neighbors.len() <= self.max_length()`. + // SAFETY: `neighbors.len() <= self.max_degree()`. unsafe { lock.write_unchecked(neighbors) }; Ok(()) } @@ -235,7 +238,7 @@ impl Neighbors { /// Errors returned by [`Neighbors::new`]. #[derive(Debug, Error)] pub(crate) enum NeighborsError { - /// Computing the per-list byte size `(max_length + 1) * size_of::()` overflowed + /// Computing the per-list byte size `(max_degree + 1) * size_of::()` overflowed /// `usize`. /// /// Unreachable on 64-bit targets. @@ -420,7 +423,7 @@ mod tests { #[test] fn out_of_bounds_rejects_indices_beyond_entries() { - let n = Neighbors::new(4, 4).unwrap(); + let n = Neighbors::new(IdLimit::new(4), 4).unwrap(); // entries == 4, so valid indices are 0..=3. // Regression test: a buggy `check` using `i == entries()` would let // `entries+1`, `entries+2`, ... slip through to UB. @@ -434,7 +437,7 @@ mod tests { #[test] fn empty_neighbors_rejects_all_access() { - let n = Neighbors::new(0, 4).unwrap(); + let n = Neighbors::new(IdLimit::new(0), 4).unwrap(); let mut out = AdjacencyList::with_capacity(4); for i in [0u32, 1, u32::MAX] { assert!(matches!(n.get(i, &mut out), Err(OutOfBounds(_)))); @@ -447,21 +450,21 @@ mod tests { #[test] fn set_rejects_oversized_neighbors() { - let n = Neighbors::new(4, 3).unwrap(); + let n = Neighbors::new(IdLimit::new(4), 3).unwrap(); let too_many = &[1, 2, 3, 4]; assert!(matches!(n.set(0, too_many), Err(SetError::TooLong(_)))); } #[test] fn lock_write_rejects_oversized_neighbors() { - let n = Neighbors::new(4, 3).unwrap(); + let n = Neighbors::new(IdLimit::new(4), 3).unwrap(); let lock = n.lock(0).unwrap(); assert!(lock.write(&[1, 2, 3, 4]).is_err()); } #[test] fn lock_append_rejects_overflow() { - let n = Neighbors::new(4, 3).unwrap(); + let n = Neighbors::new(IdLimit::new(4), 3).unwrap(); n.set(0, &[1, 2]).unwrap(); let lock = n.lock(0).unwrap(); assert!(lock.append(&[3, 4]).is_err()); @@ -469,7 +472,7 @@ mod tests { #[test] fn lock_implements_debug() { - let n = Neighbors::new(4, 3).unwrap(); + let n = Neighbors::new(IdLimit::new(4), 3).unwrap(); let lock = n.lock(0).unwrap(); let _ = format!("{:?}", lock); } @@ -478,7 +481,7 @@ mod tests { #[test] fn append_preserves_existing_and_adds_new() { - let n = Neighbors::new(4, 6).unwrap(); + let n = Neighbors::new(IdLimit::new(4), 6).unwrap(); n.set(0, &[10, 20]).unwrap(); let lock = n.lock(0).unwrap(); @@ -492,7 +495,7 @@ mod tests { #[test] fn append_to_empty() { - let n = Neighbors::new(4, 4).unwrap(); + let n = Neighbors::new(IdLimit::new(4), 4).unwrap(); let lock = n.lock(0).unwrap(); assert_eq!(lock.as_slice(), &[]); @@ -505,7 +508,7 @@ mod tests { #[test] fn append_fills_to_capacity() { - let n = Neighbors::new(1, 3).unwrap(); + let n = Neighbors::new(IdLimit::new(1), 3).unwrap(); n.set(0, &[1]).unwrap(); let lock = n.lock(0).unwrap(); @@ -518,7 +521,7 @@ mod tests { #[test] fn append_empty_slice_is_noop() { - let n = Neighbors::new(1, 4).unwrap(); + let n = Neighbors::new(IdLimit::new(1), 4).unwrap(); n.set(0, &[10, 20]).unwrap(); let lock = n.lock(0).unwrap(); @@ -531,7 +534,7 @@ mod tests { #[test] fn write_overwrites_longer_list() { - let n = Neighbors::new(1, 5).unwrap(); + let n = Neighbors::new(IdLimit::new(1), 5).unwrap(); n.set(0, &[1, 2, 3, 4, 5]).unwrap(); // Overwrite with a shorter list. @@ -564,9 +567,9 @@ mod tests { #[test] fn basic_test() { - let mut neighbors = Neighbors::new(10, 4).unwrap(); + let mut neighbors = Neighbors::new(IdLimit::new(10), 4).unwrap(); assert_eq!(neighbors.entries(), 10); - assert_eq!(neighbors.max_length(), 4); + assert_eq!(neighbors.max_degree(), MaxDegree::new(4)); let mut list = AdjacencyList::new(); for i in 0..neighbors.entries() { @@ -576,7 +579,7 @@ mod tests { assert!(list.is_empty()); let lock = neighbors.lock(i).unwrap(); - assert_eq!(lock.capacity(), neighbors.max_length()); + assert_eq!(lock.capacity(), neighbors.max_degree().value()); assert_eq!(lock.len(), 0); assert!(lock.is_empty()); assert_eq!(lock.as_slice(), &[]); @@ -595,7 +598,7 @@ mod tests { |round: u32, entry: u32| -> Vec { (0..(round + 1)).map(|r| entry + r).collect() }; // Test mutation via `Neighbors::set`. - for round in 0..neighbors.max_length_u32() { + for round in 0..neighbors.max_degree_u32() { for i in 0..neighbors.entries() { let v = generate(round, i); neighbors.set(i, &v).unwrap(); @@ -614,7 +617,7 @@ mod tests { clear(&mut neighbors); // Test mutation via `lock + write`. - for round in 0..neighbors.max_length_u32() { + for round in 0..neighbors.max_degree_u32() { for i in 0..neighbors.entries() { let v = generate(round, i); neighbors.lock(i).unwrap().write(&v).unwrap(); @@ -633,7 +636,7 @@ mod tests { clear(&mut neighbors); // Test mutation via `lock + append`. - for round in 0..neighbors.max_length_u32() { + for round in 0..neighbors.max_degree_u32() { for i in 0..neighbors.entries() { neighbors.lock(i).unwrap().append(&[round + i]).unwrap(); } @@ -660,7 +663,7 @@ mod tests { #[test] fn lock_blocks_get() { for _ in 0..10 { - let neighbors = Neighbors::new(3, 4).unwrap(); + let neighbors = Neighbors::new(IdLimit::new(3), 4).unwrap(); let seq = Sequencer::new(); std::thread::scope(|s| { @@ -684,9 +687,9 @@ mod tests { #[test] fn many_appends() { - let max_length = if cfg!(miri) { 100 } else { 1000 }; + let max_degree = if cfg!(miri) { 100 } else { 1000 }; - let neighbors = Neighbors::new(1, max_length).unwrap(); + let neighbors = Neighbors::new(IdLimit::new(1), max_degree).unwrap(); let num_threads = 4; let barrier = std::sync::Barrier::new(num_threads); @@ -699,7 +702,7 @@ mod tests { s.spawn(move || { barrier_ref.wait(); let mut i = thread_id as u32; - let upper = neighbors_ref.max_length() as u32; + let upper = neighbors_ref.max_degree_u32(); while i < upper { neighbors_ref.lock(0).unwrap().append(&[i]).unwrap(); i += num_threads as u32; @@ -709,7 +712,9 @@ mod tests { }); let mut list = AdjacencyList::new(); - let expected: Vec<_> = (0..neighbors.max_length()).map(|i| i as u32).collect(); + let expected: Vec<_> = (0..neighbors.max_degree().value()) + .map(|i| i as u32) + .collect(); neighbors.get(0, &mut list).unwrap(); list.sort(); diff --git a/diskann-inmem/src/num.rs b/diskann-inmem/src/num.rs index 98c20d82be..2d734c11ec 100644 --- a/diskann-inmem/src/num.rs +++ b/diskann-inmem/src/num.rs @@ -46,12 +46,6 @@ impl Bytes { } } - /// Perform integer division of `self` by `other`. - #[inline] - pub(crate) const fn div(self, other: NonZeroUsize) -> Bytes { - Bytes::new(self.value() / other.get()) - } - /// Subtract `other` from `self` without checking for underflow. #[inline] pub(crate) const fn unchecked_sub(self, other: Bytes) -> Bytes { @@ -161,6 +155,82 @@ impl std::fmt::Display for Align { } } +//-------------------------// +// General Number Wrappers // +//-------------------------// + +macro_rules! typed_int { + ($(#[$doc:meta])* $vis:vis $name:ident, $T:ty $(,)?) => { + $(#[$doc])* + #[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] + #[repr(transparent)] + $vis struct $name($T); + + impl $name { + $vis const fn new(value: $T) -> Self { + Self(value) + } + + $vis const fn value(self) -> $T { + self.0 + } + } + + impl std::fmt::Display for $name { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, concat!(stringify!($name), "({})"), self.value()) + } + } + }; +} + +// TODO: Provide a linkable reference for "immutable" points. + +typed_int!( + /// The number of distinct slots a [`crate::Provider`] or [`crate::repr::Representation`] + /// has capacity for. This is logically distinct from [`IdLimit`], which may be greater + /// due to immutable points within a storage container. + pub Capacity, + usize, +); + +typed_int!( + /// The maximum degree of an adjacency list. + pub MaxDegree, + usize +); + +typed_int!( + /// One larger than the maximum ID that a [`crate::Provider`], + /// [`crate::repr::Representation`], or other such store in this crate can access in-bounds. + /// + /// This implies that access to ids `[0..self)` are in-bounds. + /// + /// [`Capacity`] is related, but the [`IdLimit`] for a collection may be larger due to + /// immutable points. + pub IdLimit, + u32 +); + +impl IdLimit { + /// Return `true` if `i` is within `[0..self)`. + pub const fn is_in_bounds(self, i: u32) -> bool { + i < self.value() + } + + /// Return the [`Self::value`] as a [`usize`]. + pub const fn as_usize(self) -> usize { + // We cannot use the `IntoUsize` trait in a const function unfortunately. + // + // Instead, we need to re-create the check that makes this conversion safe. + const { + assert!(std::mem::size_of::() <= std::mem::size_of::()); + } + + self.value() as usize + } +} + /////////// // Tests // /////////// diff --git a/diskann-inmem/src/prefetch.rs b/diskann-inmem/src/prefetch.rs new file mode 100644 index 0000000000..c6997bab0f --- /dev/null +++ b/diskann-inmem/src/prefetch.rs @@ -0,0 +1,263 @@ +/* + * Copyright (c) Microsoft Corporation. + * Licensed under the MIT license. + */ + +//! Utilities for prefetching. + +use crate::num::Bytes; + +/// A validated [`Prefetch`]. +#[derive(Debug, Clone, Copy)] +pub(crate) struct Checked

(P); + +impl

Checked

+where + P: Prefetch, +{ + /// Construct a new [`Checked`] containing `prefetcher`, validating the prefetcher against + /// `len`. + pub(crate) fn new(prefetcher: P, len: Bytes) -> Result { + prefetcher.check(len)?; + Ok(Self(prefetcher)) + } + + /// Prefetch the slice defined by `[ptr, ptr.add(len.value()))`. + /// + /// # Safety + /// + /// * The slice must point to valid memory within a single allocation. There are no + /// aliasing requirements. + /// + /// * `self` must be compatible with `len`, either through [`Self::new`] or [`Self::check`]. + pub(crate) unsafe fn prefetch(self, ptr: *const u8, len: Bytes) { + debug_assert!(self.check(len).is_ok()); + + // SAFETY: Inherited from caller. + unsafe { self.0.prefetch(ptr, len) } + } + + /// Check if `self` can prefetch slices of length `len`. + pub(crate) fn check(self, len: Bytes) -> Result<(), InvalidPrefetch> { + self.0.check(len) + } + + #[cfg(test)] + fn safe_prefetch(self, x: &[u8]) -> Result<(), InvalidPrefetch> { + let bytes = Bytes::new(x.len()); + self.check(bytes)?; + + // SAFETY: We've checked the length, and slices satisfy the memory and lifetime + // requirements. + unsafe { self.prefetch(x.as_ptr(), bytes) }; + Ok(()) + } +} + +/// Prefetch contiguous chunks of memory. +/// +/// Prefetchers are created ahead of time and reused. This allows specialized prefetchers +/// (e.g. fully or partially unrolled) to be created. +/// +/// # Safety +/// +/// The function [`Self::check`] **must** be accurate. A successful return from +/// [`Self::check`] must imply that [`Self::prefetch`] on a valid slice of that length is safe. +pub(crate) unsafe trait Prefetch: + std::fmt::Debug + Send + Sync + 'static + Copy +{ + /// Check that slices of length `len` are compatible with this prefetcher. + /// + /// If this function returns `Ok(())`, calling [`Self::prefetch`] with a valid slice of + /// length `len` must be safe. + fn check(self, len: Bytes) -> Result<(), InvalidPrefetch>; + + /// Prefetch the slice defined by `[ptr, ptr.add(len.value()))`. + /// + /// # Safety + /// + /// * The slice must point to valid memory within a single allocation. There are no + /// aliasing requirements. + /// + /// * [`Self::check`] must return `Ok(())` for `len`. + unsafe fn prefetch(self, ptr: *const u8, len: Bytes); +} + +/// A call to [`Prefetch::check`] failed. +#[derive(Debug)] +pub(crate) struct InvalidPrefetch(()); + +impl InvalidPrefetch { + const fn new() -> Self { + Self(()) + } +} + +impl std::fmt::Display for InvalidPrefetch { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "invalid prefetch") + } +} + +impl std::error::Error for InvalidPrefetch {} + +diskann::convert_error!(InvalidPrefetch); + +/// Prefetch data using a simple `for` loop. +#[derive(Debug, Clone, Copy)] +pub(crate) struct Loop(()); + +impl Loop { + /// Construct a new `Loop`. + pub(crate) const fn new() -> Self { + Self(()) + } +} + +// SAFETY: The `Loop` prefetcher is compatible with all lengths of slices as long as the +// provided slice is valid. +unsafe impl Prefetch for Loop { + fn check(self, _len: Bytes) -> Result<(), InvalidPrefetch> { + Ok(()) + } + + #[inline(always)] + unsafe fn prefetch(self, ptr: *const u8, len: Bytes) { + // SAFETY: Inherited from caller. + unsafe { prefetch(ptr, len.value()) } + } +} + +/// A prefetcher for a fixed number of bytes. +#[derive(Debug, Clone, Copy)] +pub(crate) struct Unrolled(()); + +impl Unrolled { + /// Construct a new `Unrolled`. + pub(crate) const fn new() -> Self { + Self(()) + } +} + +// SAFETY: This prefetcher is only valid for slices of length `BYTES`, which is correctly +// reported in the implementation of `check`. +unsafe impl Prefetch for Unrolled { + fn check(self, bytes: Bytes) -> Result<(), InvalidPrefetch> { + if bytes == Bytes::new(BYTES) { + Ok(()) + } else { + Err(InvalidPrefetch::new()) + } + } + + #[inline(always)] + unsafe fn prefetch(self, ptr: *const u8, _len: Bytes) { + debug_assert!(self.check(_len).is_ok()); + + // SAFETY: Inherited from caller. + unsafe { prefetch(ptr, BYTES) } + } +} + +//------------------------// +// Architecture Dependent // +//------------------------// + +/// Prefetch `len` bytes beginning at `ptr`. +/// +/// Prefetch locations are spaced one cache-line width apart. The final location is +/// prefetched first, followed by the rest in ascending order. +/// +/// # Safety +/// +/// The memory range `[ptr, ptr.add(len))` must be valid. +#[cfg(all(target_arch = "x86_64", target_feature = "avx2"))] +#[inline(always)] +pub(crate) unsafe fn prefetch(ptr: *const u8, len: usize) { + use std::arch::x86_64::*; + + // Fetch the final location (the one containing the tag) first. + let stride = Bytes::CACHELINE.value(); + let ptr = ptr.cast::(); + let lines = len.div_ceil(stride); + if lines == 0 { + return; + } + + // SAFETY: Inherited from caller. + unsafe { _mm_prefetch(ptr.add(stride * (lines - 1)), _MM_HINT_T0) }; + for i in 0..(lines - 1) { + // SAFETY: Inherited from caller. + unsafe { + _mm_prefetch(ptr.add(stride * i), _MM_HINT_T0); + } + } +} + +/// Prefetch `len` bytes beginning at `ptr`. +/// +/// Prefetch locations are spaced one cache-line width apart. The final location is +/// prefetched first, followed by the rest in ascending order. +/// +/// # Safety +/// +/// The memory range `[ptr, ptr.add(len))` must be valid. +#[cfg(not(all(target_arch = "x86_64", target_feature = "avx2")))] +pub(crate) unsafe fn prefetch(_ptr: *const u8, _len: usize) {} + +/////////// +// Tests // +/////////// + +#[cfg(test)] +mod test { + use super::*; + + // The safety of this test is mainly dependent on running under Miri. + #[test] + fn test_loop() { + let p = Loop::new(); + for i in 0..=1024 { + let v = vec![0u8; i]; + + let checked = Checked::new(p, Bytes::new(v.len())).unwrap(); + checked.safe_prefetch(&v).unwrap(); + } + } + + fn unrolled_prefetch() { + let p = Unrolled::::new(); + + // Happy Path + { + let c = Checked::new(p, Bytes::new(BYTES)).unwrap(); + let v = vec![0u8; BYTES]; + c.safe_prefetch(&v).unwrap(); + } + + if let Some(under) = BYTES.checked_sub(1) { + assert!(Checked::new(p, Bytes::new(under)).is_err()); + let v = vec![0u8; under]; + let c = Checked::new(p, Bytes::new(BYTES)).unwrap(); + assert!(c.safe_prefetch(&v).is_err()); + } + + if let Some(over) = BYTES.checked_add(1) { + assert!(Checked::new(p, Bytes::new(over)).is_err()); + let v = vec![0u8; over]; + let c = Checked::new(p, Bytes::new(BYTES)).unwrap(); + assert!(c.safe_prefetch(&v).is_err()); + } + } + + #[test] + fn test_unrolled() { + unrolled_prefetch::<0>(); + unrolled_prefetch::<63>(); + unrolled_prefetch::<64>(); + unrolled_prefetch::<65>(); + unrolled_prefetch::<127>(); + unrolled_prefetch::<128>(); + unrolled_prefetch::<129>(); + } +} diff --git a/diskann-inmem/src/provider.rs b/diskann-inmem/src/provider.rs index b8286ac141..d2d2c8f82e 100644 --- a/diskann-inmem/src/provider.rs +++ b/diskann-inmem/src/provider.rs @@ -30,10 +30,7 @@ //! * Lack of save/load support: The index is currently ephemeral, but there are plans to //! address this gap. -use std::{ - hash::Hash, - num::{NonZeroU32, NonZeroUsize}, -}; +use std::hash::Hash; use diskann::{ ANNError, ANNResult, @@ -44,17 +41,14 @@ use diskann::{ }, neighbor::Neighbor, provider, - utils::IntoUsize, }; -use diskann_utils::views::Matrix; -use thiserror::Error; use crate::{ counters::{Counters, LocalCounters}, ids::IdMap, - layers::{self, QueryDistance}, - num::Bytes, - store::{self, Store}, + neighbors::Neighbors, + num::{IdLimit, MaxDegree}, + repr, }; /// Aggregate trait for the external ID type of [`Provider`]. @@ -64,82 +58,32 @@ impl Id for T where T: Send + Sync + Hash + Eq + Clone + 'static {} /// An in-memory data-provider for DiskANN's graph indexing algorithms. /// -/// The first type parameter `L` is a [`layers::Layer`] for describing the kind of data +/// The first type parameter `R` is a [`repr::Representation`] for describing the kind of data /// stored within the provider. The second parameter `M` is the associated data for items /// inserted into the provider. #[derive(Debug)] -pub struct Provider +pub struct Provider where M: Id, { - // The raw binary store - store: Store, - // Data representation. - layer: L, + // Data representation and storage. + representation: R, // ID translation. mapping: IdMap, - // Construction `Config`. - config: Config, - // `Counters` is only non-trivial under the `integration-test` feature flag. Otherwise, // all counter related operations are no-ops. counters: Counters, } -impl Provider +impl Provider where M: Id, { - /// Construct a new [`Provider`]. - /// - /// The list of `start_points` must be must be compatible with `layer`. - pub fn new(layer: L, config: Config, start_points: I) -> Result - where - I: IntoIterator, - L: layers::Set, - { - let start_points: Vec<_> = start_points.into_iter().collect(); - let bytes = layers::Layer::bytes(&layer); - let mut data = Matrix::new(0u8, start_points.len(), bytes.value()); - - for (row, point) in std::iter::zip(data.row_iter_mut(), start_points) { - layers::Set::set(&layer, point, row)?; - } - - let mut store_config = store::Config::new(config.capacity(), bytes, config.max_degree()); - - if let Some(slots) = config.epoch_guard_slots { - store_config.epoch_guard_slots(slots); - } - - if let Some(capacity) = config.freelist_recycle_capacity { - store_config.freelist_recycle_capacity(capacity); - } - - let store = Store::new(store_config, data.as_view()) - .map_err(|err| ProviderError::CreatingStore(Box::new(err)))?; - - let mapping = IdMap::new(config.capacity()); - - Ok(Self { - store, - layer, - mapping, - config, - counters: Counters::new(), - }) - } - /// A local set of counters that update the provider-wide counters in bulk. fn local_counters(&self) -> LocalCounters<'_> { self.counters.local() } - /// Return the maximum number of neighbors that can be stored in the provider's graph. - pub fn max_degree(&self) -> usize { - self.store.max_degree() - } - /// Return a snapshot of the current event counters. #[cfg(feature = "integration-test")] pub fn counters(&self) -> crate::integration::counters::CounterSnapshot { @@ -147,72 +91,29 @@ where } } -#[derive(Debug, Error)] -pub enum ProviderError { - #[error("error when trying to set start points")] - SettingStartPoints(#[from] ANNError), - #[error("could not create data store")] - CreatingStore(#[source] Box), -} - -/// Configuration for [`Provider`]. -#[derive(Debug)] -pub struct Config { - capacity: usize, - max_degree: usize, - prefetch_lookahead: Option, - epoch_guard_slots: Option, - freelist_recycle_capacity: Option, -} - -impl Config { - const DEFAULT_PREFETCH_LOOKAHEAD: NonZeroUsize = NonZeroUsize::new(8).unwrap(); - - /// Construct a new [`Config`]. - /// - /// * `capacity`: The number of dynamic entries in the resulting provider. - /// * `max_degree`: The maximum degree of any adjacency list in the graph. - pub fn new(capacity: usize, max_degree: usize) -> Self { - Self { - capacity, - max_degree, - prefetch_lookahead: Some(Self::DEFAULT_PREFETCH_LOOKAHEAD), - epoch_guard_slots: None, - freelist_recycle_capacity: None, - } - } - - /// Return the number of dynamic entries in the resulting provider. - pub fn capacity(&self) -> usize { - self.capacity - } - - /// Return the maximum degree of any adjacency list. - pub fn max_degree(&self) -> usize { - self.max_degree - } - - /// Configure the prefetch lookahead. - /// - /// This is used during beam expansion to prefetch data into CPU caches. - pub fn set_prefetch_lookahead(&mut self, prefetch_lookahead: Option) { - self.prefetch_lookahead = prefetch_lookahead; - } +impl Provider +where + R: repr::Representation, + M: Id, +{ + /// Construct a new [`Provider`]. + pub fn new(config: C) -> ANNResult + where + C: repr::RepresentationConfig, + { + let representation = <_ as repr::RepresentationConfig>::build(config)?; + let mapping = IdMap::new(representation.capacity()); - /// Configure the number of epoch guard slots. - /// - /// Increasing this number will increase the number of threads that can work concurrently - /// on the index at the cost of longer scan times for epoch advancement. - pub fn set_epoch_guard_slots(&mut self, slots: Option) { - self.epoch_guard_slots = slots; + Ok(Self { + representation, + mapping, + counters: Counters::new(), + }) } - /// Configure the capacity of the freelist recycle queue. - /// - /// Increasing the capacity of the queue will allow more recycled IDs to be retrieved - /// without triggering a scan, but will cost more memory. - pub fn set_freelist_recycle_capacity(&mut self, capacity: Option) { - self.freelist_recycle_capacity = capacity; + /// Return the maximum number of neighbors that can be stored in the provider's graph. + pub fn max_degree(&self) -> MaxDegree { + self.representation.max_degree() } } @@ -266,9 +167,9 @@ where // // `diskann` has plans to move deletion checks behind an accessor trait, which will help // with this situation. -impl diskann::provider::Delete for Provider +impl diskann::provider::Delete for Provider where - L: Send + Sync + 'static, + R: repr::Representation, M: Id, { async fn delete(&self, _context: &Context, gid: &M) -> ANNResult<()> { @@ -283,14 +184,14 @@ where Some(e) => e, }; - match self.store.retire(entry.internal().into_usize()) { - Ok(()) => { - // Successfully retired the internal slot. We can safely release the ID mapping. - entry.delete(); - Ok(()) - } - Err(err) => Err(ANNError::new(err)), - } + // An early return here will cause `entry` to be dropped, which will *not* cause + // the delete to commit. + ::retire(&self.representation, entry.internal())?; + + // Successfully retired the internal slot. We can safely release the ID mapping. + entry.delete(); + + Ok(()) } async fn release(&self, _context: &Context, _id: Self::InternalId) -> ANNResult<()> { @@ -302,9 +203,7 @@ where _context: &Context, id: u32, ) -> ANNResult { - // Not that this check is approximate. A full check requires materialization of - // a `reader`. - match self.store.can_read_approximate(id.into_usize()) { + match ::is_readable(&self.representation, id) { Some(true) => Ok(diskann::provider::ElementStatus::Valid), Some(false) => Ok(diskann::provider::ElementStatus::Deleted), None => Err(ANNError::message("accessed invalid internal ID")), @@ -331,9 +230,9 @@ where std::future::ready(f()) } -impl diskann::provider::SetElement for Provider +impl diskann::provider::SetElement for Provider where - L: layers::Set, + R: repr::Set, M: Id, { type SetError = ANNError; @@ -345,19 +244,18 @@ where element: T, ) -> impl std::future::Future> + Send { let work = move || { - let mut slot = self - .store - .acquire() - .ok_or_else(|| ANNError::message("could not allocate a new slot"))?; - // TODO: Proper cleanup via `Guard` or some other mechanism on the event of // insert failure after `set_element` returns. - >::set(&self.layer, element, slot.as_mut_slice())?; - self.mapping.insert(id.clone(), slot.slot())?; + // + // The internal `Guard` is sufficient for local rollback, but not after + // `set_element` returns. + let guard = >::set(&self.representation, element)?; + let internal = <_ as repr::Guard>::id(&guard); + self.mapping.insert(id.clone(), internal)?; // Now that insert has succeeded - publish the slot. This method cannot fail, so // we do not need to worry about potentially unwinding the ID mapping. - let id = slot.publish(); + <_ as repr::Guard>::publish(guard); // This is a rather expensive update. // @@ -365,7 +263,7 @@ where // is not expected to be enabled for general use. self.local_counters().set_vector(1); - Ok(diskann::provider::NoopGuard::new(id)) + Ok(diskann::provider::NoopGuard::new(internal)) }; ready(work) @@ -383,9 +281,10 @@ where /// kernel once and reuse it to balance compile times and performance. #[derive(Debug)] pub struct SearchAccessor<'a> { - reader: store::Reader<'a>, + neighbors: &'a Neighbors, ids: AdjacencyList, - expand_beam: Box, + expand_beam: Box, + id_limit: IdLimit, buffer: Vec<(u32, f32)>, // The parent provider for the accessor. @@ -394,6 +293,28 @@ pub struct SearchAccessor<'a> { counters: LocalCounters<'a>, } +impl<'a> SearchAccessor<'a> { + pub(crate) fn new( + neighbors: &'a Neighbors, + expand_beam: Box, + provider: &'a (dyn std::any::Any + Send + Sync), + start_points: std::ops::Range, + counters: LocalCounters<'a>, + ) -> Self { + let id_limit = expand_beam.id_limit(); + Self { + neighbors, + ids: AdjacencyList::with_capacity(neighbors.max_degree().value()), + expand_beam, + id_limit, + buffer: vec![Default::default(); neighbors.max_degree().value()], + provider, + start_points, + counters, + } + } +} + impl diskann::provider::HasId for SearchAccessor<'_> { type Id = u32; } @@ -414,13 +335,12 @@ impl glue::SearchAccessor for SearchAccessor<'_> { { let work = move || { for p in self.start_points.clone() { - match self.reader.read(p.into_usize()) { - Some(point) => { + match self.expand_beam.evaluate(p)? { + Some(distance) => { // Counters are no-ops without `integration-test`. self.counters.get_vector(1); self.counters.query_distance(1); - - f(p, self.expand_beam.evaluate(point)?); + f(p, distance); } None => { return Err(ANNError::message("could not retrieve start point")); @@ -446,22 +366,25 @@ impl glue::SearchAccessor for SearchAccessor<'_> { { let work = move || -> ANNResult<()> { for i in ids { - self.reader.neighbors().get(i, &mut self.ids)?; + self.neighbors.get(i, &mut self.ids)?; self.counters.get_neighbors(1); // Filter out unvisited IDs and ensure that all the IDs we are about self.ids - .retain(|i| pred.eval_mut(i) && self.reader.is_in_bounds(i.into_usize())); + .retain(|i| pred.eval_mut(i) && self.id_limit.is_in_bounds(*i)); // This should always hold, but let's double check. assert!(self.buffer.len() >= self.ids.len()); - // SAFETY: We've verified that each entry in `self.ids` is in-bounds and the - // `self.buffer` is long enough to hold all the IDs. - let processed = unsafe { - self.expand_beam - .expand_beam(&self.ids, &self.reader, &mut self.buffer) - }?; + // SAFETY: + // + // 1. We've verified that each entry in `self.ids` is in-bounds using + // the `IdLimit` derived from this `ExpandBeam` object (enforced in the + // constructor). + // + // 2. `self.buffer` is long enough to hold all the IDs. + let processed = + unsafe { self.expand_beam.expand_beam(&self.ids, &mut self.buffer) }?; self.counters.get_vector(processed as u64); self.counters.query_distance(processed as u64); @@ -479,216 +402,6 @@ impl glue::SearchAccessor for SearchAccessor<'_> { } } -trait ExpandBeam: Send + Sync + std::fmt::Debug { - /// Evaluate a raw distance function. - fn evaluate(&self, x: &[u8]) -> ANNResult; - - /// Compute the distance between the query and each neighbor in `list`. - /// - /// # Safety - /// - /// * All items in `list` must in-bounds with respect to `reader`. - /// * `buffer.len() >= list.len()`. - unsafe fn expand_beam( - &self, - list: &[u32], - reader: &store::Reader<'_>, - buffer: &mut [(u32, f32)], - ) -> ANNResult; -} - -#[derive(Debug)] -struct ExpandBeamImpl { - inner: T, - prefetch_lookahead: usize, -} - -impl ExpandBeamImpl { - fn new(inner: T, prefetch_lookahead: usize) -> Self { - Self { - inner, - prefetch_lookahead, - } - } -} - -impl ExpandBeam for ExpandBeamImpl -where - T: layers::QueryDistance, -{ - fn evaluate(&self, x: &[u8]) -> ANNResult { - self.inner.evaluate(x) - } - - unsafe fn expand_beam( - &self, - list: &[u32], - reader: &store::Reader<'_>, - buffer: &mut [(u32, f32)], - ) -> ANNResult { - // SAFETY: Inherited from caller. - unsafe { - expand_beam_inner::( - &self.inner, - list, - self.prefetch_lookahead, - reader, - buffer, - ) - } - } -} - -#[derive(Debug)] -struct ExpandBeamVisitor { - bytes: Bytes, - prefetch_lookahead: usize, -} - -impl<'a> layers::QueryVisitor<'a> for ExpandBeamVisitor { - type Output = Box; - - fn visit_sized(self, distance: T) -> Self::Output - where - T: QueryDistance + 'a, - { - // This is critical to ensure we emit the correct number of prefetches. - assert!(Bytes::new(BYTES + store::TAG_SIZE.value()) <= self.bytes); - Box::new(ExpandBeamImpl::<_, BYTES>::new( - distance, - self.prefetch_lookahead, - )) - } - - fn visit(self, distance: T) -> Self::Output - where - T: QueryDistance + 'a, - { - Box::new(ExpandBeamImpl::<_, 0>::new( - distance, - self.prefetch_lookahead, - )) - } -} - -/// Prefetch `len` bytes beginning at `ptr`. -/// -/// The last cache line prefetched first, followed by the rest in ascending order. -/// -/// # Safety -/// -/// The memory range `[ptr, ptr.add(len))` must be valid. -#[cfg(all(target_arch = "x86_64", target_feature = "avx2"))] -#[inline(always)] -unsafe fn prefetch(ptr: *const u8, len: usize) { - use std::arch::x86_64::*; - - // Fetch the last cache line (the one with the tag) first. - let stride = Bytes::CACHELINE.value(); - let ptr = ptr.cast::(); - let lines = len.div_ceil(stride); - if lines == 0 { - return; - } - - // SAFETY: Inherited from caller. - unsafe { _mm_prefetch(ptr.add(stride * (lines - 1)), _MM_HINT_T0) }; - for i in 0..(lines - 1) { - // SAFETY: Inherited from caller. - unsafe { - _mm_prefetch(ptr.add(stride * i), _MM_HINT_T0); - } - } -} - -/// Prefetch `len` bytes beginning at `ptr`. -/// -/// The last cache line prefetched first, followed by the rest in ascending order. -/// -/// # Safety -/// -/// The memory range `[ptr, ptr.add(len))` must be valid. -#[cfg(not(all(target_arch = "x86_64", target_feature = "avx2")))] -unsafe fn prefetch(_ptr: *const u8, _len: usize) {} - -/// # Safety -/// -/// * All items in `list` must in-bounds with respect to `reader`. -/// * The number of bytes associated with `N` cache lines must "make sense". -/// * `buffer.len() >= list.len()`. -#[inline] -unsafe fn expand_beam_inner( - distance: &T, - list: &[u32], - lookahead: usize, - reader: &store::Reader<'_>, - buffer: &mut [(u32, f32)], -) -> ANNResult -where - T: layers::QueryDistance, -{ - debug_assert!( - BYTES + store::TAG_SIZE.value() <= reader.bytes().value(), - "we really rely on this: {}, bytes = {}", - BYTES + store::TAG_SIZE.value(), - reader.bytes() - ); - - debug_assert!(buffer.len() >= list.len()); - - let bytes = if BYTES == 0 { - reader.bytes().value() - } else { - BYTES + store::TAG_SIZE.value() - }; - - let len = list.len(); - let lookahead = lookahead.min(len); - - for j in 0..lookahead { - // SAFETY: The in-bounds constraint is assured by the caller, both for `j` as well - // as the validity of the prefetch bounds. - unsafe { - prefetch( - reader - .read_raw_unchecked(list.get_unchecked(j).into_usize()) - .as_ptr() - .cast(), - bytes, - ) - } - } - - // Disable prefetching if the lookahead is 0. - let mut j = if lookahead == 0 { len } else { lookahead }; - let mut processed = 0; - for &i in list.iter() { - if j != len { - // SAFETY: The in-bounds constraint is assured by the caller, both for `j` as - // well as the validity of the prefetch bounds. - unsafe { - prefetch( - reader - .read_raw_unchecked(list.get_unchecked(j).into_usize()) - .as_ptr() - .cast(), - bytes, - ) - } - j += 1; - } - - // SAFETY: Caller asserts that `i` is in-bounds. - if let Some(data) = unsafe { reader.read_in_bounds(i.into_usize()) } { - // SAFETY: Inherited from caller. - *unsafe { buffer.get_unchecked_mut(processed) } = (i, distance.evaluate(data)?); - processed += 1; - } - } - - Ok(processed) -} - //////////// // Insert // //////////// @@ -698,33 +411,57 @@ where /// This type implements zero-copy access to the data within its parent provider during prunes. #[derive(Debug)] pub struct PruneAccessor<'a> { - reader: store::Reader<'a>, - distance: &'a dyn layers::Distance, + prune: Box, + keys: hashbrown::HashMap>, + neighbors: &'a Neighbors, counters: LocalCounters<'a>, } +impl<'a> PruneAccessor<'a> { + pub(crate) fn new( + prune: Box, + neighbors: &'a Neighbors, + counters: LocalCounters<'a>, + ) -> Self { + Self { + prune, + keys: hashbrown::HashMap::new(), + neighbors, + counters, + } + } +} + /// The distance computer for [`PruneAccessor`]. #[derive(Debug)] pub struct Distance<'a> { - distance: &'a dyn layers::Distance, + prune: &'a dyn repr::Prune, counters: LocalCounters<'a>, } impl<'a> Distance<'a> { - fn new(distance: &'a dyn layers::Distance, counters: LocalCounters<'a>) -> Self { - Self { distance, counters } + fn new(prune: &'a dyn repr::Prune, counters: LocalCounters<'a>) -> Self { + Self { prune, counters } } } -#[expect( - clippy::unwrap_used, - reason = "prune does not allow fallible distance functions yet" -)] -impl diskann_vector::DistanceFunction<&[u8], &[u8], f32> for Distance<'_> { +/// An opaque element-ref for [`PruneAccessor`]. +#[derive(Debug, Clone, Copy)] +#[repr(transparent)] +pub struct ElementRef(repr::PruneKey); + +impl<'a> diskann_utils::Reborrow<'a> for ElementRef { + type Target = ElementRef; + fn reborrow(&'a self) -> Self { + *self + } +} + +impl diskann_vector::DistanceFunction for Distance<'_> { #[inline] - fn evaluate_similarity(&self, x: &[u8], y: &[u8]) -> f32 { + fn evaluate_similarity(&self, x: ElementRef, y: ElementRef) -> f32 { self.counters.distance_ref(1); - self.distance.evaluate(x, y).unwrap() + self.prune.evaluate(x.0, y.0) } } @@ -738,7 +475,7 @@ impl glue::PruneAccessor for PruneAccessor<'_> { where Self: 'a; - type ElementRef<'a> = &'a [u8]; + type ElementRef<'a> = ElementRef; type View<'a> = &'a Self @@ -756,12 +493,17 @@ impl glue::PruneAccessor for PruneAccessor<'_> { async fn fill<'a, Itr>( &'a mut self, - _itr: Itr, + itr: Itr, ) -> ANNResult<(Self::View<'a>, Self::Distance<'a>)> where Itr: ExactSizeIterator + Clone + Send + Sync, { - Ok((self, Distance::new(self.distance, self.counters.fork()))) + self.keys.clear(); + self.keys.extend(itr.map(|i| (i, None))); + let count: usize = self.prune.prepare(self.keys.iter_mut())?; + self.counters.get_vector(count as u64); + + Ok((self, Distance::new(&*self.prune, self.counters.fork()))) } } @@ -773,7 +515,7 @@ impl provider::NeighborAccessor for PruneAccessor<'_> { ) -> impl std::future::Future> + Send { let work = move || { self.counters.get_neighbors(1); - Ok(self.reader.neighbors().get(id, neighbors)?) + Ok(self.neighbors.get(id, neighbors)?) }; ready(work) } @@ -787,7 +529,7 @@ impl provider::NeighborAccessorMut for PruneAccessor<'_> { ) -> impl std::future::Future> + Send { let work = move || { self.counters.set_neighbors(1); - Ok(self.reader.neighbors().set(id, neighbors)?) + Ok(self.neighbors.set(id, neighbors)?) }; ready(work) } @@ -799,7 +541,7 @@ impl provider::NeighborAccessorMut for PruneAccessor<'_> { ) -> impl std::future::Future> + Send { let work = move || -> ANNResult<()> { self.counters.append_vector(1); - let lock = self.reader.neighbors().lock(id)?; + let lock = self.neighbors.lock(id)?; // Due to race conditions between calls to `get_neighbors` and `append_vector` // in `diskann` - it's possible that the state of the adjacency list has changed @@ -823,19 +565,14 @@ impl provider::NeighborAccessorMut for PruneAccessor<'_> { } impl workingset::View for &PruneAccessor<'_> { - type ElementRef<'a> = &'a [u8]; + type ElementRef<'a> = ElementRef; type Element<'a> - = &'a [u8] + = ElementRef where Self: 'a; - fn get(&self, id: u32) -> Option<&[u8]> { - match self.reader.read(id.into_usize()) { - Some(data) => { - self.counters.get_vector_ref(1); - Some(data) - } - None => None, - } + + fn get(&self, id: u32) -> Option { + self.keys.get(&id)?.map(ElementRef) } } @@ -846,9 +583,9 @@ impl workingset::View for &PruneAccessor<'_> { #[derive(Debug, Clone, Copy)] pub struct Strategy; -impl<'a, L, M> glue::SearchStrategy<'a, Provider, L::Query<'a>> for Strategy +impl<'a, R, M> glue::SearchStrategy<'a, Provider, R::Query<'a>> for Strategy where - L: layers::Search, + R: repr::Search, M: Id, { type SearchAccessor = SearchAccessor<'a>; @@ -856,57 +593,43 @@ where fn search_accessor( &'a self, - provider: &'a Provider, + provider: &'a Provider, _context: &'a Context, - query: L::Query<'a>, + query: R::Query<'a>, ) -> ANNResult> { - let reader = provider.store.reader()?; - let expand_beam = ::query_distance( - &provider.layer, + ::search_accessor( + &provider.representation, query, - ExpandBeamVisitor { - bytes: provider.store.bytes(), - prefetch_lookahead: provider.config.prefetch_lookahead.map_or(0, |x| x.get()), - }, - )?; - - let accessor = SearchAccessor { - reader, - ids: AdjacencyList::new(), - expand_beam, - buffer: vec![(0, 0.0); provider.max_degree()], provider, - start_points: provider.store.frozen(), - counters: provider.local_counters(), - }; - Ok(accessor) + provider.local_counters(), + ) } } // This is a utility for helping inspect the generated code for `ExpandBeam`. // pub fn test_function<'a>( - x: &'a Provider>, + x: &'a Provider>, strategy: &'a Strategy, context: &'a Context, - query: &'a [f32], + query: &'a [u8], ) -> ANNResult> { glue::SearchStrategy::search_accessor(strategy, x, context, query) } /// Perform ID translation during post-processing. #[derive(Debug, Clone, Copy)] -pub struct Translate(std::marker::PhantomData<(L, M)>); +pub struct Translate(std::marker::PhantomData<(R, M)>); -impl Default for Translate { +impl Default for Translate { fn default() -> Self { Self(std::marker::PhantomData) } } -impl<'a, L, M> glue::SearchPostProcess, L::Query<'a>, M> for Translate +impl<'a, R, M> glue::SearchPostProcess, R::Query<'a>, M> for Translate where - L: layers::Search, + R: repr::Search, M: Id, { type Error = ANNError; @@ -914,7 +637,7 @@ where fn post_process( &self, accessor: &mut SearchAccessor<'_>, - _query: L::Query<'a>, + _query: R::Query<'a>, candidates: I, output: &mut B, ) -> impl std::future::Future> + Send @@ -924,7 +647,7 @@ where { let work = move || { // By construction - the downcast should succeed. Otherwise, this is a program bug. - let provider = match accessor.provider.downcast_ref::>() { + let provider = match accessor.provider.downcast_ref::>() { Some(provider) => provider, None => return Err(ANNError::message("bad any cast")), }; @@ -949,17 +672,17 @@ where } } -impl<'a, L, M> glue::DefaultPostProcessor<'a, Provider, L::Query<'a>, M> for Strategy +impl<'a, R, M> glue::DefaultPostProcessor<'a, Provider, R::Query<'a>, M> for Strategy where - L: layers::Search, + R: repr::Search, M: Id, { - diskann::default_post_processor!(Translate); + diskann::default_post_processor!(Translate); } -impl glue::PruneStrategy> for Strategy +impl glue::PruneStrategy> for Strategy where - L: layers::Layer + layers::AsDistance, + R: repr::Insert, M: Id, { type PruneAccessor<'a> = PruneAccessor<'a>; @@ -967,21 +690,17 @@ where fn prune_accessor<'a>( &self, - provider: &'a Provider, + provider: &'a Provider, _context: &'a Context, _capacity: usize, ) -> ANNResult> { - Ok(PruneAccessor { - reader: provider.store.reader()?, - distance: ::as_distance(&provider.layer), - counters: provider.local_counters(), - }) + ::prune_accessor(&provider.representation, provider.local_counters()) } } -impl<'a, L, M> glue::InsertStrategy<'a, Provider, L::Query<'a>> for Strategy +impl<'a, R, M> glue::InsertStrategy<'a, Provider, R::Query<'a>> for Strategy where - L: layers::Insert, + R: repr::Insert, M: Id, { type PruneStrategy = Self; @@ -990,10 +709,10 @@ where } } -impl glue::InplaceDeleteStrategy, M>> for Strategy +impl glue::InplaceDeleteStrategy, M>> for Strategy where M: Id, - T: layers::FullPrecision, + T: repr::FullPrecision, { type DeleteElement<'a> = &'a [T]; type DeleteElementGuard = Box<[T]>; @@ -1018,26 +737,12 @@ where fn get_delete_element<'a>( &'a self, - provider: &'a Provider, M>, + provider: &'a Provider, M>, _context: &'a Context, id: u32, ) -> impl Future> + Send { - let work = move || { - let reader = provider.store.reader()?; - let data = match reader.read(id.into_usize()) { - Some(data) => data, - None => { - return Err(ANNError::message("item could not be read")); - } - }; - - let mut buf: Box<[_]> = - std::iter::repeat_n(T::zeroed(), provider.layer.dim()).collect(); - - bytemuck::must_cast_slice_mut::(&mut buf).copy_from_slice(data); - Ok(buf) - }; + let work = move || provider.representation.get(id); ready(work) } } @@ -1055,9 +760,10 @@ mod tests { neighbor::Neighbor, provider::{DataProvider, Delete}, }; + use diskann_utils::views::Matrix; use diskann_vector::distance::Metric; - use crate::layers::Full; + use crate::num::Capacity; /// The true tests live in the integration tests for this repo. /// @@ -1085,17 +791,24 @@ mod tests { let start = grid.start_point(size); let degree = 6; - let full = Full::::new(grid.dim().into(), Metric::L2); + let config = repr::full::Config::new( + Capacity::new(grid.num_points(size)), + MaxDegree::new(degree), + Metric::L2, + Matrix::row_vector(start.into()), + ) + .unwrap(); + + // let full = Full::::new(grid.dim().into(), Metric::L2); - let config = Config::new(grid.num_points(size), degree); + // let config = Config::new(grid.num_points(size), degree); - let provider = - Provider::<_, u64>::new(full, config, std::iter::once(start.as_slice())).unwrap(); - assert_eq!(provider.max_degree(), degree); + let provider = Provider::<_, u64>::new(config).unwrap(); + assert_eq!(provider.max_degree(), MaxDegree::new(degree)); let config = diskann::graph::config::Builder::new( 2 * (grid.dim() as usize), - diskann::graph::config::MaxDegree::new(provider.max_degree()), + diskann::graph::config::MaxDegree::new(provider.max_degree().value()), 10, (Metric::L2).into(), ) diff --git a/diskann-inmem/src/repr/full.rs b/diskann-inmem/src/repr/full.rs new file mode 100644 index 0000000000..743300e020 --- /dev/null +++ b/diskann-inmem/src/repr/full.rs @@ -0,0 +1,1558 @@ +/* + * Copyright (c) Microsoft Corporation. + * Licensed under the MIT license. + */ + +//! # Full-Precision +//! +//! A concurrent data store for [`crate::Provider`] enabling full-precision searches and +//! inserts for collections consisting of `f32`, `f16`, `u8`, or `i8` data types. +//! +//! The [`FullPrecision`] generic bound can be used to constrain these data types. + +mod internal_docs { + //! Internally, the [`super::repr::Search`] and [`super::repr::Insert`] traits + //! are implemented via [`super::FullPrecisionImpl`], which creates: + //! + //! * [`super::ExpandBeam`]: For index search. + //! * [`super::Prune`]: For index construction. + //! + //! These two structs are modular with respect to their exact distance function and + //! prefetcher. Since [`super::repr::ExpandBeam`] and [`super::repr::Prune`] are + //! used as trait objects, this allows the implementation structs in this module to be + //! highly specialized, including: + //! + //! * Inlining of distance functions. + //! * Specializing distance functions on dimension. + //! * Specializing prefetches on dimension. + //! * Dispatching to different micro-architecture levels. + //! * Specialized query preprocessing. + //! + //! Picking the best combination of all of these requires extensive experimentation. + //! The choices made here are mainly heuristic defaults, meant to try to balance + //! performance with compile time. + //! + //! Feel free to experiment and create optimized implementations for workloads that need it. +} + +use std::{fmt::Debug, marker::PhantomData, num::NonZeroUsize}; + +use diskann::{ANNError, ANNResult, utils::IntoUsize}; +use diskann_utils::views::Matrix; +use diskann_vector::{ + UnalignedSlice, + conversion::SliceCast, + distance::{ + Cosine, CosineNormalized, DistanceProvider, InnerProduct, Metric, Specialize, SquaredL2, + }, +}; +use diskann_wide::{ + ARCH, + arch::{Current, FTarget2}, +}; +use half::f16; +use thiserror::Error; + +use crate::{ + counters::LocalCounters, + epoch, + num::{Bytes, Capacity, IdLimit, MaxDegree}, + prefetch::{self, Prefetch}, + repr, + store::{ + self, Store, + invasive::{self, Invasive}, + }, + tag::AtomicTag, +}; + +/// A useful trait bound for types compatible with [`Full`]. +/// +/// This encompasses *everything* required for `Full: repr::Insert` and can be used as +/// a single bound. +pub trait FullPrecision: bytemuck::Pod + std::fmt::Debug + Send + Sync { + #[doc(hidden)] + fn __search_accessor<'a>( + representation: &'a Full, + query: &'a [Self], + provider: &'a (dyn std::any::Any + Send + Sync), + counters: LocalCounters<'a>, + ) -> ANNResult>; + + #[doc(hidden)] + fn __prune_accessor<'a>( + representation: &'a Full, + counters: LocalCounters<'a>, + ) -> ANNResult>; +} + +/// A configuration struct for [`Full`]. +#[derive(Debug, Clone)] +pub struct Config { + layout: store::Layout, + metric: Metric, + start_points: Matrix, + store: store::Config, + lookahead: Option, +} + +const DEFAULT_LOOKAHEAD: NonZeroUsize = NonZeroUsize::new(12).unwrap(); + +impl Config { + /// Create a new [`Config`] for a [`Full`]. + /// + /// The resulting store will hold `capacity` writable items and `start_points.nrows()` + /// frozen points at internal IDs `[capacity, capacity + start_points.nrows())`. The + /// dimensionality of the full-precision data will be inferred from + /// `start_points.ncols()`. + /// + /// The associated graph will be bounded with `max_degree` and `metric` will be used to + /// compute distances among the stored points. + /// + /// # Errors + /// + /// Returns an error if the number of start points exceeds `u32::MAX` or the number of + /// bytes required for each point exceeds `usize::MAX`. + pub fn new( + capacity: Capacity, + max_degree: MaxDegree, + metric: Metric, + start_points: Matrix, + ) -> Result { + let num_start_points: u32 = match start_points.nrows().try_into() { + Ok(points) => points, + Err(_) => return Err(ConfigError::TooManyStartPoints(start_points.nrows())), + }; + + // Check that we won't overflow when computing the number of bytes required for each + // data point. This can happen if `start_points` has 0 rows but a large number of + // columns. + if start_points + .ncols() + .checked_mul(std::mem::size_of::()) + .is_none() + { + return Err(ConfigError::DimTooLarge(start_points.ncols())); + } + + Ok(Self { + layout: store::Layout::new(capacity, max_degree, num_start_points), + metric, + start_points, + store: store::Config::default(), + lookahead: Some(DEFAULT_LOOKAHEAD), + }) + } + + /// Override the [`store::Config`] for tailoring concurrency details. + pub fn store(mut self, config: store::Config) -> Self { + self.store = config; + self + } + + /// Set the prefetch lookahead. + /// + /// This controls how many iterations ahead in + /// [`diskann::graph::glue::SearchAccessor::expand_beam`] data is prefetched into the CPU + /// cache. Passing `None` disables prefetching. + pub fn prefetch(mut self, lookahead: Option) -> Self { + self.lookahead = lookahead; + self + } + + /// Return the vector dimension of this configuration and the resulting [`Full`]. + pub fn dim(&self) -> usize { + self.start_points.ncols() + } + + /// Construct a [`Full`] from the [`Config`]. + pub fn build(self) -> ANNResult> + where + T: FullPrecision, + { + Full::new(self) + } +} + +/// Errors that can arise when constructing [`Config`]. +#[derive(Debug, Error)] +#[non_exhaustive] +pub enum ConfigError { + #[error("{} start points exceed `u32::MAX`", 0)] + TooManyStartPoints(usize), + #[error( + "the number of bytes to hold {}-dimensional data exceeds `usize::MAX`", + 0 + )] + DimTooLarge(usize), +} + +diskann::convert_error!(ConfigError); + +impl repr::RepresentationConfig for Config +where + T: FullPrecision, +{ + type Representation = Full; + + fn build(self) -> ANNResult> { + >::build(self) + } +} + +/// Internal helper for implementing [`FullPrecision`]. +trait FullPrecisionImpl: bytemuck::Pod + std::fmt::Debug + Send + Sync { + fn make_expand_beam<'a>( + full: &'a Full, + query: &'a [Self], + ) -> ANNResult>; + + #[doc(hidden)] + fn make_prune<'a>(full: &'a Full) -> ANNResult>; +} + +/// Full-precision data representation. +#[derive(Debug)] +pub struct Full +where + T: 'static, +{ + store: Store, + metric: Metric, + lookahead: Option, + _type: PhantomData, +} + +impl Full +where + T: 'static, +{ + /// Initialize a [`Config`] for this representation. + /// + /// See also: [`Config::new`]. + /// + /// # Errors + /// + /// Returns the errors described by [`Config::new`]. + pub fn config( + capacity: Capacity, + max_degree: MaxDegree, + metric: Metric, + start_points: Matrix, + ) -> Result, ConfigError> { + Config::new(capacity, max_degree, metric, start_points) + } + + /// Create a new full-precision representation from `config`. + /// + /// See: [`Config::build`]. + fn new(config: Config) -> ANNResult + where + T: FullPrecision, + { + let Config { + layout, + metric, + start_points, + store, + lookahead, + } = config; + + let bytes = Bytes::new(start_points.ncols() * std::mem::size_of::()); + let invasive = Invasive::config(bytes); + let store = Store::new(layout, store, invasive)?; + + // Initialize start points. + for (i, row) in std::iter::zip(store.frozen(), start_points.row_iter()) { + #[expect( + clippy::expect_used, + reason = "failing this is an internal, unrecoverable bug" + )] + let mut slot = store + .slot(i) + .expect("internal store should leave frozen-points available for writing"); + slot.data() + .as_mut_slice() + .copy_from_slice(bytemuck::must_cast_slice::(row)); + + slot.freeze(); + } + + Ok(Self { + store, + metric, + lookahead, + _type: PhantomData, + }) + } + + /// Return the logical dimension of the data handled by this [`repr::Representation`]. + pub fn dim(&self) -> usize { + self.bytes().value() / std::mem::size_of::() + } + + /// Return the number of payload bytes in each stored vector. + pub fn bytes(&self) -> Bytes { + self.store.slots().bytes() + } + + #[cfg(test)] + fn bytes_plus_tag(&self) -> Bytes { + self.store.slots().bytes_plus_tag() + } + + /// Return the [`Metric`] for this representation. + pub fn metric(&self) -> Metric { + self.metric + } + + fn check_dim(&self, dim: usize) -> Result<(), ExpandBeamError> { + if self.dim() != dim { + Err(ExpandBeamError { + expected: self.dim(), + xlen: dim, + }) + } else { + Ok(()) + } + } + + fn reader(&self) -> Result, epoch::Unavailable> { + Invasive::reader(&self.store) + } +} + +impl Full +where + T: FullPrecision, +{ + pub(crate) fn get(&self, i: u32) -> ANNResult> { + let reader = self.reader()?; + let data = match reader.read(i.into_usize()) { + Some(data) => data, + None => { + return Err(ANNError::message("item could not be read")); + } + }; + + let mut buf: Box<[_]> = std::iter::repeat_n(T::zeroed(), self.dim()).collect(); + bytemuck::must_cast_slice_mut::(&mut buf).copy_from_slice(data); + Ok(buf) + } +} + +impl repr::Representation for Full +where + T: FullPrecision, +{ + fn max_degree(&self) -> MaxDegree { + self.store.neighbors().max_degree() + } + + fn retire(&self, i: u32) -> ANNResult<()> { + Ok(self.store.retire(i.into_usize())?) + } + + fn is_readable(&self, i: u32) -> Option { + self.store.can_read_approximate(i.into_usize()) + } + + fn id_limit(&self) -> IdLimit { + self.store.id_limit() + } + + fn capacity(&self) -> Capacity { + self.store.capacity() + } +} + +impl repr::Set<&[T]> for Full +where + T: FullPrecision, +{ + type Guard<'a> = Guard<'a>; + + fn set(&self, v: &[T]) -> ANNResult> { + if v.len() != self.dim() { + return Err(ANNError::from(SetError { + got: v.len(), + expected: self.dim(), + })); + } + + let mut slot = self + .store + .acquire() + .ok_or_else(|| ANNError::message("could not allocate a new slot"))?; + + slot.data() + .as_mut_slice() + .copy_from_slice(bytemuck::must_cast_slice::(v)); + + Ok(Guard::new(slot)) + } +} + +/// A [`repr::Guard`] for [`Full`]. +#[derive(Debug)] +pub struct Guard<'a> { + slot: store::Slot<'a, invasive::Slot<'a>>, +} + +impl<'a> Guard<'a> { + fn new(slot: store::Slot<'a, invasive::Slot<'a>>) -> Self { + Self { slot } + } +} + +impl repr::Guard for Guard<'_> { + fn publish(self) { + self.slot.publish(); + } + fn id(&self) -> u32 { + self.slot.slot() + } +} + +#[derive(Debug, Error)] +#[error( + "data of dimension {} does not match full precision representation's dimension {}", + self.got, + self.expected +)] +struct SetError { + got: usize, + expected: usize, +} + +diskann::convert_error!(SetError); + +impl repr::Search for Full +where + T: FullPrecision, +{ + type Query<'a> = &'a [T]; + + fn search_accessor<'a>( + &'a self, + query: Self::Query<'a>, + provider: &'a (dyn std::any::Any + Send + Sync), + counters: LocalCounters<'a>, + ) -> ANNResult> { + T::__search_accessor(self, query, provider, counters) + } +} + +impl repr::Insert for Full +where + T: FullPrecision, +{ + fn prune_accessor<'a>( + &'a self, + counters: LocalCounters<'a>, + ) -> ANNResult> { + T::__prune_accessor(self, counters) + } +} + +//----------------------// +// Expand Beam (Search) // +//----------------------// + +// A baby [`std::borrow::Cow`]. +#[derive(Debug)] +enum Calf<'a, T> { + Borrowed(&'a [T]), + Owned(Box<[T]>), +} + +impl std::ops::Deref for Calf<'_, T> { + type Target = [T]; + fn deref(&self) -> &Self::Target { + match self { + Self::Borrowed(slice) => slice, + Self::Owned(boxed) => boxed, + } + } +} + +/// A temporary precursor for [`ExpandBeam`] to simplify macros. +#[derive(Debug)] +struct IntoExpandBeam<'a, T, U> { + query: Calf<'a, T>, + reader: store::invasive::Reader<'a>, + lookahead: Option, + _data: PhantomData, +} + +impl<'a, T, U> IntoExpandBeam<'a, T, U> { + /// Construct a new [`IntoExpandBeam`], validating the query dimension and acquiring a + /// reader for `full`. + fn new(full: &'a Full, query: Calf<'a, T>) -> ANNResult { + full.check_dim(query.len())?; + let reader = full.reader()?; + let lookahead = full.lookahead; + Ok(Self { + query, + reader, + lookahead, + _data: PhantomData, + }) + } +} + +trait Distance: std::fmt::Debug + Send + Sync + 'static { + fn eval(&self, x: UnalignedSlice<'_, T>, y: UnalignedSlice<'_, U>) -> f32; +} + +#[derive(Debug)] +struct Pure(PhantomData); + +impl Pure { + const fn new() -> Self { + Self(PhantomData) + } +} + +impl Distance for Pure +where + D: for<'any> FTarget2, UnalignedSlice<'any, U>> + + std::fmt::Debug + + Send + + Sync + + 'static, +{ + #[inline(always)] + fn eval(&self, x: UnalignedSlice<'_, T>, y: UnalignedSlice<'_, U>) -> f32 { + D::run(ARCH, x, y) + } +} + +impl Distance for diskann_vector::distance::Distance +where + T: std::fmt::Debug + 'static, + U: std::fmt::Debug + 'static, +{ + #[inline(always)] + fn eval(&self, x: UnalignedSlice<'_, T>, y: UnalignedSlice<'_, U>) -> f32 { + self.call_unaligned(x, y) + } +} + +/// A fused query distance based on [`diskann_vector::PureDistanceFunction`] to enable +/// inlining of the final distance function (`D`). +/// +/// The type of the embedded query (`T`) is distinct from the expected data-set (`U`) to +/// allow `f16` queries to be pre-converted to `f32`, saving on-the-fly conversion that +/// would otherwise be needed. +#[derive(Debug)] +struct ExpandBeam<'a, P, T, U, D> { + // The original query. + query: Calf<'a, T>, + // A reader into a representation's store. + reader: store::invasive::Reader<'a>, + // The prefetch lookahead. + lookahead: Option, + // The type of the data prefetcher. + prefetch: prefetch::Checked

, + // The type of the distance used for the arguments + distance: D, + // The type of the data in the original dataset. + _data: PhantomData, +} + +impl<'a, P, T, U, D> ExpandBeam<'a, P, T, U, D> { + fn new(into: IntoExpandBeam<'a, T, U>, prefetch: P, distance: D) -> Self + where + P: Prefetch, + { + let IntoExpandBeam { + query, + reader, + lookahead, + _data, + } = into; + + // TAG: PREFETCH-CHECK + #[expect( + clippy::expect_used, + reason = "internal APIs should only provide valid prefetchers" + )] + let prefetch = prefetch::Checked::new(prefetch, reader.bytes_plus_tag()) + .expect("internal APIs should only provide valid prefetchers"); + + Self { + query, + reader, + lookahead, + prefetch, + distance, + _data, + } + } + + fn bytes(&self) -> usize { + std::mem::size_of::() * self.query.len() + } + + fn boxed(self) -> Box { + Box::new(self) + } + + /// Compute the distance between the embedded query and `x`. + /// + /// # Safety + /// + /// `x.len()` must be exactly `self.bytes()` bytes long and contain + /// `self.query.len()` valid values of `U`. + #[inline(always)] + unsafe fn run_unchecked(&self, x: &[u8]) -> f32 + where + D: Distance, + { + debug_assert_eq!(x.len(), self.bytes()); + + // SAFETY: We've validated that `x` has the correct length. + let x = unsafe { UnalignedSlice::new(x.as_ptr().cast::(), self.query.len()) }; + self.distance.eval((*self.query).into(), x) + } +} + +// SAFETY: Our implementation of `repr::ExpandBeam::id_limit` is consistent with our +// `repr::ExpandBeam::expand_beam` implementation. They are both dependent on +// `invasive::Reader`'s internal bounds. +unsafe impl repr::ExpandBeam for ExpandBeam<'_, P, T, U, D> +where + P: Prefetch, + T: Send + Sync + 'static + Debug, + U: Send + Sync + 'static + Debug, + D: Distance, +{ + fn evaluate(&self, i: u32) -> ANNResult> { + if !self.reader.is_in_bounds(i.into_usize()) { + Err(ANNError::new(OutOfBounds(i))) + } else { + // SAFETY: We have checked that `i` is in-bounds. + match unsafe { self.reader.read_in_bounds(i.into_usize()) } { + Some(data) => { + // SAFETY: Since we just read `data` from `self.reader`, we know it's + // exactly `self.bytes()` long. + let distance = unsafe { self.run_unchecked(data) }; + Ok(Some(distance)) + } + None => Ok(None), + } + } + } + + fn id_limit(&self) -> IdLimit { + self.reader.id_limit() + } + + unsafe fn expand_beam(&self, list: &[u32], buffer: &mut [(u32, f32)]) -> ANNResult { + debug_assert!(buffer.len() >= list.len()); + + let len = list.len(); + let lookahead = self.lookahead.map(|l| l.get()).unwrap_or(0).min(len); + + for j in list.iter().take(lookahead) { + // SAFETY: The in-bounds constraint is assured by the caller, both for `j` as well + // as the validity of the prefetch bounds. + // + // We validated `self.prefetch` with `self.reader.bytes_with_tag()` upon construction. + // + // We do not materialize the `RawSlice` as a reference. + unsafe { + let raw = self.reader.read_raw_unchecked(j.into_usize()); + self.prefetch.prefetch(raw.as_ptr(), raw.len()); + } + } + + // Disable prefetching if the lookahead is 0. + let mut j = if lookahead == 0 { len } else { lookahead }; + let mut processed = 0; + for &i in list.iter() { + if j != len { + // SAFETY: The in-bounds constraint is assured by the caller, both for `j` as + // well as the validity of the prefetch bounds. + // + // We validated `self.prefetch` with `self.reader.bytes_with_tag()` upon + // construction. + // + // We do not materialize the `RawSlice` as a reference. + unsafe { + let raw = self + .reader + .read_raw_unchecked(list.get_unchecked(j).into_usize()); + self.prefetch.prefetch(raw.as_ptr(), raw.len()); + } + j += 1; + } + + // SAFETY: Caller asserts that `i` is in-bounds. + if let Some(data) = unsafe { self.reader.read_in_bounds(i.into_usize()) } { + // SAFETY: We just read `data` from `self.reader`, so it has a length of + // exactly `self.bytes()`. + let distance = unsafe { self.run_unchecked(data) }; + + // SAFETY: Inherited from caller. + *unsafe { buffer.get_unchecked_mut(processed) } = (i, distance); + processed += 1; + } + } + + Ok(processed) + } +} + +#[derive(Debug, Error)] +#[error("index {} is out-of-bounds", self.0)] +struct OutOfBounds(u32); + +diskann::convert_error!(OutOfBounds); + +#[derive(Debug, Error)] +#[error( + "expected slice of length {} - instead got {}", + self.expected, + self.xlen, +)] +struct ExpandBeamError { + expected: usize, + xlen: usize, +} + +diskann::convert_error!(ExpandBeamError); + +//-------// +// Prune // +//-------// + +#[derive(Debug)] +struct Prune<'a, T, D> { + // Buffered data to prune over. + buffer: Vec>, + // A reader into a representation's store. + reader: store::invasive::Reader<'a>, + // The distance implementation used for pruning. + distance: D, +} + +impl<'a, T, D> Prune<'a, T, D> { + fn new(reader: store::invasive::Reader<'a>, distance: D) -> Self { + // This should be ensured at construction time + debug_assert!( + reader + .bytes() + .value() + .is_multiple_of(std::mem::size_of::()), + "internal invariant violated", + ); + + Self { + buffer: Vec::new(), + reader, + distance, + } + } + + fn boxed(self) -> Box { + Box::new(self) + } +} + +impl repr::Prune for Prune<'_, T, D> +where + T: Debug + Send + Sync + 'static, + D: Distance, +{ + fn prepare( + &mut self, + items: hashbrown::hash_map::IterMut<'_, u32, Option>, + ) -> ANNResult { + let mut counter = repr::PruneKey::counter(); + self.buffer.clear(); + self.buffer.reserve(items.len()); + + for (id, key) in items { + if let Some(v) = self.reader.read(id.into_usize()) { + // SAFETY: We have checked that it is safe to read this data vector and + // `self.reader` is preventing any mutation for `self`'s lifetime. + // + // Further, we know the raw slice has a length exactly `self.reader.bytes()`, + // so the formed `UnalignedSlice` is within a single allocated object. + let unaligned = unsafe { + UnalignedSlice::new( + v.as_ptr().cast::(), + self.reader.bytes().value() / std::mem::size_of::(), + ) + }; + + self.buffer.push(unaligned); + + *key = Some(counter); + + // Potential overflow issue - but it's exceedingly unlikely that + // someone will provide a prune list exceeding `u16::MAX`. + // + // In addition, `diskann` limits this bound as well. + counter = counter.increment()?; + } + } + + Ok(counter.index()) + } + + fn evaluate(&self, a: repr::PruneKey, b: repr::PruneKey) -> f32 { + self.distance + .eval(self.buffer[a.index()], self.buffer[b.index()]) + } +} + +///////////////// +// Dispatching // +///////////////// + +const fn compute_bytes(dim: usize) -> usize { + dim * std::mem::size_of::() + (AtomicTag::SIZE).value() +} + +macro_rules! expand_beam { + ($into:ident, { $T:ty, $N:literal, $f:ident }) => {{ + Box::new(ExpandBeam::<_, _, $T, _>::new( + $into, + prefetch::Unrolled::<{ compute_bytes::<$T>($N) }>::new(), + Pure::>::new(), + )) + }}; + ($into:ident, $f:ident) => {{ + Box::new(ExpandBeam::new( + $into, + prefetch::Loop::new(), + Pure::<$f>::new(), + )) + }}; +} + +macro_rules! prune { + ($self:ty, $reader:ident, $f:ident) => {{ Prune::<$self, _>::new($reader, Pure::<$f>::new()).boxed() }}; + ($self:ty, $reader:ident, { $N:literal, $f:ident }) => {{ Prune::<$self, _>::new($reader, Pure::>::new()).boxed() }}; +} + +impl FullPrecisionImpl for f32 { + fn make_expand_beam<'a>( + full: &'a Full, + query: &'a [f32], + ) -> ANNResult> { + let into = IntoExpandBeam::new(full, Calf::Borrowed(query))?; + + let output: Box = match full.metric { + Metric::L2 => { + if full.dim() == 100 { + expand_beam!(into, { f32, 100, SquaredL2 }) + } else { + expand_beam!(into, SquaredL2) + } + } + Metric::InnerProduct => expand_beam!(into, InnerProduct), + Metric::Cosine => expand_beam!(into, Cosine), + Metric::CosineNormalized => expand_beam!(into, CosineNormalized), + }; + + Ok(output) + } + + fn make_prune<'a>(full: &'a Full) -> ANNResult> { + let reader = full.reader()?; + + let output: Box = match full.metric { + Metric::L2 => prune!(Self, reader, SquaredL2), + Metric::InnerProduct => prune!(Self, reader, InnerProduct), + Metric::Cosine => prune!(Self, reader, Cosine), + Metric::CosineNormalized => prune!(Self, reader, CosineNormalized), + }; + + Ok(output) + } +} + +impl FullPrecisionImpl for f16 { + fn make_expand_beam<'a>( + full: &'a Full, + query: &'a [f16], + ) -> ANNResult> { + let mut as_f32: Box<[f32]> = std::iter::repeat_n(0.0, full.dim()).collect(); + diskann_wide::arch::dispatch2(SliceCast::new(), &mut *as_f32, query); + let query = Calf::Owned(as_f32); + + let into = IntoExpandBeam::new(full, query)?; + + let output: Box = match full.metric { + Metric::L2 => { + if full.dim() == 100 { + expand_beam!(into, { f16, 100, SquaredL2 }) + } else { + expand_beam!(into, SquaredL2) + } + } + Metric::InnerProduct => expand_beam!(into, InnerProduct), + Metric::Cosine => expand_beam!(into, Cosine), + Metric::CosineNormalized => expand_beam!(into, CosineNormalized), + }; + + Ok(output) + } + + fn make_prune<'a>(full: &'a Full) -> ANNResult> { + let reader = full.reader()?; + + let output: Box = match full.metric { + Metric::L2 => prune!(Self, reader, SquaredL2), + Metric::InnerProduct => prune!(Self, reader, InnerProduct), + Metric::Cosine => prune!(Self, reader, Cosine), + Metric::CosineNormalized => prune!(Self, reader, CosineNormalized), + }; + + Ok(output) + } +} + +impl FullPrecisionImpl for u8 { + fn make_expand_beam<'a>( + full: &'a Full, + query: &'a [u8], + ) -> ANNResult> { + let into = IntoExpandBeam::new(full, Calf::Borrowed(query))?; + + let output: Box = match full.metric { + Metric::L2 => { + if full.dim() == 128 { + expand_beam!(into, { u8, 128, SquaredL2 }) + } else { + expand_beam!(into, SquaredL2) + } + } + Metric::InnerProduct => expand_beam!(into, InnerProduct), + Metric::Cosine | Metric::CosineNormalized => expand_beam!(into, Cosine), + }; + + Ok(output) + } + + fn make_prune<'a>(full: &'a Full) -> ANNResult> { + let reader = full.reader()?; + + let output: Box = match full.metric { + Metric::L2 => prune!(Self, reader, SquaredL2), + Metric::InnerProduct => prune!(Self, reader, InnerProduct), + Metric::Cosine => prune!(Self, reader, Cosine), + Metric::CosineNormalized => prune!(Self, reader, CosineNormalized), + }; + + Ok(output) + } +} + +impl FullPrecisionImpl for i8 { + fn make_expand_beam<'a>( + full: &'a Full, + query: &'a [i8], + ) -> ANNResult> { + let into = IntoExpandBeam::new(full, Calf::Borrowed(query))?; + + let distance = + >::distance_comparer(full.metric(), Some(full.dim())); + + let output: Box = + ExpandBeam::new(into, prefetch::Loop::new(), distance).boxed(); + + Ok(output) + } + + fn make_prune<'a>(full: &'a Full) -> ANNResult> { + let reader = full.reader()?; + + let distance = + >::distance_comparer(full.metric(), Some(full.dim())); + + let output: Box = Prune::::new(reader, distance).boxed(); + Ok(output) + } +} + +/// We use a macro to stamp out implementations of [`FullPrecision`] instead of using a +/// blanket implementation from [`FullPrecisionImpl`] to make implementations more +/// discoverable through the generated rust-doc. +macro_rules! impl_full_precision { + ($T:ty) => { + impl FullPrecision for $T { + fn __search_accessor<'a>( + representation: &'a Full, + query: &'a [Self], + provider: &'a (dyn std::any::Any + Send + Sync), + counters: LocalCounters<'a>, + ) -> ANNResult> { + let expand_beam = <$T>::make_expand_beam(representation, query)?; + Ok(crate::provider::SearchAccessor::new( + representation.store.neighbors(), + expand_beam, + provider, + representation.store.frozen(), + counters, + )) + } + + fn __prune_accessor<'a>( + representation: &'a Full, + counters: LocalCounters<'a>, + ) -> ANNResult> { + let prune = <$T>::make_prune(representation)?; + Ok(crate::provider::PruneAccessor::new( + prune, + representation.store.neighbors(), + counters, + )) + } + } + }; + ($($Ts:ty),* $(,)?) => { + $(impl_full_precision!($Ts);)* + } +} + +impl_full_precision!(f32, f16, u8, i8); + +/////////// +// Tests // +/////////// + +#[cfg(test)] +mod tests { + use super::*; + + use std::fmt::Display; + + use diskann_utils::lazy_format; + use hashbrown::{HashMap, HashSet}; + use rand::{Rng, SeedableRng, rngs::StdRng}; + + /// Generate random elements of a representation's data type from a seeded RNG. + trait Sample: bytemuck::Pod { + fn sample(rng: &mut R) -> Self; + } + + impl Sample for f32 { + fn sample(rng: &mut R) -> Self { + rng.random_range(-1.0f32..1.0f32) + } + } + + impl Sample for f16 { + fn sample(rng: &mut R) -> Self { + diskann_wide::cast_f32_to_f16(rng.random_range(-1.0f32..1.0f32)) + } + } + + impl Sample for u8 { + fn sample(rng: &mut R) -> Self { + rng.random() + } + } + + impl Sample for i8 { + fn sample(rng: &mut R) -> Self { + rng.random() + } + } + + fn gen_vec(dim: usize, rng: &mut impl Rng) -> Vec { + (0..dim).map(|_| T::sample(rng)).collect() + } + + /// Compare two distances allowing for floating-point reassociation between the + /// specialized / converted kernels and the dynamic reference. + #[must_use] + fn approx_eq(got: f32, want: f32) -> bool { + (got - want).abs() <= 1e-3 + 1e-4 * want.abs() + } + + /// A simple test `Full` containing 1-dimensional `f32` values. + /// + /// This is used in dedicated `ExpandBeam` and `Prune` tests in a miri-friendly way. + /// + /// Two start points are included, initialized to `capacity` and `capacity + 1`. + fn test_full(capacity: Capacity) -> (Full, HashMap) { + let start_points = [capacity.value() as f32, (capacity.value() + 1) as f32]; + + let full = <_ as repr::RepresentationConfig>::build( + Full::::config( + capacity, + MaxDegree::new(0), + Metric::L2, + Matrix::column_vector(Box::new(start_points)), + ) + .unwrap(), + ) + .unwrap(); + + assert_eq!(full.dim(), 1, "start points only have one dimension"); + assert_eq!(full.bytes(), Bytes::size_of::()); + assert_eq!( + full.bytes_plus_tag(), + Bytes::size_of::() + .checked_add(Bytes::size_of::()) + .unwrap() + ); + assert_eq!(full.metric(), Metric::L2); + assert_eq!( + <_ as repr::Representation>::id_limit(&full), + IdLimit::new(capacity.value() as u32 + 2) + ); + assert_eq!(<_ as repr::Representation>::capacity(&full), capacity); + + let points: HashMap = { + let reader = full.reader().unwrap(); + assert_eq!( + reader.read(capacity.value()).unwrap(), + bytemuck::bytes_of(&start_points[0]) + ); + assert_eq!( + reader.read(capacity.value() + 1).unwrap(), + bytemuck::bytes_of(&start_points[1]) + ); + + [ + (capacity.value() as u32, start_points[0]), + ((capacity.value() + 1) as u32, start_points[1]), + ] + .into_iter() + .collect() + }; + + (full, points) + } + + #[derive(Debug)] + struct TestDistance; + + impl Distance for TestDistance { + fn eval(&self, x: UnalignedSlice<'_, f32>, y: UnalignedSlice<'_, f32>) -> f32 { + assert_eq!(x.len(), 1); + assert_eq!(y.len(), 1); + + // SAFETY: `UnalignedSlice`s must point to valid data, and we've checked that + // the length of each slice is exactly 1. Therefore, the pointer read is safe. + unsafe { x.as_ptr().read_unaligned() + y.as_ptr().read_unaligned() } + } + } + + /// A Miri-friendly test for [`ExpandBeam`]. + /// + /// This test covers the following: + /// + /// 1. Prefetches are in-bounds for all lookaheads. + /// 2. [`ExpandBeam`] doesn't lie about its [`IdLimit`]. + /// 3. [`ExpandBeam`]'s methods are internally consistent with each other and + /// consistent with the parent [`Full`] for item readability. + /// 4. [`ExpandBeam::expand_beam`] preserves input order and visits every item in the + /// input list. + #[test] + fn test_expand_beam() { + let capacity = Capacity::new(20); + let id_limit = IdLimit::new(22); + + let (mut full, mut points) = test_full(capacity); + + assert_eq!(<_ as repr::Representation>::capacity(&full), capacity); + assert_eq!(<_ as repr::Representation>::id_limit(&full), id_limit); + + let mut available: HashSet = (0..capacity.value()).map(|i| i as u32).collect(); + + // Insert the values 0 to 10. + for i in 0u32..10 { + let guard = <_ as repr::Set<&[f32]>>::set(&full, &[i as f32]).unwrap(); + + let id = <_ as repr::Guard>::id(&guard); + + assert!( + available.remove(&id), + "insertion should return available slots", + ); + + assert!( + points.insert(id, i as f32).is_none(), + "insertion should not repeat", + ); + + <_ as repr::Guard>::publish(guard); + } + + // Lookaheads to try. + let lookaheads: &[Option] = &[ + None, + NonZeroUsize::new(1), + NonZeroUsize::new(2), + NonZeroUsize::new(5), + NonZeroUsize::new(10), + NonZeroUsize::new(100), + ]; + + // This is the main loop for testing `ExpandBeam`. + // + // We do several things. + // + // 1. We insert two additional IDs but hold their guards without publishing. + // This tests that items remain unreadable until they are published. + // + // 2. We publish two new points and immediately retire them. + // This tests that we correctly make these points unreadable. + for lookahead in lookaheads { + full.lookahead = *lookahead; + + let g0 = <_ as repr::Set<&[f32]>>::set(&full, &[1000.0]).unwrap(); + let g1 = <_ as repr::Set<&[f32]>>::set(&full, &[2000.0]).unwrap(); + let g2 = <_ as repr::Set<&[f32]>>::set(&full, &[3000.0]).unwrap(); + let g3 = <_ as repr::Set<&[f32]>>::set(&full, &[4000.0]).unwrap(); + + { + let g0_id = <_ as repr::Guard>::id(&g0); + <_ as repr::Guard>::publish(g0); + <_ as repr::Representation>::retire(&full, g0_id).unwrap(); + } + + { + let g1_id = <_ as repr::Guard>::id(&g1); + <_ as repr::Guard>::publish(g1); + <_ as repr::Representation>::retire(&full, g1_id).unwrap(); + } + + let query = -1.0f32; + + let into = + IntoExpandBeam::new(&full, Calf::Borrowed(std::slice::from_ref(&query))).unwrap(); + + let expand = ExpandBeam::new(into, prefetch::Loop::new(), TestDistance); + + assert_eq!(<_ as repr::ExpandBeam>::id_limit(&expand), id_limit); + + let mut buf = Vec::<(u32, f32)>::new(); + let mut list = Vec::::new(); + + // Use triangular indexing from `0..id_limit` with `points` serving as the + // groundtruth. + // + // Note that we purposely make `list` extra long with redundant indices to help + // catch indexing bugs inside `ExpandBeam`. + for i in 0..=id_limit.value() { + list.clear(); + list.extend((0..i).rev()); + list.extend(0..i); + + buf.resize(list.len(), Default::default()); + + // SAFETY: By construction, all entries in `list` are within `id_limit` + // (verified against this `ExpandBeam` instance. + // + // Also by construction `buf` is at least as long as `list`. + let read = + unsafe { <_ as repr::ExpandBeam>::expand_beam(&expand, &list, &mut buf) } + .unwrap(); + + let expected: Vec<(u32, f32)> = list + .iter() + .copied() + .filter_map(|id| match points.get(&id) { + Some(point) => { + let expected = point + query; + + assert!( + <_ as repr::Representation>::is_readable(&full, id).unwrap(), + "point should be readable" + ); + + assert_eq!( + <_ as repr::ExpandBeam>::evaluate(&expand, id).unwrap(), + Some(expected), + "readable points should return valid distances", + ); + + Some((id, expected)) + } + None => { + assert!( + !<_ as repr::Representation>::is_readable(&full, id).unwrap(), + "points not yielded by ExpandBeam should be unreadable" + ); + + assert!( + <_ as repr::ExpandBeam>::evaluate(&expand, id) + .unwrap() + .is_none(), + "unreadable points should return `None` for their distance", + ); + + None + } + }) + .collect(); + + assert_eq!(&buf[..read], &*expected); + } + + assert!( + <_ as repr::ExpandBeam>::evaluate(&expand, id_limit.value()).is_err(), + "`ExpandBeam::evaluate` should catch out-of-bounds errors", + ); + + // Ensure we hold onto `g2` and `g3` for the duration of the above check. + drop(g2); + drop(g3); + } + } + + fn test_prune_inner( + points: &HashMap, + prune: &mut Prune, + ids: &[u32], + ) { + let mut items: HashMap> = + ids.iter().map(|id| (*id, None)).collect(); + + let processed = <_ as repr::Prune>::prepare(prune, items.iter_mut()).unwrap(); + assert_eq!(processed, items.values().filter(|i| i.is_some()).count()); + + // Ensure that `prepare` agrees with `points`. + for (k, v) in items.iter() { + match v { + Some(_) => assert!(points.contains_key(k)), + None => assert!(!points.contains_key(k)), + } + } + + fn filter((k, v): (&u32, &Option)) -> Option<(u32, repr::PruneKey)> { + v.map(|v| (*k, v)) + } + + // Ensure that distances agree. + for (k0, v0) in items.iter().filter_map(filter) { + for (k1, v1) in items.iter().filter_map(filter) { + // Manually implement `TestDistance`. + let expected = points[&k0] + points[&k1]; + let got = <_ as repr::Prune>::evaluate(prune, v0, v1); + assert_eq!(expected, got); + } + } + } + + /// A Miri-friendly test for `Prune`. + #[test] + fn test_prune() { + let capacity = Capacity::new(20); + let id_limit = IdLimit::new(22); + + let (full, mut points) = test_full(capacity); + + assert_eq!(<_ as repr::Representation>::capacity(&full), capacity); + assert_eq!(<_ as repr::Representation>::id_limit(&full), id_limit); + + let mut available: HashSet = (0..capacity.value()).map(|i| i as u32).collect(); + + // Insert the values 0 to 10. + for i in 0u32..10 { + let guard = <_ as repr::Set<&[f32]>>::set(&full, &[i as f32]).unwrap(); + + let id = <_ as repr::Guard>::id(&guard); + + assert!( + available.remove(&id), + "insertion should return available slots", + ); + + assert!( + points.insert(id, i as f32).is_none(), + "insertion should not repeat", + ); + + <_ as repr::Guard>::publish(guard); + } + + // We do several things. + // + // 1. We insert two additional IDs but hold their guards without publishing. + // This tests that items remain unreadable until they are published. + // + // 2. We publish two new points and immediately retire them. + // This tests that we correctly make these points unreadable. + let g0 = <_ as repr::Set<&[f32]>>::set(&full, &[1000.0]).unwrap(); + let g1 = <_ as repr::Set<&[f32]>>::set(&full, &[2000.0]).unwrap(); + let g2 = <_ as repr::Set<&[f32]>>::set(&full, &[3000.0]).unwrap(); + let g3 = <_ as repr::Set<&[f32]>>::set(&full, &[4000.0]).unwrap(); + + { + let g0_id = <_ as repr::Guard>::id(&g0); + <_ as repr::Guard>::publish(g0); + <_ as repr::Representation>::retire(&full, g0_id).unwrap(); + } + + { + let g1_id = <_ as repr::Guard>::id(&g1); + <_ as repr::Guard>::publish(g1); + <_ as repr::Representation>::retire(&full, g1_id).unwrap(); + } + + let mut prune = Prune::new(full.reader().unwrap(), TestDistance); + + // Note that we emit reads above the `IdLimit`, which we expect to be silently + // rejected. + for i in 0..=(id_limit.value() + 5) { + let mut ids: Vec = (0..i).collect(); + test_prune_inner(&points, &mut prune, &ids); + + ids.reverse(); + test_prune_inner(&points, &mut prune, &ids); + } + + // Drop the guards - verifying that they are held in-limbo during the test. + drop(g2); + drop(g3); + } + + //----------------------// + // Specialization Tests // + //----------------------// + + // These test make sure that the mapping for metrics and specializations are routed + // correctly. They do not exhaustively test the `ExpandBeam` kernls as these are left + // to tests that are more Miri friendly. + fn test_dispatch(dim: usize, metric: Metric, seed: u64, ctx: &dyn Display) + where + T: FullPrecision + FullPrecisionImpl + Sample + DistanceProvider, + { + let mut rng = StdRng::seed_from_u64(seed); + + let start_point = gen_vec::(dim, &mut rng); + let query = gen_vec::(dim, &mut rng); + + let full = <_ as repr::RepresentationConfig>::build( + Full::::config( + Capacity::new(1), + MaxDegree::new(0), + metric, + Matrix::::row_vector(start_point.clone().into()), + ) + .unwrap(), + ) + .unwrap(); + + let start_id: u32 = 1; + + let internal_query = { + let guard = <_ as repr::Set<&[T]>>::set(&full, &query).unwrap(); + let id = <_ as repr::Guard>::id(&guard); + <_ as repr::Guard>::publish(guard); + id + }; + + let distance = >::distance_comparer(metric, None); + let expected = distance.call(&start_point, &query); + + // Expand Beam - both `evaluate` and `expand_beam` share the same distance computer, + // so we can just test `evaluate`. + { + let expand_beam = ::make_expand_beam(&full, &query).unwrap(); + let got = expand_beam.evaluate(start_id).unwrap().unwrap(); + assert!( + approx_eq(expected, got), + "{ctx} - expected {expected}, got {got}" + ); + } + + // Prune + { + let mut prune = ::make_prune(&full).unwrap(); + let mut points: HashMap> = + [(internal_query, None), (start_id, None)] + .into_iter() + .collect(); + prune.prepare(points.iter_mut()).unwrap(); + let got = prune.evaluate(points[&internal_query].unwrap(), points[&start_id].unwrap()); + assert!( + approx_eq(expected, got), + "{ctx} - expected {expected}, got {got}" + ); + } + } + + fn metrics() -> [Metric; 4] { + [ + Metric::L2, + Metric::InnerProduct, + Metric::Cosine, + Metric::CosineNormalized, + ] + } + + #[test] + fn test_f32_dynamic() { + let dim = 10; + for m in metrics() { + test_dispatch::(dim, m, 0x917a80fc68f66e04, &lazy_format!("dynamic-{m}-f32")); + } + } + + // Test the specialized dispatches. + #[test] + fn test_f32_specialized() { + test_dispatch::( + 100, + Metric::L2, + 0x917a80fc68f66e04, + &lazy_format!("dynamic-l2-f32-100"), + ); + } + + #[test] + fn test_f16_dynamic() { + let dim = 10; + for m in metrics() { + test_dispatch::(dim, m, 0x917a80fc68f66e04, &lazy_format!("dynamic-{m}-f16")); + } + } + + // Test the specialized dispatches. + #[test] + fn test_f16_specialized() { + test_dispatch::( + 100, + Metric::L2, + 0x917a80fc68f66e04, + &lazy_format!("dynamic-l2-f16-100"), + ); + } + + #[test] + fn test_u8_dynamic() { + let dim = 10; + for m in [Metric::L2, Metric::InnerProduct, Metric::Cosine] { + test_dispatch::(dim, m, 0x917a80fc68f66e04, &lazy_format!("dynamic-{m}-u8")); + } + } + + #[test] + fn test_u8_specialized() { + test_dispatch::( + 128, + Metric::L2, + 0x917a80fc68f66e04, + &lazy_format!("dynamic-l2-u8-100"), + ); + } + + #[test] + fn test_i8_dynamic() { + let dim = 10; + for m in [Metric::L2, Metric::InnerProduct, Metric::Cosine] { + test_dispatch::(dim, m, 0x917a80fc68f66e04, &lazy_format!("dynamic-{m}-i8")); + } + } +} diff --git a/diskann-inmem/src/repr/mod.rs b/diskann-inmem/src/repr/mod.rs new file mode 100644 index 0000000000..c06cfe51d9 --- /dev/null +++ b/diskann-inmem/src/repr/mod.rs @@ -0,0 +1,243 @@ +/* + * Copyright (c) Microsoft Corporation. + * Licensed under the MIT license. + */ + +//! # Data Representation +//! +//! A simplified interface for [`crate::Provider`] to use for building a graph index. + +use std::num::NonZeroU16; + +use diskann::ANNResult; +use thiserror::Error; + +use crate::{ + counters::LocalCounters, + num::{Capacity, IdLimit, MaxDegree}, +}; + +pub mod full; +pub use full::{Full, FullPrecision}; + +/// Deferred creation of [`Representation`]s. +/// +/// This is used in APIs like [`crate::Provider::new`] to defer allocation of large +/// in-memory data structures. +pub trait RepresentationConfig { + /// The type of the resulting [`Representation`]. + type Representation: Representation; + + /// Build the target [`Representation`]. + fn build(self) -> ANNResult; +} + +/// Configurable data representation for [`crate::Provider`]. +/// +/// A representation owns the adjacency lists and vector data for a concurrent in-memory graph +/// index. Representations are expected to be indexed using `u32` IDs from +/// `0..self.id_limit()`, with internal IDs in `0..self.capacity()` available for writing. +/// +/// See also: +/// +/// - [`Set`]: For assigning into the store. +/// - [`Search`]: Search compatibility with [`crate::Provider`]. +/// - [`Insert`]: Insert compatibility with [`crate::Provider`]. +pub trait Representation: Send + Sync + 'static { + /// Return the [`MaxDegree`] of the internal graph. + fn max_degree(&self) -> MaxDegree; + + /// Return the [`IdLimit`] for the data store. + fn id_limit(&self) -> IdLimit; + + /// Return the functional [`Capacity`] for the data store. + fn capacity(&self) -> Capacity; + + /// Retire the internal ID `i`. Such IDs will eventually be recycled for reuse. + fn retire(&self, i: u32) -> ANNResult<()>; + + /// Return `true` if internal ID `i` is currently readable, returning `None` if `i` + /// is outside `0..self.id_limit()`. + fn is_readable(&self, i: u32) -> Option; +} + +/// Attempt to write data into a [`Representation`]. +/// +/// This will attempt to find an available internal ID to which `element` can be assigned, +/// failing if no such ID can be found. The write is not eagerly committed. Instead, a +/// [`Guard`] is returned, allowing writes to be aborted if necessary. +pub trait Set: Representation { + /// The type of the [`Guard`] used to defer commitment of the write. + type Guard<'a>: Guard; + + /// Attempt to write the data in `element` into the [`Representation`]. + /// + /// Returns [`Self::Guard`] to retrieve the allocated internal ID for `element` and to + /// defer commitment of the write until external code is ready. + /// + /// Dropping an unpublished [`Guard`] must abort the write and leave its data unpublished. + fn set(&self, element: T) -> ANNResult>; +} + +/// An insert guard for [`Set`] providing deferred commitment of pending writes. +/// +/// Guards provide several services: +/// +/// * [`Guard::id`] returns the associated internal ID. +/// * [`Guard::publish`] commits the change, making the data at the slot publicly visible. +/// +/// Dropping a guard without calling [`Guard::publish`] indicates a failed insert and +/// implementations should abort the write and leave the data unpublished. +pub trait Guard { + fn id(&self) -> u32; + fn publish(self); +} + +/// Enable search over vectors defined by a [`Representation`]. +pub trait Search: Send + Sync + 'static { + /// The type of the query. This should be equivalent to the generic parameter in + /// [`Set`], but needs to be replicated here due to limitations in the current trait + /// design. + type Query<'a>; + + /// Construct a [`crate::provider::SearchAccessor`] for the query. + #[doc(hidden)] + fn search_accessor<'a>( + &'a self, + query: Self::Query<'a>, + provider: &'a (dyn std::any::Any + Send + Sync), + counters: LocalCounters<'a>, + ) -> ANNResult>; +} + +/// An insert-specific specialization of [`Search`]. +/// +/// Note that the bounds for this trait are unnecessarily complicated, but require changes +/// to [`diskann`] to fully resolve. +pub trait Insert: Search + for<'a> Set> { + #[doc(hidden)] + fn insert_search_accessor<'a>( + &'a self, + query: Self::Query<'a>, + provider: &'a (dyn std::any::Any + Send + Sync), + counters: LocalCounters<'a>, + ) -> ANNResult> { + self.search_accessor(query, provider, counters) + } + + #[doc(hidden)] + fn prune_accessor<'a>( + &'a self, + counters: LocalCounters<'a>, + ) -> ANNResult>; +} + +//-----------------// +// Internal Traits // +//-----------------// + +/// Trait-object-based implementation of +/// [`diskann::graph::glue::SearchAccessor::expand_beam`]. +/// +/// Dynamic dispatch is used to enable aggressive specialization of this primitive without +/// monomorphizing the entire search algorithm. Example specializations include: +/// +/// * Optimizing for certain fixed dimensions. +/// * Inlining metric-specific distance functions. +/// * Tailoring prefetching to the dimension. +/// +/// # Safety +/// +/// Implementors must ensure that every ID accepted by [`Self::id_limit`] can be passed to +/// [`Self::expand_beam`] without an out-of-bounds memory access, including accesses performed +/// for prefetching. +pub(crate) unsafe trait ExpandBeam: Send + Sync + std::fmt::Debug { + /// Evaluate a raw distance against index `i`. + fn evaluate(&self, i: u32) -> ANNResult>; + + /// Return an [`IdLimit`] for this primitive. + /// + /// Callers must be able to use this limit to satisfy the safety pre-conditions for + /// [`Self::expand_beam`]. + /// + /// See also: [`IdLimit::is_in_bounds`]. + fn id_limit(&self) -> IdLimit; + + /// Compute the distance between the query and each neighbor in `list`. + /// + /// Unreadable entries may be omitted. Return the number of entries in `buffer` that were + /// written so that `buffer[..returned]` contains the expansion IDs and distances. + /// + /// # Safety + /// + /// * All items in `list` must be in bounds with respect to [`Self::id_limit`]. + /// * `buffer.len() >= list.len()`. + unsafe fn expand_beam(&self, list: &[u32], buffer: &mut [(u32, f32)]) -> ANNResult; +} + +/// Trait-object-based implementation for [`diskann::graph::glue::PruneAccessor`]. +/// +/// [`Self::prepare`] assigns a [`PruneKey`] to each retrieved entry, allowing implementations +/// to buffer data in a representation suitable for pruning. A prune session consists of one +/// call to [`Self::prepare`] followed by calls to [`Self::evaluate`] using the keys it +/// produced. +/// +/// [`Self::prepare`] can be called multiple times for a single [`Prune`] object. +/// Implementations may assume that each call starts a new prune session and do not need to +/// buffer entries from all calls. +pub(crate) trait Prune: Send + Sync + std::fmt::Debug { + /// Prepare items for pruning. + /// + /// The `items` iterator contains all the internal IDs that need to be pruned as keys. + /// Each value in `items` is an output slot that should be populated with a [`PruneKey`] + /// if retrieval succeeds. Implementations are responsible for providing unique + /// [`PruneKey`]s and may assume these slots are initialized to `None`. + /// + /// Returns the total number of internal IDs that were successfully buffered. + fn prepare( + &mut self, + items: hashbrown::hash_map::IterMut<'_, u32, Option>, + ) -> ANNResult; + + /// Compute the distance between the elements referenced by the two [`PruneKey`]s. + /// + /// Implementations may assume that `a` and `b` were produced by the most recent call to + /// [`Self::prepare`] and are allowed to panic if this is violated. This property may + /// **not** be relied on for `unsafe` code. + fn evaluate(&self, a: PruneKey, b: PruneKey) -> f32; +} + +/// A [`Prune`]-local miniature internal ID. +#[derive(Debug, Clone, Copy)] +pub(crate) struct PruneKey(NonZeroU16); + +impl PruneKey { + const ONE: Self = Self(NonZeroU16::new(1).unwrap()); + + /// Return the initial [`PruneKey`] in a sequence. + pub(crate) fn counter() -> Self { + Self::ONE + } + + /// Increment `self` by 1. + /// + /// Overflows if called approximately 2^16 times. + pub(crate) fn increment(self) -> Result { + match self.0.checked_add(1) { + Some(v) => Ok(Self(v)), + None => Err(Overflow), + } + } + + /// Return the zero-based index represented by this key. + pub(crate) fn index(self) -> usize { + usize::from(self.0.get()) - 1 + } +} + +/// Incrementing a [`PruneKey`] overflowed. +#[derive(Debug, Error)] +#[error("prune list exceeded u16::MAX")] +pub(crate) struct Overflow; + +diskann::convert_error!(Overflow); diff --git a/diskann-inmem/src/store.rs b/diskann-inmem/src/store.rs deleted file mode 100644 index cd845230b4..0000000000 --- a/diskann-inmem/src/store.rs +++ /dev/null @@ -1,1049 +0,0 @@ -/* - * Copyright (c) Microsoft Corporation. - * Licensed under the MIT license. - */ - -//! A concurrent in-memory data store for uniformly sized data. -//! -//! This supports concurrent data access, deletes, and inserts through a safe interface. -//! Data is stored internally in slots indexed from `[0..N)` with `K` points reserved at the -//! end at positions `[N..N+K)`. -//! -//! ## Reading -//! -//! Read access requires a [`Reader`] produced by [`Store::reader`]. [`Reader::read`] -//! provides read-only access to data at slot `i` if the data is valid for reads. -//! -//! ## Writing -//! -//! [`Store::acquire`] is used to find and claim an unused internal [`Slot`]. A [`Slot`] -//! provides write access to its corresponding data which is published when the [`Slot`] is -//! dropped. -//! -//! The index of the slot chosen may be obtained via [`Slot::slot`]. -//! -//! ## Deleting -//! -//! Data is deleted via [`Store::retire`]. This immediately marks the corresponding slot as -//! unavailable for future readers. However, the retired slot will not be reused until the -//! [`Store`] can guarantee that no [`Reader`]s that could be using the data are active. -//! -//! Slots are automatically reclaimed as part of slot acquisition in the "writing" phase. -//! -//! ## Neighbor Access -//! -//! The [`Store`] also contains a [`Neighbors`] instance to store adjacency lists. Since -//! neighbors are generally accessed less frequently than data with a higher volume of write -//! traffic, fine-grained locks are used for this data structure. -//! -//! # Details -//! -//! This uses an implementation of the epoch-based reclamation (EBR) provided by [`Registry`]. -//! Concurrency tags are mirrored inline with the stored data (just after the data payload) -//! to keep memory access localized. As such, high-performance implementations will want to -//! fetch the last cache line of data first to ensure the tag is resident in cache for faster -//! data checks. -//! -//! The EBR scheme allows readers to safely access data while only generating read traffic to -//! the CPU caches. The cost is that there is a delay between when slots are retired and when -//! they can be reused, with a long lived [`Reader`] blocking this reclamation. As such, -//! users of this data structure should ensure that [`Reader`]s are reasonably short lived. -//! -//! Internally, the data belongs to a single allocation. - -use std::{ - iter::repeat_n, - num::{NonZeroU32, NonZeroUsize}, - sync::atomic::Ordering, -}; - -use diskann::utils::IntoUsize; -use diskann_utils::views::MatrixView; -use thiserror::Error; - -use crate::{ - buffer::{Buffer, BufferError, RawSlice}, - epoch::{self, Registry}, - freelist::{self, Freelist}, - neighbors::{Neighbors, NeighborsError}, - num::{Align, Bytes}, - tag::{AtomicTag, Tag}, -}; - -/// Configuration for the concurrenct store. -#[derive(Debug)] -pub(crate) struct Config { - /// The number of non-frozen slots to create space for. - entries: usize, - - /// The size of each slot. - bytes: Bytes, - - /// The maximum number of neighbors in each adjacency list. - max_neighbors: usize, - - /// The number of epoch guard slots. - /// - /// Increasing this number will increase the number of threads that can work concurrently - /// on the index at the cost of longer scan times for epoch advancement. - epoch_guard_slots: NonZeroUsize, - - /// The capacity of the fast free list. - freelist_recycle_capacity: NonZeroU32, -} - -impl Config { - /// Create a new `Config` capable of holding `entries` non-frozen points each of size - /// `bytes`. All adjacency lists will have a maximum capacity of `max_neighbors`. - pub(crate) fn new(entries: usize, bytes: Bytes, max_neighbors: usize) -> Self { - const DEFAULT_FREELIST_RECYCLE_CAPACITY: NonZeroU32 = NonZeroU32::new(1024).unwrap(); - Self { - entries, - bytes, - max_neighbors, - epoch_guard_slots: Registry::default_guard_slots(), - freelist_recycle_capacity: DEFAULT_FREELIST_RECYCLE_CAPACITY, - } - } - - /// Overwrite the number of epoch guard slots. - /// - /// Increasing this number will increase the number of threads that can work concurrently - /// on the index at the cost of longer scan times for epoch advancement. - pub(crate) fn epoch_guard_slots(&mut self, epoch_guard_slots: NonZeroUsize) -> &mut Self { - self.epoch_guard_slots = epoch_guard_slots; - self - } - - /// Overwrite the capacity of the freelist recycle queue. - /// - /// Increasing the capacity of the queue will allow more recycled IDs to be retrieved - /// without triggering a scan, but will cost more memory. - pub(crate) fn freelist_recycle_capacity( - &mut self, - freelist_recycle_capacity: NonZeroU32, - ) -> &mut Self { - self.freelist_recycle_capacity = freelist_recycle_capacity; - self - } -} - -/// A concurrent data and graph store. -#[derive(Debug)] -pub(crate) struct Store { - // The invasive store where concurrency tags are stored inline with the data. - // - // These tags are mirrored from `tags` - with the latter being used for secondary scans - // offering slightly better locality. - // - // The inline tags are stored after the data. - buffer: Buffer, - - // The unpadded size of each row in `buffer`. This includes both the data **and** the - // 1-byte tag. Tags are located at byte `unpadded - 1`. - unpadded: Bytes, - - // The number of unfrozen points. This is guaranteed to be less than `buffer`. - unfrozen: usize, - - // The authoritative source of truth for the state of each slot. - tags: Vec, - freelist: Freelist, - - // EBR registry. - registry: Registry, - - // Graph. - neighbors: Neighbors, -} - -/// The number of bytes occupied by the in-line concurrency tag. -pub(crate) const TAG_SIZE: Bytes = Bytes::size_of::(); - -const TWO: NonZeroUsize = NonZeroUsize::new(2).unwrap(); - -// TODO: This is a guess and probably needs tuning. -const RETRY_LIMIT: usize = 20; - -impl Store { - /// Create a new [`Store`]. The entries within `init` will be used as frozen points - /// within the store and must be compatible the the number of bytes in `config`. - pub(crate) fn new( - config: Config, - // entries: usize, - // bytes: Bytes, - // max_neighbors: usize, - init: MatrixView<'_, u8>, - ) -> Result { - let Config { - entries, - bytes, - max_neighbors, - epoch_guard_slots, - freelist_recycle_capacity, - } = config; - - if init.ncols() != bytes.value() { - return Err(StoreError::mismatched_frozen_point_dim(init.ncols(), bytes)); - } - - if init.nrows() == 0 { - return Err(StoreError::need_frozen_point()); - } - - #[expect( - clippy::expect_used, - reason = "we expect `init` to have at least one row, so this should never happen" - )] - let unpadded = bytes - .checked_add(TAG_SIZE) - .expect("unreachable because `init` cannot exceed `isize::MAX` bytes"); - - // Pad to half a cache line. When data occupies just part of a cache line, this - // results in the same total number of cache lines being fetched while potentially - // enabling more compact memory. - #[expect( - clippy::expect_used, - reason = "we expect `init` to have at least one row, so this should never happen" - )] - let padded_bytes = unpadded - .checked_next_multiple_of(Bytes::CACHELINE.div(TWO)) - .expect("unreachable because `init` cannot exceed `isize::MAX` bytes"); - - let too_many_entries = || StoreError::too_many_entries(entries, init.nrows()); - - // We have a hard upper-bound of `u32::MAX` total slots. - // - // This enforces that bound. - let entries: u32 = entries.try_into().map_err(|_| too_many_entries())?; - - let frozen: u32 = init.nrows().try_into().map_err(|_| too_many_entries())?; - - let total: u32 = entries.checked_add(frozen).ok_or_else(too_many_entries)?; - - let max_neighbors: u32 = max_neighbors - .try_into() - .map_err(|_| StoreError::too_many_neighbors(max_neighbors))?; - - let me = Self { - buffer: Buffer::new(total.into_usize(), padded_bytes, Align::_128)?, - unpadded, - unfrozen: entries.into_usize(), - tags: repeat_n(Tag::AVAILABLE, total.into_usize()) - .map(AtomicTag::new) - .collect(), - - // NOTE: The `Freelist` is initialized to `entries` and not `total` because - // we do not want it to release frozen IDs. - freelist: Freelist::new(entries, freelist_recycle_capacity), - registry: Registry::with_capacity(epoch_guard_slots), - neighbors: Neighbors::new(total, max_neighbors)?, - }; - - // Populate frozen points. - for (i, data) in init.row_iter().enumerate() { - // We have checked that the total number of entries fits in `u32`, so this - // arithmetic cannot overflow. - #[expect(clippy::expect_used, reason = "this should always succeed")] - let mut slot = me - .slot(entries + (i as u32)) - .expect("store was just created - claiming the slot must succeed"); - - slot.as_mut_slice().copy_from_slice(data); - slot.freeze(); - } - - Ok(me) - } - - /// Return the range of slots containing frozen items in `self`. - pub(crate) fn frozen(&self) -> std::ops::Range { - (self.unfrozen as u32)..(self.buffer.len() as u32) - } - - /// Return the number of bytes occupied by each entry. - pub(crate) fn bytes(&self) -> Bytes { - self.unpadded - } - - /// Return the maximum degree that can be stored in the graph. - pub(crate) fn max_degree(&self) -> usize { - self.neighbors.max_length() - } - - /// Attempt to reclaim retired slots. - /// - /// If successful, returns the number of slots reclaimed. - pub(crate) fn try_drain(&self) -> Option { - fn release(tag: &AtomicTag, kind: &'static str) { - // Use `Release` ordering to ensure that the store to the mirror cannot get moved - // after the store to the authoritative list. - // - // The `load + check` is just runtime validation. The calling thread is expected - // to have exclusive ownership of this tag. - // - // Using a load + store avoids using a CAS style loop, which can be cheaper for - // the bulk styl operations we're going here at the cost of precision. - assert_eq!( - tag.load(Ordering::Relaxed), - Tag::RETIRING, - "CONCURRENCY VIOLATION: {}", - kind, - ); - - tag.store(Tag::AVAILABLE, Ordering::Release); - } - - let drain = self.registry.try_advance()?; - let items = drain.len(); - for i in drain { - assert!( - i.into_usize() < self.buffer.len(), - "received an invalid ID ({}) while reclaiming slots - max allowed is {}", - i, - self.buffer.len(), - ); - - // We release the mirror before the main tag. The other direction would - // prematurely advertise availability. - // - // SAFETY: We've verified that `i` is in-bounds. - let (mirror, _) = unsafe { self.data_unchecked(i.into_usize()) }; - release(mirror, "mirror"); - release(&self.tags[i.into_usize()], "tag"); - self.freelist.push(i); - } - Some(items) - } - - /// Return a [`Reader`] into the store. - /// - /// # Errors - /// - /// Returns [`epoch::Unavailable`] if there are too many active readers. - pub(crate) fn reader(&self) -> Result, epoch::Unavailable> { - Ok(Reader { - buffer: &self.buffer, - unpadded: self.unpadded, - neighbors: &self.neighbors, - _guard: self.registry.guard()?, - }) - } - - /// Attempt to acquire a new [`Slot`] for writing. - /// - /// This method first consults the freelist and falls back to scanning the tags list - /// if no ID is available from the fast path. - pub(crate) fn acquire(&self) -> Option> { - for _ in 0..RETRY_LIMIT { - match self.freelist.pop() { - freelist::Id::Found(id) => { - if let Some(slot) = self.slot(id) { - return Some(slot); - } - } - freelist::Id::Scan => match self.scan_acquire() { - Some(slot) => return Some(slot), - None => { - self.try_drain(); - } - }, - } - } - None - } - - /// Attempt to retire slot `i`. If successful, this slot will be placed in an internal - /// retirement queue for reclamation once we can prove no readers are active that could - /// have observed this transition. - /// - /// Returns `Ok(())` if the slot was successfully retired. - /// - /// # Errors - /// - /// Returns an error in any of the following conditions: - /// - /// * The slot index `i` is out-of-bounds. - /// * The slot is not in a state that can be retired (e.g., it is already retired or - /// is owned by a different thread). - /// * An [`epoch::Guard`] could not be obtained due to registration slot exhaustion. - /// * An attempt to acquire the slot after these checks races with another thread and - /// the race was lost. - pub(crate) fn retire(&self, i: usize) -> Result<(), RetireError> { - let tag = self.tags.get(i).ok_or(RetireError::OutOfBounds)?; - let current = tag.load(Ordering::Relaxed); - - // We can only perform a deletion if the generation is not in a reserved state. - if current.is_reserved() { - return Err(RetireError::SlotIsReserved { tag: current }); - } - - let guard = self - .registry - .guard() - .map_err(RetireError::GuardUnavailable)?; - - let retiring = Tag::RETIRING; - - // Even if we make this change, we can't access any data until we wait for the - // epoch to be bumped. As such, relaxed semantics are fine. - match tag.compare_exchange(current, retiring, Ordering::Relaxed, Ordering::Relaxed) { - Ok(_) => { - // Set the metadata in the mirror as well. - // - // SAFETY: We've checked that `i` is in-bounds. - let (mirror, _) = unsafe { self.data_unchecked(i) }; - mirror.store(retiring, Ordering::Relaxed); - guard.retire(i as u32); - Ok(()) - } - Err(_) => Err(RetireError::CouldNotClaimSlot), - } - } - - /// A somewhat crude algorithm for cooperatively performing slot scanning. - /// - /// This uses [`Freelist::scan`] to acquire a disjoint chunk of the ID space for scanning, - /// spreading out the search across multiple threads. - /// - /// If we successfully acquire a slot, we continue for the rest of the bucket returned - /// by [`Freelist::scan`] and add any available slots to the freelist (allowing other - /// threads to find them). - /// - /// Periodically, the freelist is checked to see if another thread has found an available - /// slot for us. - fn scan_acquire(&self) -> Option> { - // This is potentially quite slow - but stop if we've scanned the entire range - // without finding anything. - let mut remaining = self.unfrozen.div_ceil(RETRY_LIMIT); - let mut chunks_since_freelist_check = 0; - let mut acquired: Option> = None; - - while remaining != 0 { - let chunk = self.freelist.scan(); - remaining = remaining.saturating_sub(chunk.len()); - - for slot in chunk { - #[expect( - clippy::expect_used, - reason = "this is a serious bug with the freelist" - )] - let tag = self - .tags - .get(slot.into_usize()) - .expect("freelist scan should not give out invalid IDs"); - - // If this slot is available and we haven't claimed a slot yet, try to - // claim it. Otherwise, continue with the scan to partially repopulate the - // freelist for other threads. - if tag.load(Ordering::Relaxed) == Tag::AVAILABLE { - if acquired.is_none() { - // SAFETY: We're guaranteed that `tag` belongs to `slot`. - acquired = unsafe { self.try_acquire(tag, slot) }; - } else { - self.freelist.push(slot); - } - } - } - - if acquired.is_some() { - return acquired; - } - - chunks_since_freelist_check += 1; - if chunks_since_freelist_check == 4 { - if let Some(id) = self.freelist.pop_recycled() - && let Some(slot) = self.slot(id) - { - return Some(slot); - } - chunks_since_freelist_check = 0; - } - } - None - } - - fn slot(&self, i: u32) -> Option> { - let tag = &self.tags.get(i.into_usize())?; - - // SAFETY: We've guaranteed that `tag` belongs to `slot`. - unsafe { self.try_acquire(tag, i) } - } - - /// Try to acquire `slot` with the associated `tag`. - /// - /// # Safety - /// - /// Caller asserts that `tag` was obtained from `self.tags[slot]`. This is meant as - /// a performance optimization where `tag` is first queried for potential availability. - unsafe fn try_acquire<'a>(&'a self, tag: &'a AtomicTag, slot: u32) -> Option> { - if tag.load(Ordering::Relaxed) != Tag::AVAILABLE { - return None; - } - - match tag.compare_exchange( - Tag::AVAILABLE, - Tag::OWNED, - Ordering::Relaxed, - Ordering::Relaxed, - ) { - Ok(_) => { - // SAFETY: Inherited from caller - `slot` is in-bounds. - let (mirror, data) = unsafe { self.data_unchecked(slot.into_usize()) }; - Some(Slot { - tag, - mirror, - data, - slot, - }) - } - Err(_) => None, - } - } - - /// Return the data at position `i` without bound-checking. - /// - /// # Safety - /// - /// The index `i` must be less then `self.buffer.len()`. - unsafe fn data_unchecked(&self, i: usize) -> (&AtomicTag, RawSlice<'_>) { - // SAFETY: inherited from caller. - let (data, mirror) = unsafe { self.buffer.get_unchecked(i) } - .truncate(self.unpadded) - .split(self.unpadded.unchecked_sub(TAG_SIZE)); - ( - // SAFETY: We're careful in this module to ensure the inline tags are only - // ever accessed atomically. - unsafe { AtomicTag::from_ptr(mirror.as_mut_ptr().cast()) }, - data, - ) - } - - /// Return whether or not it is probably okay to read from the slot `i`. - /// - /// This check is approximate and non-synchronizing. To fully check, [`Reader::can_read`] - /// must be used. - /// - /// Returns `None` is index `i` is out-of-bounds. - pub(crate) fn can_read_approximate(&self, i: usize) -> Option { - self.tags - .get(i) - .map(|tag| tag.load(Ordering::Relaxed).can_read()) - } - - #[cfg(test)] - fn writable(&self) -> std::ops::Range { - 0..self.unfrozen as u32 - } -} - -/// Errors occurring during [`Store::new`]. -#[derive(Debug, Error)] -#[error(transparent)] -pub(crate) struct StoreError(StoreErrorInner); - -impl StoreError { - fn mismatched_frozen_point_dim(dim: usize, bytes: Bytes) -> Self { - Self(StoreErrorInner::MismatchedFrozenPointDim { dim, bytes }) - } - - fn need_frozen_point() -> Self { - Self(StoreErrorInner::NeedFrozenPoint) - } - - fn too_many_entries(entries: usize, frozen: usize) -> Self { - Self(StoreErrorInner::TooManyEntries { entries, frozen }) - } - - fn too_many_neighbors(neighbors: usize) -> Self { - Self(StoreErrorInner::TooManyNeighbors { neighbors }) - } -} - -impl From for StoreError { - fn from(err: BufferError) -> Self { - Self(err.into()) - } -} - -impl From for StoreError { - fn from(err: NeighborsError) -> Self { - Self(err.into()) - } -} - -#[derive(Debug, Error)] -enum StoreErrorInner { - #[error( - "frozen point dim ({}) must have the same dimensionality as requested bytes ({})", - dim, - bytes - )] - MismatchedFrozenPointDim { dim: usize, bytes: Bytes }, - #[error("at least one frozen point must be provided")] - NeedFrozenPoint, - #[error( - "total points ({} + {} frozen) must not exceed `u32::MAX`", - entries, - frozen - )] - TooManyEntries { entries: usize, frozen: usize }, - #[error("number of neighbors ({}) may not exceed `u32::MAX`", neighbors)] - TooManyNeighbors { neighbors: usize }, - #[error(transparent)] - BufferError(#[from] BufferError), - #[error(transparent)] - NeighborsError(#[from] NeighborsError), -} - -/// Error conditions for [`Store::retire`]. -#[derive(Debug, Error)] -pub(crate) enum RetireError { - /// Slot index was out-of-bounds. - #[error("index out of bounds")] - OutOfBounds, - /// The slot cannot be retired because it is in a reserved state. - #[error("slot is reserved: {}", tag)] - SlotIsReserved { tag: Tag }, - /// An [`epoch::Guard`] could not be acquired. - #[error(transparent)] - GuardUnavailable(epoch::Unavailable), - /// Another thread won the retirement race. - #[error("could not claim slot")] - CouldNotClaimSlot, -} - -/// An epoch protected reader into a [`Store`]. -/// -/// Created via [`Store::reader`]. -#[derive(Debug)] -pub(crate) struct Reader<'a> { - buffer: &'a Buffer, - unpadded: Bytes, - neighbors: &'a Neighbors, - // It's important that we hold onto this, even if we don't use it. - _guard: epoch::Guard<'a>, -} - -impl<'a> Reader<'a> { - /// Attempt to read the value at index `i`. This can fail for any of the - /// following reasons: - /// - /// 1. Index `i` is out-of-bounds. - /// 2. The read cannot be guaranteed to be race-free. - #[inline] - pub(crate) fn read(&self, i: usize) -> Option<&[u8]> { - if self.is_in_bounds(i) { - // SAFETY: `i` is in-bounds. - unsafe { self.read_in_bounds(i) } - } else { - None - } - } - - /// Return `true` if the index `i` is in-bounds. - #[inline] - #[must_use = "this function has no side-effects"] - pub(crate) fn is_in_bounds(&self, i: usize) -> bool { - i < self.buffer.len() - } - - /// Return `true` if it is safe to read the data at position `i`. - /// - /// This guarantee only holds while `self` is alive. Construction of a new [`Reader`] - /// requires a separate check. - #[cfg_attr( - not(test), - expect( - dead_code, - reason = "this is non-trivial method that likely be used in the future" - ) - )] - pub(crate) fn can_read(&self, i: usize) -> Option { - if !self.is_in_bounds(i) { - return None; - } - - // SAFETY: We've checked that `i` is in-bounds. - // - // Further, we guarantee that `self.unpadded >= TAG_SIZE`, so the pointer arithmetic - // is in-bounds. - let tag_ptr = unsafe { - self.buffer - .get_unchecked(i) - .as_mut_ptr() - .add(self.unpadded.unchecked_sub(TAG_SIZE).value()) - }; - - // SAFETY: We only access tag pointers atomically. - let can_read = unsafe { AtomicTag::from_ptr(tag_ptr.cast()) } - .load(Ordering::Acquire) - .can_read(); - - Some(can_read) - } - - /// Read the data as position `i` if it is guaranteed to be race-free without bounds - /// checking. - /// - /// # Safety - /// - /// The index `i` must satisfy [`Self::is_in_bounds`]. - #[inline] - pub(crate) unsafe fn read_in_bounds(&self, i: usize) -> Option<&[u8]> { - debug_assert!(self.is_in_bounds(i)); - - // SAFETY: - // - // * The caller asserts `i` is in-bounds. - // * We maintain an internal invariant that `self.buffer.stride() <= self.unpadded`. - // * Further, we maintain that `self.unpadded >= TAG_SIZE`. - let (data, tag_ptr) = unsafe { - self.buffer - .get_unchecked(i) - .truncate_unchecked(self.unpadded) - .split_unchecked(self.unpadded.unchecked_sub(TAG_SIZE)) - }; - - // NOTE: Must be `Acquire` to correctly synchronize with writes. - // - // SAFETY: We are careful in this module to ensure that inline tags are only accessed - // atomically. - let can_read = unsafe { AtomicTag::from_ptr(tag_ptr.as_mut_ptr().cast()) } - .load(Ordering::Acquire) - .can_read(); - - if can_read { - // SAFETY: We've passed the `can_read` check - `_guard` will ensure the read - // slice is valid and race-free. - Some(unsafe { data.as_slice() }) - } else { - None - } - } - - /// Return the raw data slice for index `i` without any race guarantees. - /// - /// # Safety - /// - /// The index `i` must be satisfy [`Self::is_in_bounds`]. - #[inline] - pub(crate) unsafe fn read_raw_unchecked(&self, i: usize) -> RawSlice<'_> { - // SAFETY: Inherited from caller: `i` is inbounds. - unsafe { self.buffer.get_unchecked(i) }.truncate(self.unpadded) - } - - /// Return the number of bytes for each entry. - pub(crate) fn bytes(&self) -> Bytes { - self.unpadded - } - - /// Return [`Neighbors`]. - pub(crate) fn neighbors(&self) -> &Neighbors { - self.neighbors - } -} - -/// A writable buffer into the data managed by a [`Store`], obtained from [`Store::acquire`]. -#[derive(Debug)] -pub(crate) struct Slot<'a> { - tag: &'a AtomicTag, - mirror: &'a AtomicTag, - data: RawSlice<'a>, - slot: u32, -} - -impl<'a> Slot<'a> { - /// View the managed data as a mutable slice. - pub(crate) fn as_mut_slice(&mut self) -> &mut [u8] { - // SAFETY: The slot guarantees exclusive access to its corresponding data. - unsafe { self.data.as_mut_slice() } - } - - /// Return the slot associated with this write. - pub(crate) fn slot(&self) -> u32 { - self.slot - } - - fn freeze(self) { - let me = std::mem::ManuallyDrop::new(self); - me.mirror.store(Tag::FROZEN, Ordering::Release); - me.tag.store(Tag::FROZEN, Ordering::Release); - } - - /// Consume the slot and publish the written data for all readers. - /// - /// Return the internal slot ID. - pub(crate) fn publish(self) -> u32 { - let id = self.slot(); - let me = std::mem::ManuallyDrop::new(self); - me.mirror.store(Tag::PUBLISHED, Ordering::Release); - me.tag.store(Tag::PUBLISHED, Ordering::Release); - id - } -} - -impl Drop for Slot<'_> { - fn drop(&mut self) { - self.mirror.store(Tag::AVAILABLE, Ordering::Release); - self.tag.store(Tag::AVAILABLE, Ordering::Release); - } -} - -/////////// -// Tests // -/////////// - -/// These tests are basic functionality tests for the store. -/// -/// Longer running conurrency tests are in the integration test suite. -#[cfg(test)] -mod tests { - use super::*; - - use diskann_utils::views::Matrix; - - // Build a store with `entries` writable slots of `entry_bytes` each, backed by `frozen` - // zeroed frozen points. The frozen points occupy the highest slot indices. - fn store(entries: usize, entry_bytes: usize, frozen: usize) -> Result { - let mut data = Matrix::new(0u8, frozen, entry_bytes); - let mut base = 0u8; - for row in data.row_iter_mut() { - row.fill(base); - base = base.wrapping_add(1); - } - - let mut config = Config::new(entries, Bytes::new(entry_bytes), 0); - config.epoch_guard_slots(NonZeroUsize::new(10).unwrap()); - config.freelist_recycle_capacity(NonZeroU32::new(16).unwrap()); - - Store::new(config, data.as_view()) - } - - //------------------------// - // Constructor validation // - //------------------------// - - #[test] - fn new_rejects_mismatched_frozen_dim() { - // Frozen point has 8 columns but the store is asked for 16-byte entries. - let data = Matrix::new(0u8, 1, 8); - let err = Store::new(Config::new(4, Bytes::new(16), 0), data.as_view()).unwrap_err(); - assert!(matches!( - err.0, - StoreErrorInner::MismatchedFrozenPointDim { dim: 8, .. } - )); - } - - #[test] - fn new_requires_a_frozen_point() { - let err = store(4, 8, 0).unwrap_err(); - assert!(matches!(err.0, StoreErrorInner::NeedFrozenPoint)); - } - - #[test] - fn new_rejects_total_slot_overflow() { - // `entries` alone fits in u32, but `entries + frozen` overflows it. - let data = Matrix::new(0u8, 1, 8); - let err = Store::new( - Config::new(u32::MAX as usize, Bytes::new(8), 0), - data.as_view(), - ) - .unwrap_err(); - assert!(matches!(err.0, StoreErrorInner::TooManyEntries { .. })); - } - - #[test] - fn new_rejects_too_many_neighbors() { - let data = Matrix::new(0u8, 1, 8); - let err = Store::new( - Config::new(4, Bytes::new(8), u32::MAX.into_usize() + 1), - data.as_view(), - ) - .unwrap_err(); - assert!(matches!(err.0, StoreErrorInner::TooManyNeighbors { .. })); - } - - //--------// - // Layout // - //--------// - - #[test] - fn frozen_range_follows_writable_slots() { - let s = store(4, 8, 2).unwrap(); - - // Writable slots are [0, 4); frozen points occupy [4, 6). - assert_eq!(s.frozen(), 4..6); - - let reader = s.reader().unwrap(); - for i in 0..4 { - assert!(!s.can_read_approximate(i).unwrap()); - assert!(!reader.can_read(i).unwrap()); - assert!(reader.read(i).is_none()); - } - - assert!(s.can_read_approximate(4).unwrap()); - assert!(reader.can_read(4).unwrap()); - assert_eq!(reader.read(4).unwrap(), &[0, 0, 0, 0, 0, 0, 0, 0]); - - assert!(s.can_read_approximate(5).unwrap()); - assert!(reader.can_read(5).unwrap()); - assert_eq!(reader.read(5).unwrap(), &[1, 1, 1, 1, 1, 1, 1, 1]); - - assert!(s.can_read_approximate(6).is_none()); - assert!(reader.can_read(6).is_none()); - assert!(reader.read(6).is_none()); - } - - /////////////// - // Lifecycle // - /////////////// - - #[test] - fn acquire_write_publish_read_roundtrip() { - let s = store(4, 8, 1).unwrap(); - - let reader = s.reader().expect("reader guard available"); - - let idx = { - let mut slot = s.acquire().expect("a fresh store has free slots"); - let idx = slot.slot() as usize; - slot.as_mut_slice() - .copy_from_slice(&[1, 2, 3, 4, 5, 6, 7, 8]); - - // Before the slot is dropped - we should not be able to read it. - assert!(reader.read(idx).is_none()); - assert!(!s.can_read_approximate(idx).unwrap()); - slot.publish(); - idx - }; - - assert_eq!(reader.read(idx), Some([1, 2, 3, 4, 5, 6, 7, 8].as_slice())); - assert!(s.can_read_approximate(idx).unwrap()); - } - - #[test] - fn unpublished_slots_are_immediately_available() { - let s = store(4, 8, 1).unwrap(); - - let reader = s.reader().expect("reader guard available"); - - let idx = { - let mut slot = s.acquire().expect("a fresh store has free slots"); - let idx = slot.slot() as usize; - slot.as_mut_slice() - .copy_from_slice(&[1, 2, 3, 4, 5, 6, 7, 8]); - - // Before the slot is dropped - we should not be able to read it. - assert!(reader.read(idx).is_none()); - assert!(!s.can_read_approximate(idx).unwrap()); - - // NOTE: We do not explicitly publish the slot. - idx - }; - - assert!(reader.read(idx).is_none()); - assert!(!s.can_read_approximate(idx).unwrap()); - } - - #[test] - fn acquire_exhausts_then_reports_none() { - let s = store(2, 8, 1).unwrap(); - // Hold the guards so the slots stay owned. - let _a = s.acquire().expect("first writable slot"); - let _b = s.acquire().expect("second writable slot"); - assert!( - s.acquire().is_none(), - "all writable slots are owned, so acquire must fail" - ); - } - - //--------// - // Retire // - //--------// - - #[test] - fn retire_out_of_bounds() { - let s = store(4, 8, 1).unwrap(); - assert!(matches!(s.retire(999), Err(RetireError::OutOfBounds))); - } - - #[test] - fn retire_rejects_reserved_slots() { - let s = store(4, 8, 1).unwrap(); - // An untouched writable slot is AVAILABLE, which is a reserved state. - assert!(matches!( - s.retire(0), - Err(RetireError::SlotIsReserved { .. }) - )); - // A frozen slot is likewise reserved. - let frozen = s.frozen().start as usize; - assert!(matches!( - s.retire(frozen), - Err(RetireError::SlotIsReserved { .. }) - )); - // An owned slot is not retirable. - let slot = s.acquire().unwrap(); - assert!(matches!( - s.retire(slot.slot() as usize), - Err(RetireError::SlotIsReserved { .. }) - )); - } - - #[test] - fn retire_published_slot_then_unreadable() { - let s = store(4, 8, 1).unwrap(); - - let idx = { - let slot = s.acquire().unwrap(); - slot.publish() as usize - }; - - assert!(s.retire(idx).is_ok()); - - // A reader opened after retirement must not observe the retired slot. - let reader = s.reader().unwrap(); - assert_eq!(reader.read(idx), None); - assert_eq!(reader.can_read(idx), Some(false)); - - // The slot can also not be retired again. - assert!(matches!( - s.retire(idx), - Err(RetireError::SlotIsReserved { .. }) - )); - } - - //---------// - // Recycle // - //---------// - - #[test] - fn test_recycling() { - let entries = if cfg!(miri) { 16 } else { 2048 }; - - let s = store(entries, 4, 2).unwrap(); - - // Claim all slots. - let mut count = 0; - while let Some(slot) = s.acquire() { - slot.publish(); - count += 1; - } - - assert_eq!(count, s.writable().len()); - - // Now that all slots are claimed - retire all slots. - for i in s.writable() { - s.retire(i.into_usize()).unwrap(); - } - - // Verify that we can claim all slots again. - let mut count = 0; - while let Some(slot) = s.acquire() { - slot.publish(); - count += 1; - } - - assert_eq!(count, s.writable().len()); - } -} diff --git a/diskann-inmem/src/store/checked.rs b/diskann-inmem/src/store/checked.rs new file mode 100644 index 0000000000..9f7eea836c --- /dev/null +++ b/diskann-inmem/src/store/checked.rs @@ -0,0 +1,436 @@ +/* + * Copyright (c) Microsoft Corporation. + * Licensed under the MIT license. + */ + +//! # Pedantically testing the EBR protocol +//! +//! The goal here is to detect violations of the state machine outlined in [`slots`]. +//! This is accomplished by using a [`RwLock`] to protect internal [`State`] with guards +//! **only** acquired via [`RwLock::try_read`] and [`RwLock::try_write`]. A correct EBR +//! protocol should ensure that: +//! +//! 1. States where exclusive access is needed have no readers or other concurrent exclusive +//! accesses. +//! +//! 2. States where read access is allowed have no attempts at exclusive access. +//! +//! The "try" interfaces provide us with these checks: +//! +//! 1. [`RwLock::try_write`] will fail if there is any concurrent reader or writer. +//! 2. [`RwLock::try_read`] will fail if there is a writer. +//! +//! Importantly, we *only* use these non-blocking APIs. +//! +//! Reads from the store return a [`Value`], which contains a [`RwLockReadGuard`] for the +//! corresponding slot. Many such guards can coexist. Providing long-lived guards like this +//! improves our chances of catching a bug in the EBR scheme where exclusive access is +//! attempted too early. [`Reader::read`] constrains each [`Value`] to the borrow of the +//! [`Reader`]. Since [`Reader`]s own an [`epoch::Guard`], [`Value`]s are guaranteed to be +//! dropped before their protecting [`epoch::Guard`] is dropped. +//! +//! ## Lifecycle Details +//! +//! Lifecycle transitions are carefully designed to allow readers of the slots to avoid any +//! accesses to the authoritative [`Store`] for read-only operations. The [`Checked`] test +//! code follows this pattern, but this does introduce a subtle detail that is worth +//! highlighting. [`Reader::read`] needs to be able to check a slot for readability +//! **without** trying to acquire a [`RwLockReadGuard`] for that slot. Doing so even briefly +//! will cause a [`RwLock::try_write`] on an otherwise correct [`slots::Slots`] state +//! transition to fail. +//! +//! To circumvent this, an additional [`AtomicBool`] is bundled with [`Entry`] to broadcast +//! readability. This must be checked first on an optimistic read before attempting to +//! acquire the [`RwLockReadGuard`]. This boolean flag is toggled on the following transitions: +//! +//! * "slot" -> "published"/"frozen": Publish as readable. Note that this transition goes +//! from an "exclusive" owned (with a [`RwLockWriteGuard`]) to a shared readable state. +//! +//! As such, it's necessary to release the guard before toggling the "readable" flag to +//! ensure that if [`Reader::read`] observes the "published" state it is guaranteed to +//! succeed in [`RwLockReadGuard`] acquisition. +//! +//! * "published" -> "retiring": This switches to non-readable. Note that there is a race +//! condition where: +//! +//! - (Thread A) [`Reader::read`] observes "readable" and decides to acquire the read guard. +//! - (Thread B) Toggles the "readable" flag. +//! - (Thread A) Finishes acquiring the read guard, even though the "readable" state is no +//! longer broadcasted. +//! +//! This is **exactly** what the EBR protocol makes safe. These races are perfectly fine +//! because the protected [`State::Published`] value remains valid for existing readers +//! while the lifecycle state is "retiring". The EBR protocol ensures that a slot does +//! **not** transition away from "retiring" until thread `A` (and all other concurrent +//! threads that could have observed the slot while it was published) have dropped their +//! read guards, and thus any data accessed while under the guard, since these are +//! outlived by a proper [`epoch::Guard`]. + +#![expect( + clippy::panic, + clippy::expect_used, + reason = "integration-test code is not production code" +)] + +use std::{ + assert_matches, + sync::atomic::{AtomicBool, Ordering}, +}; + +use diskann::utils::IntoUsize; +use parking_lot::{RwLock, RwLockReadGuard, RwLockWriteGuard}; + +use crate::{epoch, num::IdLimit, store::Store}; + +use super::{Lifecycle, slots}; + +/// The state of a slot. +#[derive(Debug, Default)] +enum State { + #[default] + Available, + Published { + value: u64, + }, + Frozen { + value: u64, + }, +} + +/// A slot entry. See the [module level docs](self) for a discussion on the contents of this +/// struct. The table below describes how the combination of fields in this struct maps to +/// the lifecycle states. +/// ```text +/// +-----------------+----------+-----------+------------------------+ +/// | Lifecycle state | readable | State | Lock Expectation | +/// +=================+==========+===========+========================+ +/// | Available | false | Available | unlocked | +/// +-----------------+----------+-----------+------------------------+ +/// | Slot | false | Available | write-locked | +/// +-----------------+----------+-----------+------------------------+ +/// | Published | true | Published | shared reads allowed | +/// +-----------------+----------+-----------+------------------------+ +/// | Retiring | false | Published | existing reads allowed | +/// +-----------------+----------+-----------+------------------------+ +/// | Frozen | true | Frozen | shared reads allowed | +/// +-----------------+----------+-----------+------------------------+ +/// ``` +#[derive(Debug, Default)] +struct Entry { + readable: AtomicBool, + state: RwLock, +} + +impl Entry { + /// Return whether or not this [`Entry`] is broadcasted as readable. + #[must_use] + fn is_readable(&self) -> bool { + self.readable.load(Ordering::Acquire) + } + + /// Attempt to acquire a [`ReadEntry`], failing if the [`Entry`] is not marked as + /// readable without attempting to acquire any locks. + fn try_read(&self) -> Option> { + if self.is_readable() { + // We are relying on EBR to avoid problems with the TOCTOU/ABA race between + // checking the `bool` and attempting to acquire the read guard. + // + // EBR prevents the slot from being reclaimed and reused during the race interval. + // + // See the module level docs. + Some(self.expect_read()) + } else { + None + } + } + + /// Acquire a [`ReadEntry`]. + /// + /// # Panics + /// + /// Panics if the [`RwLockReadGuard`] cannot be immediately acquired. + fn expect_read(&self) -> ReadEntry<'_> { + // NOTE: we *DO NOT* check for `entry.is_readable()` because there is a race where + // the slot is retired after checking the readable state but before this function + // is called. We still expect to acquire the `RwLockReadGuard` in this situation. + + let Some(guard) = self.state.try_read() else { + panic!("concurrency violation when acquiring read guard"); + }; + + ReadEntry { + readable: &self.readable, + guard, + } + } + + /// Acquire a [`WriteEntry`]. + /// + /// # Panics + /// + /// Panics if the [`RwLockWriteGuard`] cannot be immediately acquired. + fn expect_write(&self) -> WriteEntry<'_> { + assert!( + !self.is_readable(), + "concurrency violation - entry should not be readable" + ); + + // Correct usage of the concurrency protocol means that this `try_write` failing + // is a bug. + let Some(guard) = self.state.try_write() else { + panic!("concurrency violation when acquiring write guard"); + }; + + WriteEntry { + readable: &self.readable, + guard, + } + } +} + +/// A readable version of [`Entry`]. +#[derive(Debug)] +struct ReadEntry<'a> { + readable: &'a AtomicBool, + guard: RwLockReadGuard<'a, State>, +} + +impl ReadEntry<'_> { + /// Mark this slot as "retired". + fn retire(self) { + assert_matches!( + *self.guard, + State::Published { .. }, + "\"retire\" should transition out of the \"published\" state", + ); + + // The ordering of dropping the guard vs clearing the "readable" flag doesn't + // really matter because the EBR protocol ensures that final reclamation is + // sufficiently deferred. + let old = self.readable.swap(false, Ordering::Release); + assert!( + old, + "\"retire\" should transition out of the \"published\" state" + ); + } + + fn state(&self) -> &State { + &self.guard + } +} + +/// A writable version of [`Entry`]. +#[derive(Debug)] +struct WriteEntry<'a> { + readable: &'a AtomicBool, + guard: RwLockWriteGuard<'a, State>, +} + +impl WriteEntry<'_> { + /// Transition this slot to "published". + fn publish(mut self, value: u64) { + let old = self.replace(State::Published { value }); + assert_matches!( + old, + State::Available, + "\"publish\" must transition out of the \"available\" state", + ); + + // The ordering here matters. `Reader` must be guaranteed to acquire the read guard + // if it observes `readable = true`. We must drop the guard before broadcasting. + drop(self.guard); + self.readable.store(true, Ordering::Release); + } + + /// Transition this slot to "frozen". + fn freeze(mut self, value: u64) { + let old = self.replace(State::Frozen { value }); + assert_matches!( + old, + State::Available, + "\"freeze\" must transition out of the \"available\" state", + ); + + // See `Self::publish` for ordering details. + drop(self.guard); + self.readable.store(true, Ordering::Release); + } + + /// Transition this slot to "available". + fn reclaim(mut self) { + let old = self.replace(State::Available); + + // Note: The combination of `!readable` and `State::Published` implies "retiring". + // We check that here. + assert!( + !self.readable.load(Ordering::Relaxed), + "\"reclaim\" must transition out of \"retired\"", + ); + + assert_matches!( + old, + State::Published { .. }, + "\"reclaim\" must transition out of \"retired\"", + ); + } + + /// Replace the protected state with `state`, returning the old state. + fn replace(&mut self, mut state: State) -> State { + std::mem::swap(&mut *self.guard, &mut state); + state + } + + fn state(&self) -> &State { + &self.guard + } +} + +/// A [`slots::SlotsConfig`] for [`Checked`]. +#[derive(Debug)] +pub(crate) struct Config(()); + +impl Config { + pub(crate) fn new() -> Self { + Self(()) + } +} + +impl slots::SlotsConfig for Config { + type Slots = Checked; + type Error = diskann::error::Infallible; + + fn build(self, id_limit: IdLimit) -> Result { + Ok(Checked::new(id_limit)) + } +} + +/// A correctness checking [`slots::Slots`]. See the [module level docs](self) for details. +#[derive(Debug)] +pub(crate) struct Checked { + entries: Vec, +} + +impl Checked { + /// Create a new [`Checked`] with `id_limit` slots. + pub(crate) fn new(id_limit: IdLimit) -> Self { + Self { + entries: std::iter::repeat_with(Entry::default) + .take(id_limit.as_usize()) + .collect(), + } + } + + /// Return the [`slots::SlotsConfig`] for [`Self`]. + pub(crate) fn config() -> Config { + Config::new() + } + + /// Return the [`IdLimit`] for this store. + pub(crate) fn id_limit(&self) -> IdLimit { + IdLimit::new(self.entries.len() as u32) + } + + /// Return an epoch-protected [`Reader`] into [`Self`]. + pub(crate) fn reader(store: &Store) -> Result, epoch::Unavailable> { + store.guard(|this, guard: epoch::Guard<'_>| Reader { + parent: this, + _guard: guard, + }) + } +} + +/// A valid, readable entry of [`Checked`]. +#[derive(Debug)] +pub(crate) struct Value<'a> { + value: u64, + _entry: ReadEntry<'a>, +} + +impl Value<'_> { + /// Return the payload for this [`Value`]. + pub(crate) fn get(&self) -> u64 { + self.value + } +} + +/// A reader for [`Checked`]. +#[derive(Debug)] +pub(crate) struct Reader<'a> { + parent: &'a Checked, + _guard: epoch::Guard<'a>, +} + +impl Reader<'_> { + /// Attempt to read the value at slot `i`. + /// + /// Fails if `i` is out-of-bounds, or the slot is not in a readable state. + pub(crate) fn read(&self, i: u32) -> Option> { + if let Some(entry) = self.parent.entries.get(i.into_usize())?.try_read() { + let value = match entry.state() { + State::Frozen { value } | State::Published { value } => value, + State::Available => panic!("concurrency violation"), + }; + + Some(Value { + value: *value, + _entry: entry, + }) + } else { + None + } + } +} + +impl slots::Slots for Checked { + type Slot<'a> = Slot<'a>; + + fn id_limit(&self) -> IdLimit { + ::id_limit(self) + } + + unsafe fn acquire(&self, i: u32, _: Lifecycle) -> Self::Slot<'_> { + Slot::new(self.entries[i.into_usize()].expect_write()) + } + + unsafe fn retire(&self, i: u32, _: Lifecycle) { + self.entries[i.into_usize()].expect_read().retire(); + } + + unsafe fn reclaim(&self, i: u32, _: Lifecycle) { + self.entries[i.into_usize()].expect_write().reclaim(); + } +} + +/// A writable [`slots::Slot`] for [`Checked`]. +#[derive(Debug)] +pub(crate) struct Slot<'a> { + entry: WriteEntry<'a>, + value: Option, +} + +impl<'a> Slot<'a> { + fn new(entry: WriteEntry<'a>) -> Self { + Self { entry, value: None } + } + + /// Write `value` into the slot on a successful state transition. + pub(crate) fn set(&mut self, value: u64) { + self.value = Some(value) + } +} + +impl slots::Slot for Slot<'_> { + fn publish(self, _: Lifecycle) { + let value = self.value.expect("`value` was not set"); + self.entry.publish(value); + } + + fn freeze(self, _: Lifecycle) { + let value = self.value.expect("`value` was not set"); + self.entry.freeze(value); + } + + fn abort(self, _: Lifecycle) { + assert_matches!(self.entry.state(), State::Available); + } +} diff --git a/diskann-inmem/src/store/invasive.rs b/diskann-inmem/src/store/invasive.rs new file mode 100644 index 0000000000..03d6902a6f --- /dev/null +++ b/diskann-inmem/src/store/invasive.rs @@ -0,0 +1,646 @@ +/* + * Copyright (c) Microsoft Corporation. + * Licensed under the MIT license. + */ + +//! A [`slots::Slots`] that maintains an invasive slot state where the data in each slot is +//! a contiguous slice of memory. +//! +//! Slot state is stored as an [`AtomicTag`] immediately after the slot data. +//! +//! ## Lifecycle Details +//! +//! Lifecycle details are relatively straightforward. The invasive [`AtomicTag`] mostly +//! follows the transitions made by the [`Store`]. A [`Reader`] checks the tag for +//! readability before creating a shared reference to the data payload. +//! +//! The problematic transition from "published" to "retiring" is made safe because +//! +//! 1. Each [`Reader`] stores an [`epoch::Guard`]. +//! 2. Slices yielded by [`Reader`] are scoped to the **borrow** of [`Reader`]. +//! +//! This ensures that slices cannot outlive the [`epoch::Guard`] protecting their slot. EBR +//! prevents the slot from transitioning from "retiring" back to "available" and being +//! reused while the guard remains active. +//! +//! The transitions [`slots::Slot::publish`] and [`slots::Slot::freeze`] use release stores. +//! Since these are terminal slot operations, their release stores occur after all payload +//! writes. The acquire load in [`Reader::read`] makes those writes visible before creating +//! a shared slice. +//! +//! ## Safety +//! +//! The safety of this module depends on [`Invasive`] being embedded in a [`Store`] that +//! observes the slot lifecycle. Every lifecycle operation requires a [`Lifecycle`] token, +//! which is constructible only by the parent store module. The unsafe [`slots::Slots`] +//! methods additionally rely on [`Store`] to satisfy their documented state and exclusivity +//! preconditions. + +use std::sync::atomic::Ordering; + +use diskann::utils::IntoUsize; +use thiserror::Error; + +use crate::{ + buffer::{Buffer, BufferError, RawSlice}, + epoch, + num::{Align, Bytes, IdLimit}, + store::{Lifecycle, Store, slots}, + tag::{AtomicTag, Tag}, +}; + +/// A [`slots::SlotsConfig`] for [`Invasive`]. +#[derive(Debug, Clone)] +pub(crate) struct Config { + /// The number of bytes held in each slot. + bytes: Bytes, +} + +impl Config { + /// Create a new [`Config`] for [`Invasive`] reserving `bytes` bytes for each slot. + pub(crate) fn new(bytes: Bytes) -> Self { + Self { bytes } + } + + /// Build an [`Invasive`] store holding `id_limit` slots. + pub(crate) fn build(self, id_limit: IdLimit) -> Result { + let Self { bytes } = self; + Invasive::new(id_limit, bytes) + } +} + +impl slots::SlotsConfig for Config { + type Slots = Invasive; + type Error = InvasiveError; + fn build(self, id_limit: IdLimit) -> Result { + ::build(self, id_limit) + } +} + +/// The invasive store where concurrency tags are stored inline just after the data. +#[derive(Debug)] +pub(crate) struct Invasive { + // The inline tags are `AtomicTag`s stored after the data. + buffer: Buffer, + + // The unpadded size of each row in `buffer`. This includes both the data **and** the + // 1-byte tag. Tags are located at byte `unpadded - 1`. + unpadded: Bytes, +} + +impl Invasive { + /// Construct the [`Config`] for [`Self`]. + /// + /// See also: [`Config::new`]. + pub(crate) fn config(bytes: Bytes) -> Config { + Config::new(bytes) + } + + /// Create a new [`Invasive`] with capacity for `id_limit` slots of `bytes`. + /// + /// # Errors + /// + /// Returns an error if the internal buffer allocation exceeds `isize::MAX` or + /// computation of the padded, invasive bytes exceeds `usize::MAX`. + pub(crate) fn new(id_limit: IdLimit, bytes: Bytes) -> Result { + let Some(unpadded) = bytes.checked_add(AtomicTag::SIZE) else { + return Err(InvasiveError::bytes_overflowed()); + }; + let Some(padded_bytes) = unpadded.checked_next_multiple_of(Bytes::CACHELINE) else { + return Err(InvasiveError::bytes_overflowed()); + }; + + let buffer = match Buffer::new(id_limit.as_usize(), padded_bytes, Align::_128) { + Ok(buffer) => buffer, + Err(err) => return Err(InvasiveError::buffer_error(err)), + }; + + Ok(Self { buffer, unpadded }) + } + + /// Return the [`IdLimit`] for this store. + pub(crate) fn id_limit(&self) -> IdLimit { + // The numeric cast is safe because `Invasive::new` takes an `IdLimit` in its + // constructor, and thus `self.buffer.len()` cannot exceed `u32::MAX`. + IdLimit::new(self.buffer.len() as u32) + } + + /// Return the number of bytes for each entry. + pub(crate) fn bytes(&self) -> Bytes { + self.bytes_plus_tag().unchecked_sub(AtomicTag::SIZE) + } + + /// Return the number of bytes plus the atomic tag. + pub(crate) fn bytes_plus_tag(&self) -> Bytes { + self.unpadded + } + + /// Return a [`Reader`] over [`Self`] inside `store`. + pub(crate) fn reader(store: &Store) -> Result, epoch::Unavailable> { + store.guard(|this, guard: epoch::Guard<'_>| Reader { + buffer: &this.buffer, + unpadded: this.unpadded, + _guard: guard, + }) + } + + /// Return the data at position `i` without bound-checking. + /// + /// # Safety + /// + /// The index `i` must be less than `self.buffer.len()`. + unsafe fn data_unchecked(&self, i: usize) -> (&AtomicTag, RawSlice<'_>) { + // SAFETY: inherited from caller. + let (data, mirror) = unsafe { self.buffer.get_unchecked(i) } + .truncate(self.unpadded) + .split(self.unpadded.unchecked_sub(AtomicTag::SIZE)); + ( + // SAFETY: The tag byte lies within the zero-initialized row, is sufficiently + // aligned for `AtomicTag`, and is only accessed through atomic operations. + unsafe { AtomicTag::from_ptr(mirror.as_mut_ptr().cast()) }, + data, + ) + } + + fn data(&self, i: usize) -> Option<(&AtomicTag, RawSlice<'_>)> { + if i >= self.buffer.len() { + None + } else { + // SAFETY: We've checked that `i` is in-bounds. + Some(unsafe { self.data_unchecked(i) }) + } + } +} + +#[derive(Debug, Error)] +#[error(transparent)] +pub(crate) struct InvasiveError(InvasiveErrorInner); + +impl InvasiveError { + fn bytes_overflowed() -> Self { + Self(InvasiveErrorInner::BytesOverflowed) + } + + fn buffer_error(err: BufferError) -> Self { + Self(InvasiveErrorInner::BufferError(err)) + } +} + +#[derive(Debug, Error)] +enum InvasiveErrorInner { + #[error("computation of the bytes per slot overflowed")] + BytesOverflowed, + #[error(transparent)] + BufferError(BufferError), +} + +impl slots::Slots for Invasive { + type Slot<'a> = Slot<'a>; + + fn id_limit(&self) -> IdLimit { + ::id_limit(self) + } + + #[expect(clippy::panic, reason = "out-of-bounds is a hard program bug")] + unsafe fn acquire(&self, i: u32, _: Lifecycle) -> Self::Slot<'_> { + let Some((tag, data)) = self.data(i.into_usize()) else { + panic!("index {i} is out-of-bounds"); + }; + + // This is a pessimistic check to ensure that the caller is correctly using the + // `slots` API. + debug_assert_eq!( + tag.load(Ordering::Relaxed), + Tag::AVAILABLE, + "concurrency violation", + ); + + // While we can leave this tag as `Tag::AVAILABLE` since it's just a mirror, setting + // it to `Tag::OWNED` lets us more precisely detect misuse from the caller. + tag.store(Tag::OWNED, Ordering::Relaxed); + Slot { tag, data } + } + + #[expect(clippy::panic, reason = "out-of-bounds is a hard program bug")] + unsafe fn reclaim(&self, i: u32, _: Lifecycle) { + let Some((tag, _)) = self.data(i.into_usize()) else { + panic!("index {i} is out-of-bounds"); + }; + + tag.store(Tag::AVAILABLE, Ordering::Release); + } + + #[expect(clippy::panic, reason = "out-of-bounds is a hard program bug")] + unsafe fn retire(&self, i: u32, _: Lifecycle) { + let Some((tag, _)) = self.data(i.into_usize()) else { + panic!("index {i} is out-of-bounds"); + }; + + tag.store(Tag::RETIRING, Ordering::Relaxed); + } +} + +/// A reader into an [`Invasive`] store. +#[derive(Debug)] +pub(crate) struct Reader<'a> { + buffer: &'a Buffer, + unpadded: Bytes, + _guard: epoch::Guard<'a>, +} + +impl<'a> Reader<'a> { + /// Attempt to read the value at index `i`. This can fail for any of the + /// following reasons: + /// + /// 1. Index `i` is out-of-bounds. + /// 2. The read cannot be guaranteed to be race-free. + #[inline] + pub(crate) fn read(&self, i: usize) -> Option<&[u8]> { + if self.is_in_bounds(i) { + // SAFETY: `i` is in-bounds. + unsafe { self.read_in_bounds(i) } + } else { + None + } + } + + /// Return `true` if the index `i` is in-bounds. + #[inline] + #[must_use = "this function has no side-effects"] + pub(crate) fn is_in_bounds(&self, i: usize) -> bool { + i < self.buffer.len() + } + + /// Return the [`IdLimit`] for this collection. + #[inline] + #[must_use = "this function has no side-effects"] + pub(crate) fn id_limit(&self) -> IdLimit { + // Like `Invasive::id_limit`, the numeric cast is safe because by construction, + // the underlying buffer is limited to `u32::MAX`. + IdLimit::new(self.buffer.len() as u32) + } + + /// Return `true` if it is safe to read the data at position `i`. + /// + /// This guarantee only holds while `self` is alive. Construction of a new [`Reader`] + /// requires a separate check. + #[cfg_attr( + not(test), + expect( + dead_code, + reason = "this is non-trivial method that is likely to be used in the future" + ) + )] + pub(crate) fn can_read(&self, i: usize) -> Option { + if !self.is_in_bounds(i) { + return None; + } + + // SAFETY: We've checked that `i` is in-bounds. + // + // Further, we guarantee that `self.unpadded >= AtomicTag::SIZE`, so the pointer + // arithmetic is in-bounds. + let tag_ptr = unsafe { + self.buffer + .get_unchecked(i) + .as_mut_ptr() + .add(self.unpadded.unchecked_sub(AtomicTag::SIZE).value()) + }; + + // SAFETY: We only access tag pointers atomically. + let can_read = unsafe { AtomicTag::from_ptr(tag_ptr.cast()) } + .load(Ordering::Acquire) + .can_read(); + + Some(can_read) + } + + /// Read the data as position `i` if it is guaranteed to be race-free without bounds + /// checking. + /// + /// # Safety + /// + /// The index `i` must satisfy [`Self::is_in_bounds`]. + #[inline] + pub(crate) unsafe fn read_in_bounds(&self, i: usize) -> Option<&[u8]> { + debug_assert!(self.is_in_bounds(i)); + + // SAFETY: + // + // * The caller asserts `i` is in-bounds. + // * We maintain the internal invariant that `self.unpadded <= self.buffer.stride()`. + // * Further, we maintain that `self.unpadded >= AtomicTag::SIZE`. + let (data, tag_ptr) = unsafe { + self.buffer + .get_unchecked(i) + .truncate_unchecked(self.unpadded) + .split_unchecked(self.unpadded.unchecked_sub(AtomicTag::SIZE)) + }; + + // NOTE: Must be `Acquire` to correctly synchronize with writes. + // + // SAFETY: We are careful in this module to ensure that inline tags are only accessed + // atomically. + let can_read = unsafe { AtomicTag::from_ptr(tag_ptr.as_mut_ptr().cast()) } + .load(Ordering::Acquire) + .can_read(); + + if can_read { + // SAFETY: We've passed the `can_read` check - `_guard` will ensure the read + // slice is valid and race-free. + Some(unsafe { data.as_slice() }) + } else { + None + } + } + + /// Return the raw data slice for index `i` without any race guarantees. + /// + /// This includes both the data **and** the invasive tag. + /// + /// # Safety + /// + /// The index `i` must satisfy [`Self::is_in_bounds`]. + /// + /// The returned [`RawSlice`] may only be used for prefetching. Callers must never + /// materialize it as a proper slice or reference. + #[inline] + pub(crate) unsafe fn read_raw_unchecked(&self, i: usize) -> RawSlice<'_> { + // SAFETY: Inherited from caller: `i` is in bounds. + unsafe { self.buffer.get_unchecked(i) }.truncate(self.unpadded) + } + + /// Return the number of bytes for each entry. + pub(crate) fn bytes(&self) -> Bytes { + self.bytes_plus_tag().unchecked_sub(AtomicTag::SIZE) + } + + /// Return the number of bytes plus the atomic tag. + pub(crate) fn bytes_plus_tag(&self) -> Bytes { + self.unpadded + } +} + +/// A [`slots::Slot`] for [`Invasive`]. +#[derive(Debug)] +pub(crate) struct Slot<'a> { + // NOTE: `tag` and `data` must belong to the same slot. + tag: &'a AtomicTag, + data: RawSlice<'a>, +} + +impl<'a> Slot<'a> { + /// Return the data within this slot as a mutable slice. + /// + /// The length of this slice is guaranteed to be the number of bytes passed to + /// [`Invasive::new`] or [`Config::new`]. + pub(crate) fn as_mut_slice(&mut self) -> &mut [u8] { + // SAFETY: Users of the `slots::Slot` are obligated to ensure exclusivity. + // + // Since `Reader` obeys the slots life-cycle requirements, a concurrent reader + // of this data should not be possible. + unsafe { self.data.as_mut_slice() } + } +} + +impl slots::Slot for Slot<'_> { + fn publish(self, _: Lifecycle) { + self.tag.store(Tag::PUBLISHED, Ordering::Release); + } + fn freeze(self, _: Lifecycle) { + self.tag.store(Tag::FROZEN, Ordering::Release); + } + fn abort(self, _: Lifecycle) { + self.tag.store(Tag::AVAILABLE, Ordering::Release); + } +} + +/////////// +// Tests // +/////////// + +#[cfg(test)] +mod tests { + use super::*; + + use std::{ + assert_matches, + num::{NonZeroU32, NonZeroUsize}, + }; + + use crate::{ + num::{Capacity, MaxDegree}, + store, + }; + + // Build a store with `entries` writable slots of `entry_bytes` each, backed by `frozen` + // zeroed frozen points. The frozen points occupy the highest slot indices. + fn store( + entries: usize, + entry_bytes: usize, + frozen: usize, + ) -> Result, store::StoreError> { + let store = Store::new( + store::Layout::new( + Capacity::new(entries), + MaxDegree::new(0), + frozen.try_into().unwrap(), + ), + store::Config::__exhaustive( + NonZeroUsize::new(10).unwrap(), + NonZeroU32::new(16).unwrap(), + ), + Config::new(Bytes::new(entry_bytes)), + )?; + + for (base, id) in store.frozen().enumerate() { + let mut slot = store.slot(id).unwrap(); + slot.data().as_mut_slice().fill(base as u8); + slot.freeze(); + } + + Ok(store) + } + + //--------// + // Layout // + //--------// + + #[test] + fn frozen_range_follows_writable_slots() { + let s = store(4, 8, 2).unwrap(); + + // Writable slots are [0, 4); frozen points occupy [4, 6). + assert_eq!(s.frozen(), 4..6); + + let reader = Invasive::reader(&s).unwrap(); + for i in 0..4 { + assert!(!s.can_read_approximate(i).unwrap()); + assert!(!reader.can_read(i).unwrap()); + assert!(reader.read(i).is_none()); + } + + assert!(s.can_read_approximate(4).unwrap()); + assert!(reader.can_read(4).unwrap()); + assert_eq!(reader.read(4).unwrap(), &[0, 0, 0, 0, 0, 0, 0, 0]); + + assert!(s.can_read_approximate(5).unwrap()); + assert!(reader.can_read(5).unwrap()); + assert_eq!(reader.read(5).unwrap(), &[1, 1, 1, 1, 1, 1, 1, 1]); + + assert!(s.can_read_approximate(6).is_none()); + assert!(reader.can_read(6).is_none()); + assert!(reader.read(6).is_none()); + } + + /////////////// + // Lifecycle // + /////////////// + + #[test] + fn acquire_write_publish_read_roundtrip() { + let s = store(4, 8, 1).unwrap(); + + let reader = Invasive::reader(&s).expect("reader guard available"); + + let idx = { + let mut slot = s.acquire().expect("a fresh store has free slots"); + let idx = slot.slot() as usize; + slot.data() + .as_mut_slice() + .copy_from_slice(&[1, 2, 3, 4, 5, 6, 7, 8]); + + // Before the slot is dropped - we should not be able to read it. + assert!(reader.read(idx).is_none()); + assert!(!s.can_read_approximate(idx).unwrap()); + slot.publish(); + idx + }; + + assert_eq!(reader.read(idx), Some([1, 2, 3, 4, 5, 6, 7, 8].as_slice())); + assert!(s.can_read_approximate(idx).unwrap()); + } + + #[test] + fn unpublished_slots_are_immediately_available() { + let s = store(4, 8, 1).unwrap(); + + let reader = Invasive::reader(&s).expect("reader guard available"); + + let idx = { + let mut slot = s.acquire().expect("a fresh store has free slots"); + let idx = slot.slot() as usize; + slot.data() + .as_mut_slice() + .copy_from_slice(&[1, 2, 3, 4, 5, 6, 7, 8]); + + // Before the slot is dropped - we should not be able to read it. + assert!(reader.read(idx).is_none()); + assert!(!s.can_read_approximate(idx).unwrap()); + + // NOTE: We do not explicitly publish the slot. + idx + }; + + assert!(reader.read(idx).is_none()); + assert!(!s.can_read_approximate(idx).unwrap()); + } + + #[test] + fn acquire_exhausts_then_reports_none() { + let s = store(2, 8, 1).unwrap(); + // Hold the guards so the slots stay owned. + let _a = s.acquire().expect("first writable slot"); + let _b = s.acquire().expect("second writable slot"); + assert!( + s.acquire().is_none(), + "all writable slots are owned, so acquire must fail" + ); + } + + //--------// + // Retire // + //--------// + + #[test] + fn retire_out_of_bounds() { + let s = store(4, 8, 1).unwrap(); + assert_matches!(s.retire(999), Err(store::RetireError::OutOfBounds)); + } + + #[test] + fn retire_rejects_reserved_slots() { + let s = store(4, 8, 1).unwrap(); + // An untouched writable slot is AVAILABLE, which is a reserved state. + assert_matches!(s.retire(0), Err(store::RetireError::SlotIsReserved { .. })); + // A frozen slot is likewise reserved. + let frozen = s.frozen().start as usize; + assert_matches!( + s.retire(frozen), + Err(store::RetireError::SlotIsReserved { .. }) + ); + // An owned slot is not retirable. + let slot = s.acquire().unwrap(); + assert_matches!( + s.retire(slot.slot() as usize), + Err(store::RetireError::SlotIsReserved { .. }) + ); + } + + #[test] + fn retire_published_slot_then_unreadable() { + let s = store(4, 8, 1).unwrap(); + + let idx = { + let slot = s.acquire().unwrap(); + slot.publish() as usize + }; + + assert!(s.retire(idx).is_ok()); + + // A reader opened after retirement must not observe the retired slot. + let reader = Invasive::reader(&s).unwrap(); + assert_eq!(reader.read(idx), None); + assert_eq!(reader.can_read(idx), Some(false)); + + // The slot can also not be retired again. + assert_matches!( + s.retire(idx), + Err(store::RetireError::SlotIsReserved { .. }) + ); + } + + //---------// + // Recycle // + //---------// + + #[test] + fn test_recycling() { + let entries = if cfg!(miri) { 16 } else { 2048 }; + + let s = store(entries, 4, 2).unwrap(); + + // Claim all slots. + let mut count = 0; + while let Some(slot) = s.acquire() { + slot.publish(); + count += 1; + } + + assert_eq!(count, s.writable().len()); + + // Now that all slots are claimed - retire all slots. + for i in s.writable() { + s.retire(i.into_usize()).unwrap(); + } + + // Verify that we can claim all slots again. + let mut count = 0; + while let Some(slot) = s.acquire() { + slot.publish(); + count += 1; + } + + assert_eq!(count, s.writable().len()); + } +} diff --git a/diskann-inmem/src/store/mod.rs b/diskann-inmem/src/store/mod.rs new file mode 100644 index 0000000000..4d9825b598 --- /dev/null +++ b/diskann-inmem/src/store/mod.rs @@ -0,0 +1,1028 @@ +/* + * Copyright (c) Microsoft Corporation. + * Licensed under the MIT license. + */ + +//! Concurrency configuration. + +mod internal_docs { + //! A concurrent in-memory data store for driving [`slots::Slots`]. + //! + //! This supports concurrent data access, deletes, and inserts through a safe interface. + //! Data is stored internally in slots indexed from `[0..N)` with `K` points reserved at the + //! end at positions `[N..N+K)`. + //! + //! ## Reading + //! + //! A [`Store`] provides no direct way of reading data. Instead, the [`slots::Slots`] is + //! responsible for exposing an appropriate reader (e.g., [`invasive::Invasive::reader`]) in + //! accordance with its lifecycle implementation. [`Store::guard`] can be used for + //! this purpose by acquiring an [`epoch::Guard`] for a [`Store`]. + //! + //! ## Writing + //! + //! [`Store::acquire`] is used to find and claim an unused internal [`Slot`]. A [`Slot`] + //! provides write access to its corresponding [`Slot::data`]. Either [`Slot::publish`] + //! or [`Slot::freeze`] can be used to make data readable. + //! + //! If a [`Slot`] is dropped, its corresponding slot is returned to the [`Store`] without + //! publishing its contents. + //! + //! The index of the slot chosen may be obtained via [`Slot::slot`]. + //! + //! ## Deleting + //! + //! Data is deleted via [`Store::retire`]. This immediately marks the corresponding slot + //! as unavailable for future readers. However, the retired slot will not be reused until + //! the [`Store`] can guarantee that no readers that could be using the data are active. + //! + //! Slots are automatically reclaimed as part of slot acquisition in the "writing" phase. + //! + //! ## Neighbor Access + //! + //! The [`Store`] also contains a [`Neighbors`] instance to store adjacency lists. Since + //! neighbors are generally accessed less frequently than data with a higher volume of + //! write traffic, fine-grained locks are used for this data structure. + //! + //! # Details + //! + //! This uses an implementation of the epoch-based reclamation (EBR) provided by + //! [`Registry`]. Slots follow the lifecycle process defined in the [module docs](slots). + //! + //! The EBR scheme allows readers to safely access data while only generating read traffic + //! to the CPU caches. The cost is that there is a delay between when slots are retired + //! and when they can be reused, with a long lived reader blocking this reclamation. As + //! such, users of this data structure should ensure that readers are reasonably short + //! lived. + + #[expect( + unused_imports, + reason = "this keeps cross-references nicer for internal docs" + )] + use super::*; +} + +use std::{ + iter::repeat_n, + mem::ManuallyDrop, + num::{NonZeroU32, NonZeroUsize}, + sync::atomic::Ordering, +}; + +use diskann::{ANNError, utils::IntoUsize}; +use thiserror::Error; + +use crate::{ + buffer::BufferError, + epoch::{self, Registry}, + freelist::{self, Freelist}, + neighbors::{Neighbors, NeighborsError}, + num::{Capacity, IdLimit, MaxDegree}, + tag::{AtomicTag, Tag}, +}; + +pub(crate) mod invasive; +pub(crate) mod slots; + +#[cfg(any(test, feature = "integration-test"))] +pub(crate) mod checked; + +/// To make extra sure that [`slots::Slots`] life-cycle arguments are not callable outside +/// of this module (i.e., elsewhere in this crate), this [`Lifecycle`] marker type is used +/// that is only constructible in this module. +#[derive(Debug)] +pub(crate) struct Lifecycle(()); + +impl Lifecycle { + /// Construct a new [`Lifecycle`]. + /// + /// DO NOT MAKE THIS `pub(anything)`. It helps prevent accidentally interacting with + /// slots when all uses should be managed in this file instead. + const fn new() -> Self { + Self(()) + } +} + +/// Configuration for the concurrent store. +#[derive(Debug, Clone)] +pub struct Config { + /// The number of epoch guard slots. + /// + /// Increasing this number will increase the number of threads that can work concurrently + /// on the index at the cost of longer scan times for epoch advancement. + epoch_guard_slots: NonZeroUsize, + + /// The capacity of the fast free list. + freelist_recycle_capacity: NonZeroU32, +} + +impl Config { + /// Create a new [`Config`] with default concurrency parameters. + pub fn new() -> Self { + const DEFAULT_FREELIST_RECYCLE_CAPACITY: NonZeroU32 = NonZeroU32::new(1024).unwrap(); + Self { + epoch_guard_slots: Registry::default_guard_slots(), + freelist_recycle_capacity: DEFAULT_FREELIST_RECYCLE_CAPACITY, + } + } + + /// Overwrite the number of epoch guard slots. + /// + /// Increasing this number will increase the number of threads that can work concurrently + /// on the index at the cost of longer scan times for epoch advancement. + pub fn epoch_guard_slots(&mut self, epoch_guard_slots: NonZeroUsize) -> &mut Self { + self.epoch_guard_slots = epoch_guard_slots; + self + } + + /// Overwrite the capacity of the freelist recycle queue. + /// + /// Increasing the capacity of the queue will allow more recycled IDs to be retrieved + /// without triggering a scan, but will cost more memory. + pub fn freelist_recycle_capacity( + &mut self, + freelist_recycle_capacity: NonZeroU32, + ) -> &mut Self { + self.freelist_recycle_capacity = freelist_recycle_capacity; + self + } + + /// An exhaustive constructor initializing every element. + /// + /// This is exposed under "integration-test" since it will change to reflect the state + /// of the underlying data structure, potentially causing more churn for users if it were + /// unconditionally exposed. + #[cfg(any(test, feature = "integration-test"))] + #[doc(hidden)] + pub fn __exhaustive( + epoch_guard_slots: NonZeroUsize, + freelist_recycle_capacity: NonZeroU32, + ) -> Self { + Self { + epoch_guard_slots, + freelist_recycle_capacity, + } + } +} + +impl Default for Config { + fn default() -> Self { + Self::new() + } +} + +/// Layout parameters for [`Store`] and the corresponding [`slots::Slots`]. +#[derive(Debug, Clone)] +pub(crate) struct Layout { + /// The number of non-frozen slots to create space for. + capacity: Capacity, + + /// The maximum number of neighbors in each adjacency list. + max_degree: MaxDegree, + + /// The number of immutable points to reserve at the end of the [`Store`]. + frozen: u32, +} + +impl Layout { + /// Create a new [`Layout`] capable of holding `capacity` non-frozen points and `frozen` + /// reserved points at the end. + /// + /// After construction, the only distinction between non-frozen and frozen points is that + /// IDs for frozen points will not be selected through [`Store::acquire`]. Frozen slots + /// IDs can be obtained via [`Store::frozen`], and direct acquisition can be done with + /// [`Store::slot`]. + /// + /// All adjacency lists will have a maximum capacity of `max_degree`. + pub(crate) fn new(capacity: Capacity, max_degree: MaxDegree, frozen: u32) -> Self { + Self { + capacity, + max_degree, + frozen, + } + } +} + +/// A concurrent data and graph store. +#[derive(Debug)] +pub(crate) struct Store { + // The [`slots::Slots`] managed by this [`Store`]. + slots: T, + + // The number of unfrozen points. + unfrozen: Capacity, + + // The authoritative source of truth for the state of each slot. + tags: Vec, + + // Acceleration of finding free slot IDs. + freelist: Freelist, + + // EBR registry. + registry: Registry, + + // Graph. + neighbors: Neighbors, +} + +// TODO: This is a guess and probably needs tuning. +const RETRY_LIMIT: usize = 20; + +impl Store +where + T: slots::Slots, +{ + /// Create a new [`Store`]. + pub(crate) fn new(layout: Layout, config: Config, slots: C) -> Result + where + C: slots::SlotsConfig, + { + let Layout { + capacity, + max_degree, + frozen, + } = layout; + + let Config { + epoch_guard_slots, + freelist_recycle_capacity, + } = config; + + let too_many_entries = || StoreError::too_many_entries(capacity, frozen); + + // We have a hard upper-bound of `u32::MAX` total slots. + // + // This enforces that bound. + let entries: u32 = capacity + .value() + .try_into() + .map_err(|_| too_many_entries())?; + + let id_limit = IdLimit::new(entries.checked_add(frozen).ok_or_else(too_many_entries)?); + + let max_degree: u32 = max_degree + .value() + .try_into() + .map_err(|_| StoreError::too_many_neighbors(max_degree))?; + + let slots = slots::SlotsConfig::build(slots, id_limit).map_err(StoreError::slots)?; + + let slots_id_limit = slots.id_limit(); + if slots_id_limit != id_limit { + return Err(StoreError::invalid_construction(slots_id_limit, id_limit)); + } + + let me = Self { + slots, + unfrozen: capacity, + tags: repeat_n(Tag::AVAILABLE, id_limit.as_usize()) + .map(AtomicTag::new) + .collect(), + + // NOTE: The `Freelist` is initialized to `entries` and not `total` because + // we do not want it to release frozen IDs. + freelist: Freelist::new(entries, freelist_recycle_capacity), + registry: Registry::with_capacity(epoch_guard_slots), + neighbors: Neighbors::new(id_limit, max_degree)?, + }; + + Ok(me) + } + + /// Return the [`slots::Slots`] for this store. + pub(crate) fn slots(&self) -> &T { + &self.slots + } + + /// Return the range of slots containing frozen items in `self`. + pub(crate) fn frozen(&self) -> std::ops::Range { + (self.unfrozen.value() as u32)..self.neighbors.entries() + } + + /// Return the [`IdLimit`] for this store. + pub(crate) fn id_limit(&self) -> IdLimit { + // The numeric cast is safe: We verify during construction that `self.tags.len()` fits + // in a `u32`. + IdLimit::new(self.tags.len() as u32) + } + + /// Return the [`Capacity`] for this store. + pub(crate) fn capacity(&self) -> Capacity { + self.unfrozen + } + + /// Return the [`Neighbors`] for this store. + pub(crate) fn neighbors(&self) -> &Neighbors { + &self.neighbors + } + + /// Attempt to reclaim retired slots. + /// + /// If successful, returns the number of slots reclaimed. + pub(crate) fn try_drain(&self) -> Option { + let drain = self.registry.try_advance()?; + let items = drain.len(); + for i in drain { + #[expect(clippy::panic, reason = "this is an unrecoverable program bug")] + let Some(tag) = self.tags.get(i.into_usize()) else { + panic!( + "received an invalid ID ({}) while reclaiming slots - max allowed is {}", + i, + self.neighbors.entries(), + ); + }; + + // We release the slot before the main tag. The other direction would + // prematurely advertise availability. + // + // SAFETY: IDs only get added to the `epoch::Registry` upon retiring, and are + // not released until the registry confirms that all guards active when the id + // was retired have been dropped. + // + // Therefore, this slot has no accessors and is ready to be reclaimed. + unsafe { slots::Slots::reclaim(self.slots(), i, Lifecycle::new()) }; + + // Use `Release` ordering to ensure that the store to the mirror cannot get moved + // after the store to the authoritative list. + // + // The `load + check` is just runtime validation. The calling thread is expected + // to have exclusive ownership of this tag. + // + // Using a load followed by a store avoids a CAS loop, which may be cheaper for + // bulk reclamation at the cost of detecting concurrent modification only through + // the assertion. + assert_eq!( + tag.load(Ordering::Relaxed), + Tag::RETIRING, + "CONCURRENCY VIOLATION", + ); + + tag.store(Tag::AVAILABLE, Ordering::Release); + self.freelist.push(i); + } + Some(items) + } + + /// Create an [`epoch::Guard`] for the [`epoch::Registry`] within `self` and invoke `f` + /// with that guard, returning the result. + /// + /// This can be used by [`slots::Slots`] readers to establish a verifiable chain of + /// custody for an [`epoch::Guard`] over the slots. + pub(crate) fn guard<'a, F, R>(&'a self, f: F) -> Result + where + F: FnOnce(&'a T, epoch::Guard<'a>) -> R, + { + let guard = self.registry.guard()?; + Ok(f(self.slots(), guard)) + } + + /// Attempt to acquire a new [`Slot`] for writing. + /// + /// This method first consults the freelist and falls back to scanning the tags list + /// if no ID is available from the fast path. + pub(crate) fn acquire(&self) -> Option::Slot<'_>>> { + for _ in 0..RETRY_LIMIT { + match self.freelist.pop() { + freelist::Id::Found(id) => { + if let Some(slot) = self.slot(id) { + return Some(slot); + } + } + freelist::Id::Scan => match self.scan_acquire() { + Some(slot) => return Some(slot), + None => { + self.try_drain(); + } + }, + } + } + None + } + + /// Attempt to retire slot `i`. If successful, this slot will be placed in an internal + /// retirement queue for reclamation once we can prove no readers are active that could + /// have observed this transition. + /// + /// Returns `Ok(())` if the slot was successfully retired. + /// + /// # Errors + /// + /// Returns an error in any of the following conditions: + /// + /// * The slot index `i` is out-of-bounds. + /// * The slot is not in a state that can be retired (e.g., it is already retired or + /// is owned by a different thread). + /// * An [`epoch::Guard`] could not be obtained due to registration slot exhaustion. + /// * An attempt to acquire the slot after these checks races with another thread and + /// the race was lost. + pub(crate) fn retire(&self, i: usize) -> Result<(), RetireError> { + let tag = self.tags.get(i).ok_or(RetireError::OutOfBounds)?; + let current = tag.load(Ordering::Relaxed); + + // We can only perform a deletion if the generation is not in a reserved state. + if current.is_reserved() { + return Err(RetireError::SlotIsReserved { tag: current }); + } + + let guard = self + .registry + .guard() + .map_err(RetireError::GuardUnavailable)?; + + let retiring = Tag::RETIRING; + + // Even if we make this change, we can't access any data until we wait for the + // epoch to be bumped. As such, relaxed semantics are fine. + match tag.compare_exchange(current, retiring, Ordering::Relaxed, Ordering::Relaxed) { + Ok(_) => { + // Set the metadata in the mirror as well. + // + // SAFETY: The above compare-exchange ensures that we transitioned the + // authoritative state from "published" to "retired" and prevents other + // threads from attempting the same transition. + unsafe { slots::Slots::retire(self.slots(), i as u32, Lifecycle::new()) }; + guard.retire(i as u32); + Ok(()) + } + Err(_) => Err(RetireError::CouldNotClaimSlot), + } + } + + /// A somewhat crude algorithm for cooperatively performing slot scanning. + /// + /// This uses [`Freelist::scan`] to acquire a disjoint chunk of the ID space for scanning, + /// spreading out the search across multiple threads. + /// + /// If we successfully acquire a slot, we continue for the rest of the bucket returned + /// by [`Freelist::scan`] and add any available slots to the freelist (allowing other + /// threads to find them). + /// + /// Periodically, the freelist is checked to see if another thread has found an available + /// slot for us. + fn scan_acquire(&self) -> Option::Slot<'_>>> { + // This is potentially quite slow, so scan approximately `1 / RETRY_LIMIT` of the + // writable range. The outer retry loop provides broader coverage. + let mut remaining = self.unfrozen.value().div_ceil(RETRY_LIMIT); + let mut chunks_since_freelist_check = 0; + let mut acquired: Option::Slot<'_>>> = None; + + while remaining != 0 { + let chunk = self.freelist.scan(); + remaining = remaining.saturating_sub(chunk.len()); + + for slot in chunk { + #[expect( + clippy::expect_used, + reason = "this is a serious bug with the freelist" + )] + let tag = self + .tags + .get(slot.into_usize()) + .expect("freelist scan should not give out invalid IDs"); + + // If this slot is available and we haven't claimed a slot yet, try to + // claim it. Otherwise, continue with the scan to partially repopulate the + // freelist for other threads. + if tag.load(Ordering::Relaxed) == Tag::AVAILABLE { + if acquired.is_none() { + // SAFETY: We're guaranteed that `tag` belongs to `slot`. + acquired = unsafe { self.try_acquire(tag, slot) }; + } else { + self.freelist.push(slot); + } + } + } + + if acquired.is_some() { + return acquired; + } + + chunks_since_freelist_check += 1; + if chunks_since_freelist_check == 4 { + if let Some(id) = self.freelist.pop_recycled() + && let Some(slot) = self.slot(id) + { + return Some(slot); + } + chunks_since_freelist_check = 0; + } + } + None + } + + /// Attempt to directly acquire a [`Slot`] to id `i`. + /// + /// Returns `None` if `i` is not within [`Self::id_limit`] or if the slot is not currently + /// acquirable. + pub(crate) fn slot(&self, i: u32) -> Option::Slot<'_>>> { + let tag = &self.tags.get(i.into_usize())?; + + // SAFETY: We've guaranteed that `tag` belongs to `slot`. + unsafe { self.try_acquire(tag, i) } + } + + /// Try to acquire `slot` with the associated `tag`. + /// + /// # Safety + /// + /// Caller asserts that `tag` was obtained from `self.tags[slot]`. This is meant as + /// a performance optimization where `tag` is first queried for potential availability. + unsafe fn try_acquire<'a>( + &'a self, + tag: &'a AtomicTag, + slot: u32, + ) -> Option::Slot<'a>>> { + if tag.load(Ordering::Relaxed) != Tag::AVAILABLE { + return None; + } + + match tag.compare_exchange( + Tag::AVAILABLE, + Tag::OWNED, + Ordering::Acquire, + Ordering::Relaxed, + ) { + Ok(_) => { + // SAFETY: The above compare-exchange ensures that this slot was previously + // "available" and prevents other threads from trying to acquire this slot. + // The acquire ordering synchronizes with the release transition to + // "available", making slot reclamation or abort work visible before + // `Slots::acquire`. + // + // The `Slot` data structure ensures that exactly one of the terminal methods + // for `slot::Slot` is called. + let data = unsafe { slots::Slots::acquire(self.slots(), slot, Lifecycle::new()) }; + + Some(Slot { + tag, + data: ManuallyDrop::new(data), + slot, + }) + } + Err(_) => None, + } + } + + /// Return whether or not it is probably okay to read from the slot `i`. + /// + /// This check is approximate and non-synchronizing. A full check requires the + /// slot's specific reader. + /// + /// Returns `None` if `i` is not within [`Self::id_limit`]. + pub(crate) fn can_read_approximate(&self, i: usize) -> Option { + self.tags + .get(i) + .map(|tag| tag.load(Ordering::Relaxed).can_read()) + } + + #[cfg(test)] + fn writable(&self) -> std::ops::Range { + 0..self.unfrozen.value() as u32 + } +} + +/// Errors occurring during [`Store::new`]. +#[derive(Debug, Error)] +#[error(transparent)] +pub(crate) struct StoreError(StoreErrorInner); + +impl StoreError { + fn too_many_entries(capacity: Capacity, frozen: u32) -> Self { + Self(StoreErrorInner::TooManyEntries { + entries: capacity.value(), + frozen, + }) + } + + fn too_many_neighbors(neighbors: MaxDegree) -> Self { + Self(StoreErrorInner::TooManyNeighbors { + neighbors: neighbors.value(), + }) + } + + #[track_caller] + fn slots(err: E) -> Self + where + E: std::error::Error + Send + Sync + 'static, + { + Self(StoreErrorInner::SlotsError(ANNError::new(err))) + } + + fn invalid_construction(got: IdLimit, expected: IdLimit) -> Self { + Self(StoreErrorInner::InvalidConstruction { got, expected }) + } +} + +impl From for StoreError { + fn from(err: BufferError) -> Self { + Self(err.into()) + } +} + +impl From for StoreError { + fn from(err: NeighborsError) -> Self { + Self(err.into()) + } +} + +diskann::convert_error!(StoreError); + +#[derive(Debug, Error)] +enum StoreErrorInner { + #[error( + "total points ({} + {} frozen) must not exceed `u32::MAX`", + entries, + frozen + )] + TooManyEntries { entries: usize, frozen: u32 }, + #[error("number of neighbors ({}) may not exceed `u32::MAX`", neighbors)] + TooManyNeighbors { neighbors: usize }, + #[error(transparent)] + BufferError(#[from] BufferError), + #[error(transparent)] + NeighborsError(#[from] NeighborsError), + #[error("error creating slots")] + SlotsError(ANNError), + #[error("requested {} but the slots returned {}", expected, got)] + InvalidConstruction { got: IdLimit, expected: IdLimit }, +} + +/// Error conditions for [`Store::retire`]. +#[derive(Debug, Error)] +pub(crate) enum RetireError { + /// Slot index was out-of-bounds. + #[error("index out of bounds")] + OutOfBounds, + /// The slot cannot be retired because it is in a reserved state. + #[error("slot is reserved: {}", tag)] + SlotIsReserved { tag: Tag }, + /// An [`epoch::Guard`] could not be acquired. + #[error(transparent)] + GuardUnavailable(epoch::Unavailable), + /// Another thread won the retirement race. + #[error("could not claim slot")] + CouldNotClaimSlot, +} + +diskann::convert_error!(RetireError); + +/// A writable buffer into the data managed by a [`Store`], obtained from [`Store::acquire`]. +/// +/// This is the only safe way to interact with a [`slots::Slot`] since this ensures that one +/// of the terminal methods is called. Dropping a [`Slot`] without calling [`Slot::publish`] +/// or [`Slot::freeze`] automatically invokes [`slots::Slot::abort`]. +#[derive(Debug)] +pub(crate) struct Slot<'a, S> +where + S: slots::Slot, +{ + tag: &'a AtomicTag, + data: ManuallyDrop, + slot: u32, +} + +impl<'a, S> Slot<'a, S> +where + S: slots::Slot, +{ + /// View the raw inner slot. + pub(crate) fn data(&mut self) -> &mut S { + &mut self.data + } + + /// Return the slot associated with this write. + pub(crate) fn slot(&self) -> u32 { + self.slot + } + + pub(crate) fn freeze(self) { + // Suppress normal `Drop`. + let mut me = ManuallyDrop::new(self); + + // Freeze the inner slot. + slots::Slot::freeze( + // SAFETY: The `ManuallyDrop` `data` is not used after this call. + unsafe { ManuallyDrop::take(&mut me.data) }, + Lifecycle::new(), + ); + + // Update the authoritative store. + me.tag.store(Tag::FROZEN, Ordering::Release); + } + + /// Consume the slot and publish the written data for all readers. + /// + /// Return the internal slot ID. + pub(crate) fn publish(self) -> u32 { + let id = self.slot(); + + // Suppress normal `Drop`. + let mut me = ManuallyDrop::new(self); + + // Publish the inner slot. + slots::Slot::publish( + // SAFETY: The `ManuallyDrop` `data` is not used after this call. + unsafe { ManuallyDrop::take(&mut me.data) }, + Lifecycle::new(), + ); + + // Update the authoritative store. + me.tag.store(Tag::PUBLISHED, Ordering::Release); + id + } +} + +impl Drop for Slot<'_, S> +where + S: slots::Slot, +{ + fn drop(&mut self) { + slots::Slot::abort( + // SAFETY: The `ManuallyDrop` `data` is not used after this call. + unsafe { ManuallyDrop::take(&mut self.data) }, + Lifecycle::new(), + ); + self.tag.store(Tag::AVAILABLE, Ordering::Release); + } +} + +/////////// +// Tests // +/////////// + +/// These tests are basic functionality tests for the store. +/// +/// Longer running concurrency tests are in the integration test suite. +#[cfg(test)] +mod tests { + use super::{checked::Checked, *}; + + use std::assert_matches; + + /// A faulty config for [`Checked`] that doesn't respect the [`IdLimit`]. + #[derive(Debug)] + struct FaultyConfig; + + impl slots::SlotsConfig for FaultyConfig { + type Slots = Checked; + type Error = diskann::error::Infallible; + + fn build(self, id_limit: IdLimit) -> Result { + let faulty = id_limit.value().checked_sub(1).unwrap_or(1); + Ok(Checked::new(IdLimit::new(faulty))) + } + } + + // Build a store with `entries` writable slots of `entry_bytes` each, backed by `frozen` + // zeroed frozen points. The frozen points occupy the highest slot indices. + fn store(entries: usize, frozen: u32) -> Result, StoreError> { + let config = + Config::__exhaustive(NonZeroUsize::new(10).unwrap(), NonZeroU32::new(16).unwrap()); + + let layout = Layout::new(Capacity::new(entries), MaxDegree::new(0), frozen); + let store = Store::new(layout, config, Checked::config())?; + assert_eq!(store.frozen().len(), frozen.into_usize()); + + for (i, id) in store.frozen().enumerate() { + let mut slot = store.slot(id).unwrap(); + slot.data().set(i as u64); + slot.freeze(); + } + + Ok(store) + } + + fn reader(store: &Store) -> checked::Reader<'_> { + Checked::reader(store).unwrap() + } + + //------------------------// + // Constructor validation // + //------------------------// + + #[test] + fn new_rejects_total_slot_overflow() { + // `entries` alone fits in u32, but `entries + frozen` overflows it. + let err = Store::new( + Layout::new(Capacity::new(u32::MAX as usize), MaxDegree::new(0), 1), + Config::default(), + Checked::config(), + ) + .unwrap_err(); + assert_matches!(err.0, StoreErrorInner::TooManyEntries { .. }); + } + + #[test] + fn new_rejects_too_many_neighbors() { + let err = Store::new( + Layout::new( + Capacity::new(4), + MaxDegree::new(u32::MAX.into_usize() + 1), + 0, + ), + Config::default(), + Checked::config(), + ) + .unwrap_err(); + assert_matches!(err.0, StoreErrorInner::TooManyNeighbors { .. }); + } + + #[test] + fn new_rejects_faulty_slots() { + let err = Store::new( + Layout::new(Capacity::new(4), MaxDegree::new(10), 0), + Config::default(), + FaultyConfig, + ) + .unwrap_err(); + assert_matches!(err.0, StoreErrorInner::InvalidConstruction { .. }); + } + + //--------// + // Layout // + //--------// + + #[test] + fn frozen_range_follows_writable_slots() { + let s = store(4, 2).unwrap(); + + // Writable slots are [0, 4); frozen points occupy [4, 6). + assert_eq!(s.frozen(), 4..6); + + let reader = reader(&s); + for i in 0u32..4 { + assert!(!s.can_read_approximate(i.into_usize()).unwrap()); + assert!(reader.read(i).is_none()); + } + + assert!(s.can_read_approximate(4).unwrap()); + assert_eq!(reader.read(4).unwrap().get(), 0); + + assert!(s.can_read_approximate(5).unwrap()); + assert_eq!(reader.read(5).unwrap().get(), 1); + + assert!(s.can_read_approximate(6).is_none()); + assert!(reader.read(6).is_none()); + } + + /////////////// + // Lifecycle // + /////////////// + + #[test] + fn acquire_write_publish_read_roundtrip() { + let s = store(4, 1).unwrap(); + + let reader = reader(&s); + + let idx = { + let mut slot = s.acquire().expect("a fresh store has free slots"); + let idx = slot.slot(); + slot.data().set(10); + + // Before the slot is dropped - we should not be able to read it. + assert!(reader.read(idx).is_none()); + assert!(!s.can_read_approximate(idx.into_usize()).unwrap()); + slot.publish(); + idx + }; + + assert_eq!(reader.read(idx).unwrap().get(), 10,); + assert!(s.can_read_approximate(idx.into_usize()).unwrap()); + } + + #[test] + fn unpublished_slots_are_immediately_available() { + let s = store(4, 1).unwrap(); + + let reader = reader(&s); + + let idx = { + let mut slot = s.acquire().expect("a fresh store has free slots"); + let idx = slot.slot(); + slot.data().set(100); + + // Before the slot is dropped - we should not be able to read it. + assert!(reader.read(idx).is_none()); + assert!(!s.can_read_approximate(idx.into_usize()).unwrap()); + + // NOTE: We do not explicitly publish the slot. + idx + }; + + assert!(reader.read(idx).is_none()); + assert!(!s.can_read_approximate(idx.into_usize()).unwrap()); + } + + #[test] + fn acquire_exhausts_then_reports_none() { + let s = store(2, 1).unwrap(); + // Hold the guards so the slots stay owned. + let _a = s.acquire().expect("first writable slot"); + let _b = s.acquire().expect("second writable slot"); + assert!( + s.acquire().is_none(), + "all writable slots are owned, so acquire must fail" + ); + } + + //--------// + // Retire // + //--------// + + #[test] + fn retire_out_of_bounds() { + let s = store(4, 1).unwrap(); + assert!(matches!(s.retire(999), Err(RetireError::OutOfBounds))); + } + + #[test] + fn retire_rejects_reserved_slots() { + let s = store(4, 1).unwrap(); + // An untouched writable slot is AVAILABLE, which is a reserved state. + assert!(matches!( + s.retire(0), + Err(RetireError::SlotIsReserved { .. }) + )); + // A frozen slot is likewise reserved. + let frozen = s.frozen().start as usize; + assert!(matches!( + s.retire(frozen), + Err(RetireError::SlotIsReserved { .. }) + )); + // An owned slot is not retirable. + let slot = s.acquire().unwrap(); + assert!(matches!( + s.retire(slot.slot() as usize), + Err(RetireError::SlotIsReserved { .. }) + )); + } + + #[test] + fn retire_published_slot_then_unreadable() { + let s = store(4, 1).unwrap(); + + let idx = { + let mut slot = s.acquire().unwrap(); + slot.data().set(101); + slot.publish() + }; + + assert!(s.retire(idx.into_usize()).is_ok()); + + // A reader opened after retirement must not observe the retired slot. + let reader = reader(&s); + assert_matches!(reader.read(idx), None); + + // The slot can also not be retired again. + assert!(matches!( + s.retire(idx.into_usize()), + Err(RetireError::SlotIsReserved { .. }) + )); + } + + //---------// + // Recycle // + //---------// + + #[test] + fn test_recycling() { + let entries = if cfg!(miri) { 16 } else { 2048 }; + + let s = store(entries, 2).unwrap(); + + assert_eq!(s.writable().len(), entries); + + // Claim all slots. + let mut count = 0; + while let Some(mut slot) = s.acquire() { + slot.data().set(count as u64); + slot.publish(); + count += 1; + } + + assert_eq!(count, entries); + + { + let reader = reader(&s); + for i in 0..entries { + assert_eq!(reader.read(i as u32).unwrap().get(), i as u64); + } + } + + // Now that all slots are claimed - retire all slots. + for i in s.writable() { + s.retire(i.into_usize()).unwrap(); + } + + // Verify that we can claim all slots again. + let mut count = 0; + while let Some(mut slot) = s.acquire() { + slot.data().set(count as u64); + slot.publish(); + count += 1; + } + + assert_eq!(count, entries); + } +} diff --git a/diskann-inmem/src/store/slots.rs b/diskann-inmem/src/store/slots.rs new file mode 100644 index 0000000000..f011c8cec8 --- /dev/null +++ b/diskann-inmem/src/store/slots.rs @@ -0,0 +1,159 @@ +/* + * Copyright (c) Microsoft Corporation. + * Licensed under the MIT license. + */ + +//! # EBR lifecycle hooks for [`super::Store`] +//! +//! Please read this section carefully - the protocol is not difficult, but it *is* subtle. +//! +//! The transitions are a simplified version of the protocol described in [`crate::tag`] +//! that storage slots need to implement to be compatible. A state diagram is shown below: +//! +//! ```text +//! +--------------- `reclaim` ----------------+ +//! | | +//! V | +//! +-----------+ +----------+ +//! | Available |<---+ | Retiring | +//! +-----------+ | +----------+ +//! | | ^ +//! | | | +//! | `abort` `retire` +//! `acquire` | | +//! | | | +//! | +----------+ +-----------+ +//! +----->| Slot<'_> |---- `publish` --->| Published | +//! +----------+ +-----------+ +//! | +//! `freeze` +//! | +//! +-----------+ +//! | +//! V +//! +--------+ +//! | Frozen | +//! +--------+ +//! ``` +//! +//! ## Readable States +//! +//! * `published`: **New** references to slots may be given out in the "published" state. It is +//! possible for a transition to go from "published" to "retiring" while references are lent +//! out. This is fine as long as the lifetime of these references is bounded by a +//! [`crate::epoch::Guard`]. Using [`super::Store::guard`] will provide such a guard. +//! +//! * `frozen`: Since "frozen" is a terminal state, it is safe to give out references to +//! frozen slots. +//! +//! ## Writable States +//! +//! * [`Slot`]: Slots are a little spooky. Slots can assume that a [`Slot`] for an index +//! `i` is exclusive for its duration. This means that [`Slot`] implementations can lend +//! out mutable references to its contents (for example, +//! [`super::invasive::Slot::as_mut_slice`]). +//! +//! Code in [`super`] is very careful to maintain this invariant and all users of [`Slot`] +//! must carefully maintain this as well. +//! +//! * `reclaim`: On a call to [`Slots::reclaim`], implementations may assume exclusive access +//! to the indicated slot for the duration of the function call. +//! +//! ## Contracts +//! +//! Users of [`Slots`] must ensure that the lifecycle shown above is strictly observed. +//! Furthermore, for [`Slot`]s, exactly one of the terminal methods **must** be called. +//! +//! State transitions are driven by the authoritative [`super::Store`]. Before invoking a +//! transition, the store ensures the slot is not externally available in its previous +//! state. Further, the store commits the destination state only after the lifecycle API call +//! completes. + +use std::fmt::Debug; + +use crate::num::IdLimit; + +use super::Lifecycle; + +/// A configuration for a [`Slots`]. +pub(crate) trait SlotsConfig: Debug { + /// The type of the resulting [`Slots`]. + type Slots: Slots; + + /// Construction errors. + type Error: std::error::Error + Send + Sync + 'static; + + /// Build the associated [`Slots`] from self with the [`IdLimit`]. + fn build(self, id_limit: IdLimit) -> Result; +} + +/// A lifecycle backend for [`super::Store`]'s EBR scheme. +/// +/// See the [module level documentation](self) for details. +pub(crate) trait Slots: Debug + 'static { + /// The writable [`Slot`]. + type Slot<'a>: Slot; + + /// Return the exclusive upper bound for indices provided to this API. + /// + /// Callers should ensure that indices are in the range `[0..self.id_limit())`. + fn id_limit(&self) -> IdLimit; + + /// Immediately transition slot `i` from the "available" state to the "slot" state. + /// + /// Implementations may panic when `i` is out-of-bounds, but must not rely on + /// `i < Self::id_limit` for memory safety. + /// + /// # Safety + /// + /// Callers must ensure **all** of the following: + /// + /// 1. The slot is in the implicit "available" state according to the [module docs](self). + /// + /// 2. Access to slot `i` is exclusive before invoking this method and that exclusivity + /// is maintained until the returned [`Slot`] is consumed by a terminal method. + /// + /// 3. Exactly one of the [`Slot`] terminal methods is called. The [`Slot`] **may not** + /// be dropped or forgotten without one of these methods being called. + unsafe fn acquire(&self, i: u32, _: Lifecycle) -> Self::Slot<'_>; + + /// Transition slot `i` from the "published" state to the "retiring" state. + /// + /// Implementations may panic when `i` is out-of-bounds, but must not rely on + /// `i < Self::id_limit` for memory safety. + /// + /// # Safety + /// + /// The slot is in the implicit "published" state. + unsafe fn retire(&self, i: u32, _: Lifecycle); + + /// Transition slot `i` from the "retiring" state to the "available" state. + /// + /// Implementations may panic when `i` is out-of-bounds, but must not rely on + /// `i < Self::id_limit` for memory safety. + /// + /// # Safety + /// + /// Callers must ensure **all** of the following: + /// + /// 1. The slot is in the implicit "retiring" state. + /// + /// 2. All [`crate::epoch::Guard`]s for this [`Slots`] that could have obtained a + /// reference while this slot was in the "published" state have been dropped. + unsafe fn reclaim(&self, i: u32, _: Lifecycle); +} + +/// A writable slot for [`Slots`]. +/// +/// [`Slot`]s may assume that they have exclusive ownership of their slots for their duration +/// in accordance with [`Slots::acquire`]. +pub(crate) trait Slot: Debug { + /// Mark this slot as readable, transition it to the "published" state. + fn publish(self, _: Lifecycle); + + /// Mark this slot as "frozen". + fn freeze(self, _: Lifecycle); + + /// Abort any action, returning the slot to "available". + fn abort(self, _: Lifecycle); +} diff --git a/diskann-inmem/src/tag.rs b/diskann-inmem/src/tag.rs index 4840583fe0..27240bbe34 100644 --- a/diskann-inmem/src/tag.rs +++ b/diskann-inmem/src/tag.rs @@ -16,6 +16,8 @@ use std::sync::atomic::{AtomicU8, Ordering}; +use crate::num::Bytes; + /// A tag for controlling concurrent access to data. /// /// Tag updates and reads should use [`AtomicTag`]. @@ -169,6 +171,9 @@ impl std::fmt::Display for Tag { pub(crate) struct AtomicTag(AtomicU8); impl AtomicTag { + /// The size of an [`AtomicTag`]. + pub(crate) const SIZE: Bytes = Bytes::size_of::(); + /// Construct a new [`AtomicTag`] initialized to `tag`. pub(crate) const fn new(tag: Tag) -> Self { Self(AtomicU8::new(tag.value()))