diff --git a/Cargo.lock b/Cargo.lock
index c84f2ee..245ea7d 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3966,7 +3966,9 @@ dependencies = [
"lance-linalg",
"lance-namespace",
"lance-table",
+ "log",
"nom 7.1.3",
+ "object_store 0.14.2",
"serde",
"serde_json",
"snafu 0.8.9",
diff --git a/README.md b/README.md
index 6cf74df..bd64b03 100644
--- a/README.md
+++ b/README.md
@@ -36,51 +36,21 @@ lance-csr is built for single-machine execution. It aims to:
- **Push the limits of DataFusion for graph traversal.** We want to find out how far DataFusion's execution engine
can go in powering multi-hop graph traversal, an area with little open-source work so far.
-### Out of scope: distributed execution
-
-Graph queries across multiple machines are **explicitly out of scope** for this repository, to keep it free of
-complexity that works against the goals above. **Issues and pull requests that target distributed execution will
-be closed as out of scope.**
-
-If your use case involves distributed graph query processing or graph algorithms, that is exactly the kind of
-engine we're building at HDC Labs. [Reach out to us](https://hyperdimensionalcomputing.ai/contact/) to chat more.
-
-## Why a CSR
-
-[lance-graph](https://github.com/lance-format/lance-graph) runs each relationship hop as a join between the edge and node tables. A
-multi-hop query repeats that join at every hop, building a hash table and materializing every intermediate path each time.
-[Compressed sparse row (CSR)](https://en.wikipedia.org/wiki/Sparse_matrix#Compressed_sparse_row_(CSR,_CRS_or_Yale_format))
-is the standard compact layout for a sparse matrix, and so for a graph's adjacency: an offsets array indexed by node
-points into one array of neighbor ids, so each node's neighbors sit next to each other. Expanding a hop becomes an
-array lookup per node instead of a join. lance-csr builds this index from the relationship tables and runs it inside
-DataFusion as extension operators, falling back to a join for any hop the CSR can't express.
+Distributed execution is **out of scope**, and issues and pull requests that target it will be closed. If your use
+case needs distributed graph queries or graph algorithms, that's the kind of engine we're building at HDC Labs:
+[reach out to us](https://hyperdimensionalcomputing.ai/contact/).
## What this fork adds
-- **CSR traversal** (`use_csr=True`) for single hops and variable-length paths, directed or undirected, updated
- incrementally when the underlying Lance datasets change.
-- **A Lance-native engine** that keeps each table either on disk (read lazily from a Lance snapshot) or in memory,
- and `refresh()`es to new dataset versions.
-- **Smarter planning**: patterns start at their most selective node, and DataFusion gets column statistics to size
- its joins.
-- **Much wider Cypher**. The Graph Data Council (GDC), formerly known as the Linked Data Benchmark Council (LDBC),
- maintains the official Interactive complex queries. lance-csr runs them (IC14 without `reduce()`, which
- openCypher leaves out), with `OPTIONAL MATCH`, `EXISTS` and pattern predicates, chained `WITH`, `UNWIND`,
- list and pattern comprehensions, maps, relationship alternation, `shortestPath`, `$parameters` and more,
- following openCypher's
- [standardisation scope](https://github.com/opencypher/openCypher/blob/master/docs/standardisation-scope.adoc).
-- **Correctness fixes** to behavior inherited from [lance-graph](https://github.com/lance-format/lance-graph) (for example,
- undirected patterns dropped incoming edges), each with a regression test.
-- **Current dependencies** (Lance 13, DataFusion 54, Arrow 58, PyO3 0.28), with Delta Lake and Unity Catalog support
- removed.
-
-## Repository layout
-
-- `crates/lance-csr` – the Cypher-capable query engine implemented in Rust
-- `python/` – PyO3 bindings and Python packages:
- - `lance_csr` – thin wrapper around the Rust query engine
-
-See [`docs/project_structure.md`](docs/project_structure.md) for the full workspace layout.
+- **CSR traversal** (`use_csr=True`): relationship hops and variable-length paths become array lookups instead of
+ joins, updated incrementally as the Lance datasets change. See [the CSR index](https://lance-csr.hdclabs.ai/csr/).
+- **A Lance-native engine** that reads each table lazily from a pinned Lance snapshot or holds it in memory, and
+ `checkout_latest()`es to new versions.
+- **Much wider Cypher**, enough to run the Graph Data Council's Interactive complex queries: `OPTIONAL MATCH`,
+ pattern predicates, chained `WITH`, `UNWIND`, list and pattern comprehensions, `shortestPath`, `$parameters`
+ and more.
+- **Correctness fixes** to behaviour inherited from lance-graph, each with a regression test, and current
+ dependencies (Lance 13, DataFusion 54, Arrow 58).
## Install
@@ -88,48 +58,11 @@ See [`docs/project_structure.md`](docs/project_structure.md) for the full worksp
uv add lance-csr
```
-Prebuilt wheels cover Linux x86-64 and native Apple Silicon (CPython 3.12+); no Rust toolchain is needed.
-See [`python/README.md`](python/README.md) for details and usage.
-
-To use the engine from Rust, add the crate with `cargo add lance-csr` and see
-[`crates/lance-csr/README.md`](crates/lance-csr/README.md) and the
-[Rust guide](https://lance-csr.hdclabs.ai/rust/).
-
-## Build from source
-
-You need:
-
-- Rust, through [rustup](https://rustup.rs): `rust-toolchain.toml` pins the version, and rustup installs it on the
- first build.
-- `protoc`, which Lance's build needs: `brew install protobuf` on macOS, or
- `sudo apt-get install -y protobuf-compiler` on Debian and Ubuntu.
-- Python 3.12 or newer, and [`uv`](https://docs.astral.sh/uv/).
-
-Rust crates, from the repository root:
-
-```bash
-cargo test --workspace
-cargo clippy --workspace --all-targets -- -D warnings
-cargo fmt --all --check
-```
-
-Python package, from `python/`. `uv sync --locked` installs the exact dependency versions in `uv.lock`,
-including the development tools (maturin, pytest, ruff, pyright), into `python/.venv`:
+Prebuilt wheels cover Linux x86-64 and native Apple Silicon (CPython 3.12+); no Rust toolchain is needed. From
+Rust, run `cargo add lance-csr` and see the [Rust guide](https://lance-csr.hdclabs.ai/rust/). To build from
+source, see [`python/DEVELOPMENT.md`](python/DEVELOPMENT.md).
-```bash
-cd python
-uv sync --locked --no-install-project # locked dependencies and dev tools
-uv run --no-sync maturin develop # debug build of the extension into .venv
-uv run --no-sync pytest python/tests # run the tests
-```
-
-`--no-install-project` leaves the package itself to `maturin develop`, which makes a debug build that compiles
-quickly; uv would otherwise build it in release mode. `--no-sync` stops `uv run` from re-syncing, which would remove
-that build. Re-run `maturin develop` after changing Rust code. `make install`, `make test` and `make lint` wrap
-these steps. For timing queries, build with `maturin develop --release`: release builds use thin LTO and take
-several minutes. See [`python/DEVELOPMENT.md`](python/DEVELOPMENT.md) for more.
-
-## Python example: Cypher query
+## Example
```python
import pyarrow as pa
@@ -155,167 +88,28 @@ result = query.execute({"Person": people})
print(result.to_pydict()) # {'name': ['Bob', 'David'], 'age': [34, 42]}
```
-## Python example: Direct SQL query
-
-For data analytics workflows where you prefer standard SQL, use `SqlQuery` or `SqlEngine`. No `GraphConfig` is needed:
-
-```python
-import pyarrow as pa
-from lance_csr import SqlQuery, SqlEngine
-
-person = pa.table({
- "id": [1, 2, 3],
- "name": ["Alice", "Bob", "Carol"],
- "age": [28, 34, 29],
-})
-
-# One-off query
-result = SqlQuery(
- "SELECT name, age FROM person WHERE age > 30"
-).execute({"person": person})
-print(result.to_pydict()) # {'name': ['Bob'], 'age': [34]}
-
-# Multi-query with cached context
-engine = SqlEngine({"person": person})
-r1 = engine.execute("SELECT COUNT(*) AS cnt FROM person")
-r2 = engine.execute("SELECT name FROM person ORDER BY age DESC LIMIT 2")
-```
+More in the [getting started guide](https://lance-csr.hdclabs.ai/getting-started/) and the
+[examples](examples/README.md). For plain SQL over the same datasets, see [SQL queries](https://lance-csr.hdclabs.ai/sql/).
## Benchmarks
-The benchmark runners and raw results live in two sibling repositories, next to the other engines' runners, so
-every number below can be reproduced from there. These are independent, unaudited measurements, not GDC Benchmark
-Results. lance-csr runs with the CSR (`use_csr=True`) in two modes: *in memory* holds every table as Arrow in
-memory, and *lazy* reads Lance datasets from disk as queries need them. Neo4j runs as a server in a Docker VM; the
-other engines run embedded in the Python process. `docs/sync_benchmarks.py` regenerates the tables from the
-repositories' committed results.
-
-
-
-### GDC SNB Interactive complex queries
-
-[GDC Interactive benchmark repository](https://github.com/hyperdimensionalcomputingai/graph-benchmark-ldbc-complex) runs the
-14 complex read queries (IC1–IC14) of the GDC SNB Interactive v1 workload over the SF1 dataset, with one fixed
-parameter set per query and every result checked against a golden answer. lance-csr's
-[runner](https://github.com/hyperdimensionalcomputingai/graph-benchmark-ldbc-complex/tree/main/lance_csr) runs
-Neo4j's Cypher text for IC1–IC13 unchanged; IC14 replaces `reduce()`, which openCypher leaves out, with pattern
-comprehensions.
-
-Over the 14 queries, lance-csr's speed relative to each engine:
-
-
-
-| lance-csr mode | vs Neo4j | vs Kuzu | vs Ladybug |
-| --- | ---: | ---: | ---: |
-| In memory | 3.9× faster | 6.1× faster | 3.2× faster |
-| Lazy (Lance on disk) | 1.8× faster | 2.8× faster | 1.3× faster |
-
-
-
-Mean query latency in milliseconds (lower is better, fastest per query in bold):
-
-
-
-| Query | lance-csr 0.3.1, in memory | lance-csr 0.3.1, lazy | Neo4j 2026.09.0 | Kuzu 0.11.3 | Ladybug 0.21.2 |
-| --- | ---: | ---: | ---: | ---: | ---: |
-| IC1 | 26.23 | 32.17 | 51.39 | **13.51** | n/a |
-| IC2 | **2.74** | 26.18 | 44.73 | 24.97 | 15.56 |
-| IC3 | **59.31** | 424.82 | 894.91 | 105.21 | 100.16 |
-| IC4 | **5.94** | 13.44 | 13.05 | 29.66 | 11.83 |
-| IC5 | **81.50** | 106.39 | 1,022.62 | 747.75 | 309.02 |
-| IC6 | **24.68** | 27.14 | 231.25 | 1,114.76 | 234.21 |
-| IC7 | 20.74 | 32.28 | **7.14** | 453.74 | 370.55 |
-| IC8 | **2.04** | 3.06 | 3.00 | 73.44 | 4.95 |
-| IC9 | **41.78** | 825.02 | 802.28 | 130.96 | 62.14 |
-| IC10 | 100.21 | 136.60 | 91.68 | 74.90 | **13.69** |
-| IC11 | 3.73 | 6.05 | 11.02 | 4.71 | **2.51** |
-| IC12 | **19.77** | 26.75 | 133.87 | 92.26 | 133.84 |
-| IC13 | 1.11 | 1.45 | **0.92** | 14.16 | 47.55 |
-| IC14 | **46.80** | 47.88 | 522.56 | 1,871.19 | n/a |
-
-
-
-
-
-- Run on Apple M5, 24 GB, macOS 26.6.2, from the benchmark repository at commit [`f3e63fd`](https://github.com/hyperdimensionalcomputingai/graph-benchmark-ldbc-complex/tree/f3e63fd2b7239a1364a864458f3e7ae07f0d833f). lance-csr was measured on 2026-10-27 and the other engines on 2026-10-24, on the same machine, with the repository's `run_benchmarks.sh` settings; see its README to reproduce the runs.
-- Ratios are geometric means of the per-query ratios, over the queries both engines answered correctly.
-- Ladybug 0.21.2: IC1 returned a wrong answer; IC14 failed (RuntimeError), so they are not timed.
-
-
-
-
-
-### GDC SNB SF1, 30-query workload
-
-[GDC SF1 benchmark repository](https://github.com/hyperdimensionalcomputingai/graph-benchmark-ldbc) runs 30 custom Cypher
-queries over the GDC SNB SF1 dataset (3.18M nodes, 17.3M relationships), with every result checked against an
-expected answer. lance-csr's [runner](https://github.com/hyperdimensionalcomputingai/graph-benchmark-ldbc/tree/main/lance_csr)
-runs the same query text and Lance datasets as the [lance-graph](https://github.com/lance-format/lance-graph) runner.
-
-Over the 30 queries, lance-csr's speed relative to each engine:
-
-
-
-| lance-csr mode | vs Neo4j | vs Kuzu | vs Ladybug | vs lance-graph |
-| --- | ---: | ---: | ---: | ---: |
-| In memory | 3.6× faster | 4.1× faster | 2.6× faster | 4.6× faster |
-| Lazy (Lance on disk) | 2.1× faster | 2.3× faster | 1.5× faster | 2.6× faster |
-
-
-
-Mean query latency in milliseconds (lower is better, fastest per query in bold):
-
-
-
-| Query | lance-csr 0.3.1, in memory | lance-csr 0.3.1, lazy | Neo4j 2026.09.0 | Kuzu 0.11.3 | Ladybug 0.21.2 | lance-graph 0.5.4 |
-| --- | ---: | ---: | ---: | ---: | ---: | ---: |
-| Q1 | **0.66** | 1.13 | 1.28 | 1.73 | 1.22 | 1.30 |
-| Q2 | **0.51** | 1.33 | 3.26 | 1.32 | 1.12 | 2.34 |
-| Q3 | **0.62** | 0.77 | 0.66 | 1.03 | 0.79 | 1.86 |
-| Q4 | 0.48 | 0.75 | 2.66 | 0.85 | **0.39** | 2.98 |
-| Q5 | **0.77** | 2.23 | 2.78 | 3.44 | 4.00 | 2.04 |
-| Q6 | **0.41** | 0.71 | 2.60 | 0.66 | 0.79 | 0.73 |
-| Q7 | 0.98 | 1.05 | **0.43** | 27.73 | 25.11 | 15.68 |
-| Q8 | **0.61** | 1.75 | 7.29 | 2.72 | 3.38 | 1.14 |
-| Q9 | 0.81 | 1.32 | **0.76** | 1.73 | 1.34 | 1.89 |
-| Q10 | **0.70** | 0.91 | 3.79 | 1.55 | 1.78 | 26.69 |
-| Q11 | **1.47** | 1.89 | 10.28 | 7.96 | 4.63 | 3.08 |
-| Q12 | **0.69** | 6.80 | 2.55 | 17.16 | 7.84 | 18.68 |
-| Q13 | **0.74** | 0.99 | 6.65 | 42.66 | 5.84 | 9.62 |
-| Q14 | 0.48 | 0.65 | **0.46** | 1.56 | 0.90 | 2.91 |
-| Q15 | **0.51** | 0.70 | 2.37 | 2.38 | 1.16 | 3.02 |
-| Q16 | **0.62** | 1.06 | 1.26 | 1.78 | 1.04 | 4.33 |
-| Q17 | **0.87** | 1.37 | 3.09 | 2.62 | 1.74 | 2.30 |
-| Q18 | **0.63** | 1.07 | 3.11 | 1.51 | 1.22 | 1.76 |
-| Q19 | **0.90** | 1.33 | 4.86 | 11.66 | 3.43 | 19.93 |
-| Q20 | **2.05** | 16.45 | 421.92 | 13.25 | 14.15 | 3.13 |
-| Q21 | 0.70 | 0.79 | 0.51 | **0.44** | 0.50 | 1.46 |
-| Q22 | **0.65** | 0.88 | 2.66 | 23.34 | 7.46 | 13.41 |
-| Q23 | **0.66** | 1.18 | 3.13 | 1.19 | 0.79 | 2.42 |
-| Q24 | **0.63** | 1.00 | 0.72 | 1.23 | 0.81 | 1.75 |
-| Q25 | **0.72** | 1.13 | 2.86 | 1.45 | 1.27 | 1.47 |
-| Q26 | 0.67 | 1.02 | **0.65** | 3.37 | 1.15 | 2.93 |
-| Q27 | **0.80** | 1.24 | 2.22 | 17.96 | 3.71 | 24.84 |
-| Q28 | **1.36** | 2.25 | 2.37 | 1.51 | 1.78 | 2.50 |
-| Q29 | **0.62** | 0.97 | 2.62 | 1.02 | 0.72 | 2.96 |
-| Q30 | 43.67 | 47.50 | 1,359.98 | 146.52 | 97.77 | **34.31** |
-
-
-
-
+lance-csr is measured against other graph engines on the Graph Data Council's SNB SF1 dataset (3.18M nodes, 17.3M
+relationships). The runners, queries, expected answers and results live in two repositories, whose READMEs explain
+how to reproduce the runs:
-- Run on Apple M5, 24 GB, macOS 26.6.2, from the benchmark repository at commit [`b649bc1`](https://github.com/hyperdimensionalcomputingai/graph-benchmark-ldbc/tree/b649bc166cb5a04a542882fc070d9c4d83507c65). lance-csr was measured on 2026-10-27 and the other engines on 2026-10-24, on the same machine, with the repository's `run_benchmarks.sh` settings; see its README to reproduce the runs.
-- Ratios are geometric means of the per-query ratios, over the queries both engines answered correctly.
+- [graph-benchmark-ldbc-complex](https://github.com/hyperdimensionalcomputingai/graph-benchmark-ldbc-complex): the
+ 14 SNB Interactive complex read queries (IC1 to IC14).
+- [graph-benchmark-ldbc](https://github.com/hyperdimensionalcomputingai/graph-benchmark-ldbc): 30 custom Cypher
+ queries.
-
+These are independent, unaudited measurements, not GDC Benchmark Results.
## Contributing
Contributions within the [project's goals](#goals) are welcome, from people and from their coding agents. Every
-pull request should explain the problem it solves and why the change is needed, and back that up with evidence:
-a regression test for a bug fix, and before-and-after results from both benchmark suites for a code change.
-[`CONTRIBUTING.md`](CONTRIBUTING.md) explains what to include and how to build the benchmark harness that
-measures your branch against `main`. If you're unsure whether an idea fits, open an issue to discuss it first.
+pull request should explain the problem it solves and back it up with evidence: a regression test for a bug fix,
+and results from both benchmark suites for a code change. [`CONTRIBUTING.md`](CONTRIBUTING.md) explains what to
+include and how to measure your branch against `main`. If you're unsure whether an idea fits, open an issue first.
## Credits
diff --git a/crates/lance-csr-python/Cargo.lock b/crates/lance-csr-python/Cargo.lock
index 835fd7e..701192a 100644
--- a/crates/lance-csr-python/Cargo.lock
+++ b/crates/lance-csr-python/Cargo.lock
@@ -3850,7 +3850,9 @@ dependencies = [
"lance-linalg",
"lance-namespace",
"lance-table",
+ "log",
"nom 7.1.3",
+ "object_store 0.14.2",
"serde",
"serde_json",
"snafu 0.8.9",
diff --git a/crates/lance-csr-python/src/graph.rs b/crates/lance-csr-python/src/graph.rs
index d8af19c..d57be91 100644
--- a/crates/lance-csr-python/src/graph.rs
+++ b/crates/lance-csr-python/src/graph.rs
@@ -26,7 +26,7 @@ use datafusion::execution::context::SessionContext;
use futures::StreamExt;
use lance_csr::{
ast::{DistanceMetric as RustDistanceMetric, GraphPattern, ReadingClause},
- engine::{CypherEngine as RustCypherEngine, EngineSource, MAX_TARGET_PARTITIONS},
+ engine::{CypherEngine as RustCypherEngine, EngineOptions, EngineSource, MAX_TARGET_PARTITIONS},
CypherQuery as RustCypherQuery, ExecutionStrategy as RustExecutionStrategy,
GraphConfig as RustGraphConfig, GraphError as RustGraphError, SqlDialect as RustSqlDialect,
SqlQuery as RustSqlQuery, VectorSearch as RustVectorSearch,
@@ -1015,14 +1015,18 @@ pub struct CypherEngine {
#[pymethods]
impl CypherEngine {
#[new]
- #[pyo3(signature = (config, datasets, use_csr = false, *, target_partitions = None))]
+ #[pyo3(signature = (config, datasets, use_csr = false, *, target_partitions = None, persist_csr = false))]
fn new(
py: Python,
config: &GraphConfig,
datasets: &Bound<'_, PyDict>,
use_csr: bool,
target_partitions: Option<&Bound<'_, PyAny>>,
+ persist_csr: bool,
) -> PyResult {
+ if persist_csr && !use_csr {
+ return Err(PyValueError::new_err("persist_csr requires use_csr=True"));
+ }
let target_partitions = target_partitions
.map(|value| {
if value.is_instance_of::() || !value.is_instance_of::() {
@@ -1051,11 +1055,14 @@ impl CypherEngine {
let inner = RT
.block_on(
Some(py),
- RustCypherEngine::with_target_partitions(
+ RustCypherEngine::with_options(
config.inner.clone(),
sources,
- use_csr,
- target_partitions,
+ EngineOptions {
+ use_csr,
+ target_partitions,
+ persist_csr,
+ },
),
)?
.map_err(graph_error_to_pyerr)?;
@@ -1112,16 +1119,26 @@ impl CypherEngine {
.map_err(graph_error_to_pyerr)
}
- fn refresh(&self, py: Python) -> PyResult<()> {
- RT.block_on(Some(py), self.inner.refresh())?
+ fn checkout_latest(&self, py: Python) -> PyResult<()> {
+ RT.block_on(Some(py), self.inner.checkout_latest())?
.map_err(graph_error_to_pyerr)
}
+ /// Deprecated alias; the Python wrapper warns before calling it.
+ fn refresh(&self, py: Python) -> PyResult<()> {
+ self.checkout_latest(py)
+ }
+
fn optimize_csr(&self, py: Python) -> PyResult<()> {
RT.block_on(Some(py), self.inner.optimize_csr())?
.map_err(graph_error_to_pyerr)
}
+ fn delete_stored_csr(&self, py: Python) -> PyResult {
+ RT.block_on(Some(py), self.inner.delete_stored_csr())?
+ .map_err(graph_error_to_pyerr)
+ }
+
fn node_index_stats(&self, py: Python) -> PyResult> {
let stats = PyDict::new(py);
for (label, entry) in self.inner.node_index_stats() {
diff --git a/crates/lance-csr/Cargo.toml b/crates/lance-csr/Cargo.toml
index 731c945..c101dc8 100644
--- a/crates/lance-csr/Cargo.toml
+++ b/crates/lance-csr/Cargo.toml
@@ -13,7 +13,7 @@ categories = ["database", "data-structures", "science"]
[dependencies]
ahash = "0.8"
-arrow = { version = "58", features = ["prettyprint"] }
+arrow = { version = "58", features = ["prettyprint", "ipc_compression"] }
arrow-array = "58"
arrow-schema = "58"
async-trait = "0.1"
@@ -38,7 +38,9 @@ lance = "13.0.0"
lance-linalg = "13.0.0"
lance-namespace = "13.0.0"
lance-table = "13.0.0"
+log = "0.4"
nom = "7.1"
+object_store = "0.14"
serde = { version = "1", features = ["derive"] }
serde_json = "1"
snafu = "0.8"
diff --git a/crates/lance-csr/README.md b/crates/lance-csr/README.md
index 1b4a9c4..167ddd8 100644
--- a/crates/lance-csr/README.md
+++ b/crates/lance-csr/README.md
@@ -15,7 +15,7 @@ crate.
- Semantic validation with typed `GraphError` diagnostics
- Pluggable execution strategies (DataFusion planner by default, Lance Native placeholder)
- `CypherEngine`: register datasets once, optionally build the CSR, and serve many queries
-- Lance datasets read lazily and pinned to a version until `refresh()`
+- Lance datasets read lazily and pinned to a version until `checkout_latest()`
- Async query execution that returns Arrow `RecordBatch` results or a stream
- JSON-serializable parameter binding for reusable query templates
- Logical plan debugging via `CypherQuery::explain`
@@ -41,7 +41,7 @@ let batch = engine
```
`examples/embedded_query.rs` is a complete program that writes two Lance datasets, queries them, and picks up a
-new write with `refresh()`:
+new write with `checkout_latest()`:
```bash
cargo run -p lance-csr --example embedded_query
@@ -154,7 +154,7 @@ clauses, patterns, expressions and functions the engine runs.
- `logical_plan` – Builders for graph logical plans.
- `datafusion_planner` – DataFusion-based execution planning.
- `config` – Graph configuration types and builders.
-- `engine` – `CypherEngine`: cached snapshot, CSR adjacencies, `refresh()`.
+- `engine` – `CypherEngine`: cached snapshot, CSR adjacencies, `checkout_latest()`.
- `query` – High level `CypherQuery` API and runtime.
- `csr_index`, `csr_expand` – The CSR adjacency index and the traversal operators that use it.
- `error` – `GraphError` and result helpers.
diff --git a/crates/lance-csr/examples/embedded_query.rs b/crates/lance-csr/examples/embedded_query.rs
index 71cf107..5e7ecff 100644
--- a/crates/lance-csr/examples/embedded_query.rs
+++ b/crates/lance-csr/examples/embedded_query.rs
@@ -2,7 +2,7 @@
//!
//! Writes a small social graph as two Lance datasets in a temporary directory, opens
//! them in a CSR-backed engine, runs a two-hop query, appends an edge and picks it up
-//! with `refresh()`, runs a query under a deadline, and repeats the in-memory Arrow
+//! with `checkout_latest()`, runs a query under a deadline, and repeats the in-memory Arrow
//! examples of the home page and the getting started guide. The documentation includes
//! this file section by section (the `--8<--` markers), so keep it compiling and its printed
//! output in step.
@@ -80,7 +80,7 @@ async fn main() -> lance_csr::Result<()> {
.node_table("Person", "id")
.rel_table("KNOWS", "Person", "Person", "src_id", "dst_id")
.build()?;
- // Each dataset is pinned to the version opened here until `engine.refresh()`.
+ // Each dataset is pinned to the version opened here until `engine.checkout_latest()`.
let sources = HashMap::from([
(
"Person".to_string(),
@@ -108,10 +108,10 @@ async fn main() -> lance_csr::Result<()> {
// +------------------+
// --8<-- [end:query]
- // --8<-- [start:refresh]
- // Bob now knows Alice too. The engine doesn't see the write until refresh().
+ // --8<-- [start:checkout_latest]
+ // Bob now knows Alice too. The engine doesn't see the write until checkout_latest().
write_lance(knows(vec![2], vec![1])?, &knows_uri, WriteMode::Append).await?;
- engine.refresh().await?;
+ engine.checkout_latest().await?;
let result = engine.execute(query, params()).await?;
print_batches(&[result])?; // Alice, Carol, David
@@ -121,7 +121,7 @@ async fn main() -> lance_csr::Result<()> {
stats.edges, stats.update, stats.version
);
// 5 edges, update=delta, version=Some(2)
- // --8<-- [end:refresh]
+ // --8<-- [end:checkout_latest]
// --8<-- [start:timeout]
// One deadline for both: checks inside nested-loop joins, and a timer that stops
diff --git a/crates/lance-csr/src/csr_expand.rs b/crates/lance-csr/src/csr_expand.rs
index bddc22f..6fac611 100644
--- a/crates/lance-csr/src/csr_expand.rs
+++ b/crates/lance-csr/src/csr_expand.rs
@@ -100,9 +100,10 @@ use crate::error::{GraphError, Result};
/// sorted offsets into each source's base-plus-appended neighbours. Offsets let
/// readers skip deleted occurrences without copying per-source deletion state.
/// Identical edges remain distinct occurrences. Each neighbour carries the row its
-/// edge came from (a Lance row address, or a row index for in-memory input), so the
-/// base holds 16 bytes per edge: relationship columns can be fetched by row, and
-/// deleted edges are found by row without reading the old dataset version.
+/// edge came from (a Lance row address, or a row index for in-memory input), so
+/// relationship columns can be fetched by row, and deleted edges are found by row
+/// without reading the old dataset version. Edge rows are packed (see
+/// [`crate::csr_index::PackedU64`]), so the base holds about 10 bytes per edge, not 16.
///
/// # Design note for future changes (read before "optimizing" the delta)
///
@@ -220,14 +221,16 @@ impl Adjacency {
.map(|(k, v)| (k, Some(v)))
.chain(added_only.map(|&k| (k, None)))
{
- let base_rows = vertex.map_or(&[][..], |v| self.base.csr.edge_rows(v));
+ let base_range = vertex.map_or(0..0, |v| self.base.csr.range(v));
+ let base_len = base_range.len();
let added_rows = self.added.get(&key).map_or(&[][..], Vec::as_slice);
let prior = self.removed.get(&key).map_or(&[][..], Vec::as_slice);
let mut prior = prior.iter().copied().peekable();
let mut positions = Vec::new();
- let rows = base_rows
- .iter()
- .copied()
+ let rows = self
+ .base
+ .csr
+ .edge_rows_at(base_range)
.chain(added_rows.iter().map(|&(_, r)| r));
for (position, row) in rows.enumerate() {
if prior.peek() == Some(&position) {
@@ -239,7 +242,7 @@ impl Adjacency {
if !positions.is_empty() {
next.removed_count += positions.len() as u64;
next.removed_added_count +=
- positions.iter().filter(|&&p| p >= base_rows.len()).count() as u64;
+ positions.iter().filter(|&&p| p >= base_len).count() as u64;
let tombstones = next.removed.entry(key).or_default();
tombstones.extend(positions);
tombstones.sort_unstable();
@@ -252,17 +255,47 @@ impl Adjacency {
next
}
+ /// An adjacency with no delta over a stored base: `keys` are the source keys in
+ /// CSR position order, as [`Self::base_parts`] returns them. `None` if a key
+ /// repeats.
+ pub(crate) fn from_base(keys: &[i64], csr: CsrIndex) -> Option {
+ let mut dense = FastMap::with_capacity_and_hasher(keys.len(), Default::default());
+ dense.extend(keys.iter().enumerate().map(|(i, &k)| (k, i as u64)));
+ if dense.len() != keys.len() {
+ return None;
+ }
+ Some(Self {
+ base: Arc::new(CsrBase { dense, csr }),
+ added: FastMap::default(),
+ removed: FastMap::default(),
+ added_count: 0,
+ removed_count: 0,
+ removed_added_count: 0,
+ })
+ }
+
+ /// The base CSR: its source keys in CSR position order, and the index over them.
+ /// The delta is not included.
+ pub(crate) fn base_parts(&self) -> (Vec, &CsrIndex) {
+ let mut keys = vec![0; self.base.dense.len()];
+ for (&key, &position) in &self.base.dense {
+ keys[position as usize] = key;
+ }
+ (keys, &self.base.csr)
+ }
+
/// Call `f` for each target endpoint value reachable from `key`, duplicates included.
pub fn for_each_neighbor(&self, key: i64, f: impl FnMut(u64)) {
self.neighbors_from(key, &mut 0).for_each(f);
}
- /// Neighbours and their edge rows in the base.
+ /// Positions of `key`'s neighbours in the base.
#[inline(always)]
- fn base_slots(&self, key: i64) -> (&[u64], &[u64]) {
- self.base.dense.get(&key).map_or((&[], &[]), |&v| {
- (self.base.csr.neighbors(v), self.base.csr.edge_rows(v))
- })
+ fn base_range(&self, key: i64) -> std::ops::Range {
+ self.base
+ .dense
+ .get(&key)
+ .map_or(0..0, |&v| self.base.csr.range(v))
}
/// Resume at an occurrence offset, borrowing all neighbour and deletion storage.
@@ -286,17 +319,27 @@ impl Adjacency {
key: i64,
offset: &'a mut usize,
) -> impl Iterator- > + 'a {
- let (base, base_rows) = self.base_slots(key);
+ let base = self.base_range(key);
let added = self.added.get(&key).map_or(&[][..], Vec::as_slice);
let removed = self.removed.get(&key).map_or(&[][..], Vec::as_slice);
let mut removed = &removed[removed.partition_point(|&position| position < *offset)..];
let added_offset = offset.saturating_sub(base.len());
- let base_start = (*offset).min(base.len());
- let base_rows = &base_rows[base_start..];
- base[base_start..]
+ let rest = base.start + (*offset).min(base.len())..base.end;
+ let csr = &self.base.csr;
+ // Never advanced without `ROWS`, so plain hops don't unpack edge rows.
+ let mut base_rows = csr.edge_rows_at(if ROWS { rest.clone() } else { 0..0 });
+ csr.neighbors_at(rest)
.iter()
- .enumerate()
- .map(move |(i, &n)| (n, if ROWS { base_rows[i] } else { 0 }))
+ .map(move |&n| {
+ (
+ n,
+ if ROWS {
+ base_rows.next().unwrap_or(0)
+ } else {
+ 0
+ },
+ )
+ })
.chain(
added
.get(added_offset..)
@@ -461,15 +504,42 @@ pub async fn edge_changes(
src: &str,
dst: &str,
) -> Result> {
- let endpoint_ids = |ds: &Dataset| -> Option<[(i32, DataType); 2]> {
- let field = |name: &str| ds.schema().field(name).map(|f| (f.id, f.data_type()));
- Some([field(src)?, field(dst)?])
+ let Some(old_fields) = endpoint_fields(old, src, dst) else {
+ return Ok(None);
+ };
+ let old_frags: Vec = old
+ .get_fragments()
+ .into_iter()
+ .map(|f| f.metadata().clone())
+ .collect();
+ edge_changes_since(&old_frags, &old_fields, new, src, dst).await
+}
+
+/// Lance field id and type of a relationship table's two endpoint columns.
+pub(crate) type EndpointFields = [(i32, String); 2];
+
+pub(crate) fn endpoint_fields(ds: &Dataset, src: &str, dst: &str) -> Option {
+ let field = |name: &str| {
+ ds.schema()
+ .field(name)
+ .map(|f| (f.id, f.data_type().to_string()))
};
- let old_ids = endpoint_ids(old);
- if old_ids.is_none() || old_ids != endpoint_ids(new) {
+ Some([field(src)?, field(dst)?])
+}
+
+/// [`edge_changes`] from a version known only by its fragments and endpoint fields,
+/// as a stored CSR records them.
+pub(crate) async fn edge_changes_since(
+ old_fragments: &[Fragment],
+ old_fields: &EndpointFields,
+ new: &Dataset,
+ src: &str,
+ dst: &str,
+) -> Result> {
+ if endpoint_fields(new, src, dst).as_ref() != Some(old_fields) {
return Ok(None);
}
- let fields: Vec = old_ids.unwrap().iter().map(|(id, _)| *id).collect();
+ let fields: Vec = old_fields.iter().map(|(id, _)| *id).collect();
let touches = |f: &[i32]| f.iter().any(|id| fields.contains(id));
// The parts of a fragment that determine its endpoint values.
let endpoint_data = |frag: &Fragment| {
@@ -482,11 +552,8 @@ pub async fn edge_changes(
format!("{:?}{:?}", files, overlays)
};
- let old_frags: HashMap = old
- .get_fragments()
- .into_iter()
- .map(|f| (f.id() as u64, f.metadata().clone()))
- .collect();
+ let old_frags: HashMap =
+ old_fragments.iter().map(|f| (f.id, f.clone())).collect();
let new_frags: HashMap = new
.get_fragments()
.into_iter()
diff --git a/crates/lance-csr/src/csr_index.rs b/crates/lance-csr/src/csr_index.rs
index 6c57f77..1b48e1a 100644
--- a/crates/lance-csr/src/csr_index.rs
+++ b/crates/lance-csr/src/csr_index.rs
@@ -17,8 +17,226 @@
//!
//! For vertex `v`, its neighbors are `neighbors[offsets[v]..offsets[v+1]]`, and
//! `edge_rows` at the same positions holds the row each edge came from.
+//!
+//! Edge rows are stored packed ([`PackedU64`]): in blocks of 128 values, each held as
+//! its offset from the block's minimum in as few bits as the block needs. They are
+//! Lance row addresses, so a single width for the whole array would waste most of the
+//! saving; per-block widths keep any position directly readable, which traversal
+//! needs to resume part-way through a list. Neighbours stay a plain array: every hop
+//! reads them, and packing them slowed lookup-heavy queries (custom Q30 by about 22%)
+//! for a smaller saving than edge rows give.
use std::collections::HashMap;
+use std::ops::Range;
+
+use arrow::buffer::ScalarBuffer;
+
+/// Values per block of a [`PackedU64`].
+const BLOCK: usize = 128;
+
+/// A `u64` array packed block by block, with direct access to any position.
+///
+/// Each block of [`BLOCK`] values stores its minimum (`bases`) and its start bit
+/// and width (`meta`: start bit << 7 | width); the values are bit-packed offsets
+/// from the minimum, back to back in `words`. A block whose values are all equal
+/// takes no bits at all.
+#[derive(Debug, Clone)]
+pub struct PackedU64 {
+ len: usize,
+ bases: ScalarBuffer,
+ meta: ScalarBuffer,
+ /// One spare word at the end, so reading two words never runs past it.
+ words: ScalarBuffer,
+}
+
+impl PackedU64 {
+ pub fn pack(values: &[u64]) -> Self {
+ let blocks = values.len().div_ceil(BLOCK);
+ let mut bases = Vec::with_capacity(blocks);
+ let mut meta = Vec::with_capacity(blocks);
+ let mut widths = Vec::with_capacity(blocks);
+ let mut bits = 0u64;
+ for block in values.chunks(BLOCK) {
+ let min = *block.iter().min().expect("chunks are not empty");
+ let max = *block.iter().max().expect("chunks are not empty");
+ let width = u64::from(64 - (max - min).leading_zeros());
+ bases.push(min);
+ meta.push(bits << 7 | width);
+ widths.push(width);
+ bits += width * block.len() as u64;
+ }
+ let mut words = vec![0u64; bits.div_ceil(64) as usize + 1];
+ for ((block, &min), (&m, &width)) in values
+ .chunks(BLOCK)
+ .zip(&bases)
+ .zip(meta.iter().zip(&widths))
+ {
+ let mut bit = m >> 7;
+ for &value in block {
+ write_bits(&mut words, bit, width as u32, value - min);
+ bit += width;
+ }
+ }
+ Self {
+ len: values.len(),
+ bases: bases.into(),
+ meta: meta.into(),
+ words: words.into(),
+ }
+ }
+
+ /// A packed array from [`Self::parts`] and its length, or `None` unless every
+ /// block header is consistent with `len` and the words hold exactly its bits.
+ pub fn from_parts(len: usize, [bases, meta, words]: [ScalarBuffer; 3]) -> Option {
+ let blocks = len.div_ceil(BLOCK);
+ if bases.len() != blocks || meta.len() != blocks {
+ return None;
+ }
+ let mut bits = 0u64;
+ for (block, &m) in meta.iter().enumerate() {
+ let (start, width) = split_meta(m);
+ if start != bits || width > 64 {
+ return None;
+ }
+ bits += u64::from(width) * (len - block * BLOCK).min(BLOCK) as u64;
+ }
+ (words.len() as u64 == bits.div_ceil(64) + 1).then_some(Self {
+ len,
+ bases,
+ meta,
+ words,
+ })
+ }
+
+ /// The block bases, block headers and words, for storing.
+ pub fn parts(&self) -> [ScalarBuffer; 3] {
+ [self.bases.clone(), self.meta.clone(), self.words.clone()]
+ }
+
+ pub fn len(&self) -> usize {
+ self.len
+ }
+
+ pub fn is_empty(&self) -> bool {
+ self.len == 0
+ }
+
+ /// Value at `index`.
+ ///
+ /// # Panics
+ /// If `index` is out of bounds.
+ #[inline]
+ pub fn get(&self, index: usize) -> u64 {
+ assert!(index < self.len, "index {index} out of bounds");
+ let block = index / BLOCK;
+ let (start, width) = split_meta(self.meta[block]);
+ let bit = start + (index % BLOCK) as u64 * u64::from(width);
+ self.bases[block].wrapping_add(read_bits(&self.words, bit, width))
+ }
+
+ /// The values at `range`, in order, reading each block's header once.
+ #[inline]
+ pub fn iter(&self, range: Range) -> PackedIter<'_> {
+ assert!(range.end <= self.len, "range {range:?} out of bounds");
+ PackedIter {
+ packed: self,
+ position: range.start,
+ end: range.end.max(range.start),
+ block_end: range.start,
+ bit: 0,
+ width: 0,
+ base: 0,
+ }
+ }
+
+ /// Every value, unpacked.
+ pub fn to_vec(&self) -> Vec {
+ self.iter(0..self.len).collect()
+ }
+
+ /// Bytes held by the packed arrays.
+ pub fn size_bytes(&self) -> usize {
+ 8 * (self.bases.len() + self.meta.len() + self.words.len())
+ }
+}
+
+#[inline(always)]
+fn split_meta(meta: u64) -> (u64, u32) {
+ (meta >> 7, (meta & 0x7f) as u32)
+}
+
+#[inline(always)]
+fn read_bits(words: &[u64], bit: u64, width: u32) -> u64 {
+ if width == 0 {
+ return 0;
+ }
+ let word = (bit >> 6) as usize;
+ let shift = (bit & 63) as u32;
+ let mut value = words[word] >> shift;
+ if shift + width > 64 {
+ value |= words[word + 1] << (64 - shift);
+ }
+ if width < 64 {
+ value & ((1u64 << width) - 1)
+ } else {
+ value
+ }
+}
+
+fn write_bits(words: &mut [u64], bit: u64, width: u32, value: u64) {
+ if width == 0 {
+ return;
+ }
+ let word = (bit >> 6) as usize;
+ let shift = (bit & 63) as u32;
+ words[word] |= value << shift;
+ if shift + width > 64 {
+ words[word + 1] |= value >> (64 - shift);
+ }
+}
+
+/// Iterator over a range of a [`PackedU64`]; see [`PackedU64::iter`].
+pub struct PackedIter<'a> {
+ packed: &'a PackedU64,
+ position: usize,
+ end: usize,
+ /// Where the block loaded into `bit`, `width` and `base` ends.
+ block_end: usize,
+ bit: u64,
+ width: u32,
+ base: u64,
+}
+
+impl Iterator for PackedIter<'_> {
+ type Item = u64;
+
+ #[inline(always)]
+ fn next(&mut self) -> Option {
+ if self.position == self.end {
+ return None;
+ }
+ if self.position == self.block_end {
+ let block = self.position / BLOCK;
+ let (start, width) = split_meta(self.packed.meta[block]);
+ self.width = width;
+ self.bit = start + (self.position % BLOCK) as u64 * u64::from(width);
+ self.base = self.packed.bases[block];
+ self.block_end = ((block + 1) * BLOCK).min(self.packed.len);
+ }
+ let value = read_bits(&self.packed.words, self.bit, self.width);
+ self.bit += u64::from(self.width);
+ self.position += 1;
+ Some(self.base.wrapping_add(value))
+ }
+
+ #[inline]
+ fn size_hint(&self) -> (usize, Option) {
+ let n = self.end - self.position;
+ (n, Some(n))
+ }
+}
+
+impl ExactSizeIterator for PackedIter<'_> {}
/// In-memory CSR adjacency index for fast neighbor lookup.
///
@@ -27,15 +245,44 @@ use std::collections::HashMap;
/// - `neighbors[offsets[v]..offsets[v+1]]` = destination vertex IDs
/// - `edge_rows`, aligned with `neighbors` = the row (a Lance row address, or a
/// row index for in-memory input) each edge came from
+///
+/// Edge rows are packed; see the module docs.
#[derive(Debug, Clone)]
pub struct CsrIndex {
- offsets: Vec,
- neighbors: Vec,
- edge_rows: Vec,
+ offsets: ScalarBuffer,
+ neighbors: ScalarBuffer,
+ edge_rows: PackedU64,
}
impl CsrIndex {
- fn range(&self, vertex_id: u64) -> std::ops::Range {
+ /// An index from its three arrays, as [`Self::packed`] returns them, or `None`
+ /// if `offsets` is empty, descends or doesn't end at `neighbors.len()`, or
+ /// `edge_rows` and `neighbors` differ in length.
+ pub fn from_packed(
+ offsets: ScalarBuffer,
+ neighbors: ScalarBuffer,
+ edge_rows: PackedU64,
+ ) -> Option {
+ let well_formed = offsets.first() == Some(&0)
+ && offsets.last() == Some(&(neighbors.len() as u64))
+ && offsets.windows(2).all(|w| w[0] <= w[1])
+ && edge_rows.len() == neighbors.len();
+ well_formed.then_some(Self {
+ offsets,
+ neighbors,
+ edge_rows,
+ })
+ }
+
+ /// The offsets, the neighbours and the packed edge rows.
+ pub fn packed(&self) -> (&ScalarBuffer, &ScalarBuffer, &PackedU64) {
+ (&self.offsets, &self.neighbors, &self.edge_rows)
+ }
+
+ /// Positions of a vertex's neighbours; empty for vertices with no edges or
+ /// beyond the index range.
+ #[inline]
+ pub fn range(&self, vertex_id: u64) -> Range {
let v = vertex_id as usize;
if v >= self.offsets.len() - 1 {
return 0..0;
@@ -43,21 +290,38 @@ impl CsrIndex {
self.offsets[v] as usize..self.offsets[v + 1] as usize
}
- /// Look up all neighbors of a vertex. Returns an empty slice for vertices
- /// with no outgoing edges or vertex IDs beyond the index range.
+ /// The neighbours at `positions`, a sub-range of one or more [`Self::range`]s.
+ #[inline]
+ pub fn neighbors_at(&self, positions: Range) -> &[u64] {
+ &self.neighbors[positions]
+ }
+
+ /// The edge rows at `positions`, aligned with [`Self::neighbors_at`].
+ #[inline]
+ pub fn edge_rows_at(&self, positions: Range) -> PackedIter<'_> {
+ self.edge_rows.iter(positions)
+ }
+
+ /// All neighbours of a vertex; empty for vertices with no edges or beyond the
+ /// index range.
pub fn neighbors(&self, vertex_id: u64) -> &[u64] {
- &self.neighbors[self.range(vertex_id)]
+ self.neighbors_at(self.range(vertex_id))
}
- /// The edge rows of a vertex's neighbors, in the same order as [`Self::neighbors`].
- pub fn edge_rows(&self, vertex_id: u64) -> &[u64] {
- &self.edge_rows[self.range(vertex_id)]
+ /// The edge rows of a vertex's neighbours, in the same order as [`Self::neighbors`].
+ pub fn edge_rows(&self, vertex_id: u64) -> PackedIter<'_> {
+ self.edge_rows_at(self.range(vertex_id))
}
/// Return the total number of edges in the index.
pub fn num_edges(&self) -> u64 {
self.neighbors.len() as u64
}
+
+ /// Bytes held by the offsets, neighbours and edge rows.
+ pub fn size_bytes(&self) -> usize {
+ 8 * (self.offsets.len() + self.neighbors.len()) + self.edge_rows.size_bytes()
+ }
}
/// Builder for constructing a CSR index from edge data.
@@ -125,13 +389,15 @@ impl CsrIndexBuilder {
}
offsets[num_vertices as usize] = running;
- let edge_rows = self.edges.iter().map(|&(_, _, row)| row).collect();
+ let edge_rows: Vec = self.edges.iter().map(|&(_, _, row)| row).collect();
neighbors.extend(self.edges.iter().map(|&(_, dst, _)| dst));
+ // Freed before packing, so the peak holds the unpacked rows or the edges, not both.
+ drop(std::mem::take(&mut self.edges));
CsrIndex {
- offsets,
- neighbors,
- edge_rows,
+ offsets: offsets.into(),
+ neighbors: neighbors.into(),
+ edge_rows: PackedU64::pack(&edge_rows),
}
}
}
@@ -170,9 +436,9 @@ mod tests {
assert_eq!(idx.neighbors(1), &[2]);
assert_eq!(idx.neighbors(2), &[3]);
assert_eq!(idx.neighbors(3), &[] as &[u64]);
- assert_eq!(idx.edge_rows(0), &[0, 1, 2]);
- assert_eq!(idx.edge_rows(2), &[4]);
- assert_eq!(idx.edge_rows(99), &[] as &[u64]);
+ assert_eq!(idx.edge_rows(0).collect::>(), [0, 1, 2]);
+ assert_eq!(idx.edge_rows(2).collect::>(), [4]);
+ assert_eq!(idx.edge_rows(99).collect::>(), Vec::::new());
}
#[test]
@@ -234,6 +500,41 @@ mod tests {
assert_eq!(idx.neighbors(1), &[1]);
}
+ #[test]
+ fn packed_values_round_trip() {
+ // Constant blocks, full 64-bit spans, values straddling words and a short
+ // last block.
+ let mut values: Vec = vec![7; 130];
+ values.extend([0, u64::MAX, 1 << 63, 3]);
+ values.extend((0..300u64).map(|i| i.wrapping_mul(0x9E37_79B9_7F4A_7C15) >> (i % 60)));
+ values.extend((0..200u64).map(|i| 2_199_023_255_552 + i * 37));
+ let packed = PackedU64::pack(&values);
+ assert_eq!(packed.to_vec(), values);
+ for (i, &v) in values.iter().enumerate() {
+ assert_eq!(packed.get(i), v, "position {i}");
+ }
+ for start in [0, 1, 127, 128, 129, 300, values.len()] {
+ let got: Vec = packed.iter(start..values.len()).collect();
+ assert_eq!(got, values[start..], "from {start}");
+ }
+ assert_eq!(packed.iter(5..5).count(), 0);
+ assert!(packed.size_bytes() < 8 * values.len());
+ assert!(PackedU64::pack(&[]).is_empty());
+
+ // Parts round-trip, and inconsistent ones are refused.
+ let again = PackedU64::from_parts(values.len(), packed.parts()).unwrap();
+ assert_eq!(again.to_vec(), values);
+ assert!(PackedU64::from_parts(values.len() + 128, packed.parts()).is_none());
+ let [bases, meta, words] = packed.parts();
+ let short = words.slice(0, words.len() - 1);
+ assert!(
+ PackedU64::from_parts(values.len(), [bases.clone(), meta.clone(), short]).is_none()
+ );
+ let mut bad_meta = meta.to_vec();
+ bad_meta[1] += 1 << 7;
+ assert!(PackedU64::from_parts(values.len(), [bases, bad_meta.into(), words]).is_none());
+ }
+
#[test]
fn test_parallel_edges() {
let idx = CsrIndexBuilder::new()
diff --git a/crates/lance-csr/src/csr_store.rs b/crates/lance-csr/src/csr_store.rs
new file mode 100644
index 0000000..fedb671
--- /dev/null
+++ b/crates/lance-csr/src/csr_store.rs
@@ -0,0 +1,325 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Persisted CSR files, for engines built with `persist_csr`.
+//!
+//! The base CSR of a relationship dataset is stored inside the dataset's own
+//! directory, one Arrow IPC file per dataset version: `_lance_csr/.arrow`.
+//! Lance's `cleanup_old_versions()` deletes only the files it recognises (data,
+//! index, deletion and transaction files, temporary manifests), so it leaves this
+//! folder alone.
+//!
+//! The file holds the CSR as it sits in memory, so a load neither builds nor packs
+//! anything. The schema has one `UInt64` column, `values`, and the file is a
+//! sequence of [`ARRAYS`]-many record batches of different lengths, one array each,
+//! for the outgoing then the incoming direction: the source keys (`i64` bit-cast)
+//! in CSR position order, the offsets, the neighbours, then the bases, block headers
+//! and words of the packed edge rows (see [`crate::csr_index::PackedU64`]). Every batch is zstd-compressed. The arrays are
+//! not meant to be read on their own; this module is their only reader.
+//!
+//! The schema metadata records the dataset version, the endpoint columns and fields,
+//! and the fragments the base was built from, so [`edge_changes_since`] can bring an
+//! older file up to date. A file is used only if it parses, every array is well
+//! formed and its endpoint fields match; anything else is ignored and the CSR is
+//! rebuilt.
+
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use arrow::array::{Array, AsArray, RecordBatch, UInt64Array};
+use arrow::buffer::{Buffer, ScalarBuffer};
+use arrow::datatypes::{DataType, Field, Schema, UInt64Type};
+use arrow::ipc::convert::fb_to_schema;
+use arrow::ipc::reader::{read_footer_length, FileDecoder};
+use arrow::ipc::root_as_footer;
+use arrow::ipc::writer::{FileWriter, IpcWriteOptions};
+use arrow::ipc::CompressionType;
+use lance::dataset::Dataset;
+use lance_table::format::Fragment;
+use object_store::path::Path;
+
+use crate::csr_expand::{endpoint_fields, Adjacency, EndpointFields};
+use crate::csr_index::{CsrIndex, PackedU64};
+use crate::error::{GraphError, Result};
+
+/// Folder at the dataset root that holds the files.
+const DIR: &str = "_lance_csr";
+/// Bumped whenever the layout changes; files of another format are ignored.
+const FORMAT: &str = "2";
+/// Arrays per direction: keys, offsets, neighbours, then the bases, headers and words
+/// of the packed edge rows.
+const ARRAYS: usize = 6;
+
+const KEY_FORMAT: &str = "lance_csr.format";
+const KEY_VERSION: &str = "lance_csr.version";
+const KEY_ENDPOINTS: &str = "lance_csr.endpoints";
+const KEY_FIELDS: &str = "lance_csr.fields";
+const KEY_FRAGMENTS: &str = "lance_csr.fragments";
+
+/// A base CSR read from a file, with what is needed to bring it up to date.
+pub(crate) struct StoredCsr {
+ /// Dataset version the base was built from.
+ pub version: u64,
+ pub fragments: Vec,
+ pub fields: EndpointFields,
+ pub outgoing: Adjacency,
+ pub incoming: Adjacency,
+}
+
+fn store_error(message: String) -> GraphError {
+ GraphError::ExecutionError {
+ message,
+ location: snafu::Location::new(file!(), line!(), column!()),
+ }
+}
+
+/// `/_lance_csr`. Lance exposes `/data` but not the base itself.
+fn dir(ds: &Dataset) -> Path {
+ let data = ds.data_dir();
+ let parts: Vec<_> = data.parts().collect();
+ Path::from_iter(parts[..parts.len().saturating_sub(1)].iter().cloned()).join(DIR)
+}
+
+fn file_path(ds: &Dataset, version: u64) -> Path {
+ dir(ds).join(format!("{version}.arrow"))
+}
+
+/// Versions of the files in `_lance_csr`, unsorted.
+async fn stored_versions(ds: &Dataset) -> Vec {
+ let Ok(store) = ds.object_store(None).await else {
+ return Vec::new();
+ };
+ // A missing folder lists as empty or fails, depending on the store.
+ store
+ .read_dir(dir(ds))
+ .await
+ .unwrap_or_default()
+ .iter()
+ .filter_map(|name| name.strip_suffix(".arrow")?.parse().ok())
+ .collect()
+}
+
+/// The newest stored base at or before `ds`'s version for columns `src -> dst`, or
+/// `None` when there is none or it can't be used.
+pub(crate) async fn load(ds: &Dataset, src: &str, dst: &str) -> Option {
+ let current = ds.manifest().version;
+ let version = stored_versions(ds)
+ .await
+ .into_iter()
+ .filter(|&v| v <= current)
+ .max()?;
+ let store = ds.object_store(None).await.ok()?;
+ let bytes = store.read_one_all(&file_path(ds, version)).await.ok()?;
+ match decode(Buffer::from(bytes), src, dst) {
+ Ok(stored) if stored.version == version => Some(stored),
+ Ok(_) => None,
+ Err(e) => {
+ log::warn!("Ignoring stored CSR {version} of {}: {e}", ds.uri());
+ None
+ }
+ }
+}
+
+/// Write the base CSR of `ds` at its pinned version, then delete older files.
+///
+/// `outgoing` and `incoming` must have no delta: they are the full build of `ds`.
+pub(crate) async fn save(
+ ds: &Dataset,
+ src: &str,
+ dst: &str,
+ outgoing: &Adjacency,
+ incoming: &Adjacency,
+) -> Result<()> {
+ let version = ds.manifest().version;
+ let fields = endpoint_fields(ds, src, dst)
+ .ok_or_else(|| store_error(format!("No endpoint columns {src}, {dst}")))?;
+ let fragments: Vec = ds
+ .get_fragments()
+ .into_iter()
+ .map(|f| f.metadata().clone())
+ .collect();
+ let bytes = encode(version, src, dst, &fields, &fragments, outgoing, incoming)?;
+
+ let store = ds
+ .object_store(None)
+ .await
+ .map_err(|e| store_error(format!("Lance: {e}")))?;
+ let path = file_path(ds, version);
+ // shortcut: the compressed file is held whole in memory before the upload (150 MiB for SF1); write it through a multipart upload if files get much larger.
+ let payload = object_store::PutPayload::from(bytes);
+ let put = store.put_if_absent(&path, payload.clone()).await;
+ let put = match put {
+ // An earlier file for this version wasn't usable (we only save after a build),
+ // or another process just wrote the same content; replace it either way.
+ Err(object_store::Error::AlreadyExists { .. }) => {
+ store
+ .delete(&path)
+ .await
+ .map_err(|e| store_error(format!("Lance: {e}")))?;
+ store.put_if_absent(&path, payload).await
+ }
+ other => other,
+ };
+ put.map_err(|e| store_error(format!("Failed to write {path}: {e}")))?;
+
+ for old in stored_versions(ds)
+ .await
+ .into_iter()
+ .filter(|&v| v < version)
+ {
+ // Best effort: a file left behind is ignored once a newer one exists.
+ let _ = store.delete(&file_path(ds, old)).await;
+ }
+ Ok(())
+}
+
+/// Delete every stored CSR file of `ds`, and the folder where the store allows it.
+/// Returns how many files were deleted.
+pub(crate) async fn delete_all(ds: &Dataset) -> Result {
+ let store = ds
+ .object_store(None)
+ .await
+ .map_err(|e| store_error(format!("Lance: {e}")))?;
+ let mut deleted = 0;
+ for version in stored_versions(ds).await {
+ store
+ .delete(&file_path(ds, version))
+ .await
+ .map_err(|e| store_error(format!("Failed to delete a stored CSR: {e}")))?;
+ deleted += 1;
+ }
+ // Object stores have no folders; on local disk this drops the now empty one.
+ let _ = store.remove_dir_all(dir(ds)).await;
+ Ok(deleted)
+}
+
+fn encode(
+ version: u64,
+ src: &str,
+ dst: &str,
+ fields: &EndpointFields,
+ fragments: &[Fragment],
+ outgoing: &Adjacency,
+ incoming: &Adjacency,
+) -> Result> {
+ let failed = |e: arrow::error::ArrowError| store_error(format!("Failed to encode CSR: {e}"));
+ let metadata = HashMap::from([
+ (KEY_FORMAT.to_string(), FORMAT.to_string()),
+ (KEY_VERSION.to_string(), version.to_string()),
+ (KEY_ENDPOINTS.to_string(), to_json(&[src, dst])?),
+ (KEY_FIELDS.to_string(), to_json(fields)?),
+ (KEY_FRAGMENTS.to_string(), to_json(fragments)?),
+ ]);
+ let schema = Arc::new(Schema::new_with_metadata(
+ vec![Field::new("values", DataType::UInt64, false)],
+ metadata,
+ ));
+ let options = IpcWriteOptions::default()
+ .try_with_compression(Some(CompressionType::ZSTD))
+ .map_err(failed)?;
+ let mut bytes = Vec::new();
+ let mut writer =
+ FileWriter::try_new_with_options(&mut bytes, &schema, options).map_err(failed)?;
+ for adjacency in [outgoing, incoming] {
+ for array in direction_arrays(adjacency) {
+ let column = Arc::new(UInt64Array::new(array, None));
+ let batch = RecordBatch::try_new(schema.clone(), vec![column]).map_err(failed)?;
+ writer.write(&batch).map_err(failed)?;
+ }
+ }
+ writer.finish().map_err(failed)?;
+ drop(writer);
+ Ok(bytes)
+}
+
+/// The [`ARRAYS`] arrays of one direction, shared with the CSR rather than copied
+/// (except the keys, collected from the source-key map).
+fn direction_arrays(adjacency: &Adjacency) -> [ScalarBuffer; ARRAYS] {
+ let (keys, csr) = adjacency.base_parts();
+ let (offsets, neighbors, edge_rows) = csr.packed();
+ let keys: Vec = keys.into_iter().map(|k| k as u64).collect();
+ let [rb, rm, rw] = edge_rows.parts();
+ [keys.into(), offsets.clone(), neighbors.clone(), rb, rm, rw]
+}
+
+fn decode(bytes: Buffer, src: &str, dst: &str) -> Result {
+ let bad = |what: &str| store_error(format!("malformed CSR file: {what}"));
+ let ipc = |e: arrow::error::ArrowError| store_error(format!("malformed CSR file: {e}"));
+ if bytes.len() < 10 {
+ return Err(bad("too short"));
+ }
+ let trailer = bytes.len() - 10;
+ let footer_len =
+ read_footer_length(bytes[trailer..].try_into().expect("10 bytes")).map_err(ipc)?;
+ let footer = root_as_footer(
+ &bytes[trailer
+ .checked_sub(footer_len)
+ .ok_or_else(|| bad("footer"))?..trailer],
+ )
+ .map_err(|e| store_error(format!("malformed CSR file: {e}")))?;
+ let schema = Arc::new(fb_to_schema(
+ footer.schema().ok_or_else(|| bad("no schema"))?,
+ ));
+ let meta = |key: &str| schema.metadata().get(key).ok_or_else(|| bad(key));
+ if meta(KEY_FORMAT)? != FORMAT {
+ return Err(bad("unknown format"));
+ }
+ let endpoints: [String; 2] = from_json(meta(KEY_ENDPOINTS)?)?;
+ if endpoints != [src, dst] {
+ return Err(bad("built for other endpoint columns"));
+ }
+ let version = meta(KEY_VERSION)?.parse().map_err(|_| bad(KEY_VERSION))?;
+ let fields = from_json(meta(KEY_FIELDS)?)?;
+ let fragments = from_json(meta(KEY_FRAGMENTS)?)?;
+
+ let blocks = footer.recordBatches().ok_or_else(|| bad("no batches"))?;
+ if blocks.len() != 2 * ARRAYS {
+ return Err(bad("wrong number of arrays"));
+ }
+ let decoder = FileDecoder::new(schema, footer.version());
+ let mut arrays = Vec::with_capacity(blocks.len());
+ for block in blocks.iter() {
+ let length = block.bodyLength() as usize + block.metaDataLength() as usize;
+ if block.offset() < 0 || block.offset() as usize + length > bytes.len() {
+ return Err(bad("array out of range"));
+ }
+ let data = bytes.slice_with_length(block.offset() as usize, length);
+ let batch = decoder
+ .read_record_batch(block, &data)
+ .map_err(ipc)?
+ .ok_or_else(|| bad("empty batch"))?;
+ let column = batch.column(0).as_primitive_opt::();
+ let column = column
+ .filter(|c| c.null_count() == 0)
+ .ok_or_else(|| bad("array type"))?;
+ arrays.push(column.values().clone());
+ }
+ let incoming = arrays.split_off(ARRAYS);
+ Ok(StoredCsr {
+ version,
+ fragments,
+ fields,
+ outgoing: decode_direction(arrays).ok_or_else(|| bad("outgoing arrays"))?,
+ incoming: decode_direction(incoming).ok_or_else(|| bad("incoming arrays"))?,
+ })
+}
+
+fn decode_direction(arrays: Vec>) -> Option {
+ let [keys, offsets, neighbors, rb, rm, rw]: [ScalarBuffer; ARRAYS] =
+ arrays.try_into().ok()?;
+ let edge_rows = PackedU64::from_parts(neighbors.len(), [rb, rm, rw])?;
+ if keys.len() + 1 != offsets.len() {
+ return None;
+ }
+ let csr = CsrIndex::from_packed(offsets, neighbors, edge_rows)?;
+ let keys: Vec = keys.iter().map(|&k| k as i64).collect();
+ Adjacency::from_base(&keys, csr)
+}
+
+fn from_json(text: &str) -> Result {
+ serde_json::from_str(text).map_err(|e| store_error(format!("malformed CSR file: {e}")))
+}
+
+fn to_json(value: &T) -> Result {
+ serde_json::to_string(value).map_err(|e| store_error(format!("Failed to encode CSR: {e}")))
+}
diff --git a/crates/lance-csr/src/engine.rs b/crates/lance-csr/src/engine.rs
index 1eedae9..de01402 100644
--- a/crates/lance-csr/src/engine.rs
+++ b/crates/lance-csr/src/engine.rs
@@ -5,7 +5,7 @@
//!
//! [`CypherEngine`] holds a snapshot of every input: the DataFusion context, the
//! catalog, join-key statistics and, with `use_csr`, the CSR adjacencies and node
-//! indexes. Queries share the snapshot read-only; [`CypherEngine::refresh`] builds a
+//! indexes. Queries share the snapshot read-only; [`CypherEngine::checkout_latest`] builds a
//! complete replacement and swaps it in.
//!
//! ```no_run
@@ -47,9 +47,10 @@ use serde_json::Value as JsonValue;
use crate::config::GraphConfig;
use crate::csr_expand::{
- edge_changes, is_supported_id_type, Adjacency, AdjacencyCatalog, CsrQueryPlanner, EdgeRows,
- NodeRows, PruneCsrTargetColumns, PushCsrTargetFilters,
+ edge_changes, edge_changes_since, is_supported_id_type, Adjacency, AdjacencyCatalog,
+ CsrQueryPlanner, EdgeChanges, EdgeRows, NodeRows, PruneCsrTargetColumns, PushCsrTargetFilters,
};
+use crate::csr_store;
use crate::error::{GraphError, Result};
use crate::key_statistics::KeyStatisticsProvider;
use crate::lance_vector_search::VectorSearch;
@@ -62,8 +63,8 @@ use crate::{GraphSourceCatalog, InMemoryCatalog};
pub enum EngineSource {
/// On-disk Lance dataset, read lazily through `LanceTableProvider`.
///
- /// `dataset` is pinned to one version; only `refresh()` replaces it. `uri` and
- /// `storage_options` are kept so refresh can reopen the same location with the
+ /// `dataset` is pinned to one version; only `checkout_latest()` replaces it. `uri` and
+ /// `storage_options` are kept so `checkout_latest()` can reopen the same location with the
/// same credentials.
Lance {
uri: String,
@@ -78,7 +79,7 @@ pub enum EngineSource {
impl EngineSource {
/// Open a Lance dataset at `uri`, pinned to `version` (`None` means the latest).
///
- /// `storage_options` are kept so [`CypherEngine::refresh`] can reopen the same
+ /// `storage_options` are kept so [`CypherEngine::checkout_latest`] can reopen the same
/// location with the same credentials.
pub async fn open_lance(
uri: impl Into,
@@ -171,8 +172,9 @@ pub struct CsrStats {
/// Live edges outside the base CSR: appended since its last full build and not
/// since removed. Lance's `num_unindexed_rows`, counted in edges.
pub unindexed_edges: u64,
- /// How the last refresh or `optimize_csr()` produced it: `build` (first), `rebuild`
- /// (full), `delta` (changes applied) or `reused` (no edge change).
+ /// How the last build, `checkout_latest()` or `optimize_csr()` produced it: `build` (first), `rebuild`
+ /// (full), `loaded` (from a stored file, with any changes since applied as a
+ /// delta), `delta` (changes applied) or `reused` (no edge change).
pub update: &'static str,
/// Relationship dataset version it reflects (`None` for Arrow input).
pub version: Option,
@@ -195,11 +197,11 @@ pub struct NodeIndexStats {
/// Everything a query needs, built once and shared read-only by concurrent queries.
///
-/// `refresh()` builds a complete replacement and swaps it in; queries already running
+/// `checkout_latest()` builds a complete replacement and swaps it in; queries already running
/// keep the `Arc` they started with, so they finish on their original snapshot.
struct EngineState {
sources: HashMap,
- /// Unoptimized plan templates for this snapshot; a refresh starts an empty cache.
+ /// Unoptimized plan templates for this snapshot; `checkout_latest()` starts an empty cache.
plans: Mutex>,
catalog: Arc,
context: SessionContext,
@@ -273,13 +275,13 @@ struct CsrEntry {
base_version: Option,
/// Wall time of the last build or delta application.
build_ms: f64,
- /// How the last refresh or optimize produced this entry: build, rebuild, delta or reused.
+ /// How the last refresh or optimize produced this entry: build, rebuild, loaded, delta or reused.
update: &'static str,
}
/// Rebuild instead of applying a delta once pending changes (appended edges +
/// tombstones, accumulated since the last full build) would exceed this fraction
-/// of the base. Lookups pay for the delta on every query, and a rebuild folds it in.
+/// of the base. Lookups read the delta on every query, and a rebuild clears it.
///
/// This threshold also bounds refresh cost: each delta refresh copies the whole
/// accumulated delta (see the design note on `crate::csr_expand::Adjacency`).
@@ -326,16 +328,33 @@ async fn try_apply_csr_delta(
..old.clone()
});
}
+ let (outgoing, incoming) = apply_csr_changes(&old.outgoing, &old.incoming, &changes)?;
+ Some(CsrEntry {
+ source: source.clone(),
+ outgoing: Arc::new(outgoing),
+ incoming: Arc::new(incoming),
+ edges: source.edge_rows(),
+ base_version: old.base_version,
+ build_ms: elapsed(),
+ update: "delta",
+ })
+}
+
+/// `changes` applied to both directions, or `None` when the accumulated delta would
+/// pass `CSR_REBUILD_FRACTION` of the base.
+fn apply_csr_changes(
+ outgoing: &Adjacency,
+ incoming: &Adjacency,
+ changes: &EdgeChanges,
+) -> Option<(Adjacency, Adjacency)> {
let too_large =
- |pending: u64| pending as f64 > CSR_REBUILD_FRACTION * old.outgoing.base_edges() as f64;
- if too_large(
- old.outgoing.delta_edges() + old.outgoing.tombstones() + changes.added.len() as u64,
- ) {
+ |pending: u64| pending as f64 > CSR_REBUILD_FRACTION * outgoing.base_edges() as f64;
+ if too_large(outgoing.delta_edges() + outgoing.tombstones() + changes.added.len() as u64) {
return None;
}
// Removed rows are only counted by applying them; one direction is enough to decide.
- let outgoing = old.outgoing.with_changes(&changes.added, &changes.removed);
- if too_large(outgoing.delta_edges() + outgoing.tombstones()) {
+ let new_outgoing = outgoing.with_changes(&changes.added, &changes.removed);
+ if too_large(new_outgoing.delta_edges() + new_outgoing.tombstones()) {
return None;
}
let flipped: Vec<_> = changes
@@ -343,15 +362,36 @@ async fn try_apply_csr_delta(
.iter()
.map(|&(a, b, row)| (b, a, row))
.collect();
- let incoming = old.incoming.with_changes(&flipped, &changes.removed);
+ let new_incoming = incoming.with_changes(&flipped, &changes.removed);
+ Some((new_outgoing, new_incoming))
+}
+
+/// The CSR stored for `source`, brought up to date with the changes since.
+///
+/// `None` when nothing usable is stored, the changes can't be computed, or they would
+/// pass the rebuild threshold.
+async fn load_stored_csr(source: &EngineSource, src: &str, dst: &str) -> Option {
+ let EngineSource::Lance { dataset, .. } = source else {
+ return None;
+ };
+ let started = std::time::Instant::now();
+ let stored = csr_store::load(dataset, src, dst).await?;
+ let changes = edge_changes_since(&stored.fragments, &stored.fields, dataset, src, dst)
+ .await
+ .ok()??;
+ let (outgoing, incoming) = if changes.is_empty() {
+ (stored.outgoing, stored.incoming)
+ } else {
+ apply_csr_changes(&stored.outgoing, &stored.incoming, &changes)?
+ };
Some(CsrEntry {
source: source.clone(),
outgoing: Arc::new(outgoing),
incoming: Arc::new(incoming),
edges: source.edge_rows(),
- base_version: old.base_version,
- build_ms: elapsed(),
- update: "delta",
+ base_version: Some(stored.version),
+ build_ms: started.elapsed().as_secs_f64() * 1e3,
+ update: "loaded",
})
}
@@ -460,17 +500,16 @@ fn validate_sources(config: &GraphConfig, schemas: &HashMap)
}
/// Validate sources and register them in a fresh catalog and session context.
-/// `fold_csr_deltas` rebuilds every CSR of `previous` that carries a delta.
+/// `rebuild_deltas` rebuilds every CSR of `previous` that carries a delta.
async fn build_engine_state(
config: &GraphConfig,
sources: HashMap,
- use_csr: bool,
- target_partitions: Option,
+ options: &EngineOptions,
previous: Option<&EngineState>,
- fold_csr_deltas: bool,
+ rebuild_deltas: bool,
) -> Result {
let mut session_config = SessionConfig::new();
- if let Some(partitions) = target_partitions {
+ if let Some(partitions) = options.target_partitions {
session_config = session_config.with_target_partitions(partitions);
}
// The CSR planner only changes plans that contain CSR nodes, so install it always.
@@ -508,9 +547,17 @@ async fn build_engine_state(
}
validate_sources(config, &schemas)?;
- let (csr, node_index) = if use_csr {
+ let (csr, node_index) = if options.use_csr {
(
- build_csr_entries(config, &sources, &schemas, previous, fold_csr_deltas).await?,
+ build_csr_entries(
+ config,
+ &sources,
+ &schemas,
+ previous,
+ rebuild_deltas,
+ options.persist_csr,
+ )
+ .await?,
build_node_index(config, &sources, previous).await?,
)
} else {
@@ -627,7 +674,7 @@ fn memtable_partitions(batch: &RecordBatch) -> Vec> {
/// Node ids count as unique (the row count). Edge endpoints use the CSR's distinct
/// source and target counts when the CSR is built, and otherwise one scan of the two
/// endpoint columns; only integer endpoints are counted. Reused while a dataset is
-/// the same snapshot, so an unchanged `refresh()` costs nothing.
+/// the same snapshot, so an unchanged `checkout_latest()` costs nothing.
// shortcut: the fallback scan reads every edge endpoint at construction and on each changed refresh (about one CSR build's scan); switch to an approximate counter (HyperLogLog) or Lance column statistics if that shows up in startup time.
async fn build_key_statistics(
config: &GraphConfig,
@@ -749,12 +796,18 @@ async fn build_key_statistics(
/// snapshot; node-table changes never invalidate it, since the CSR stores raw
/// target ids. Relationship types with non-integer endpoints get no entry and
/// queries over them use the join path.
+///
+/// With `persist`, a Lance relationship dataset's stored CSR is loaded instead of the
+/// first build when it can be brought up to date, and every full build is stored.
+/// Later rebuilds don't look for a file: the stored base is the one the engine already
+/// had, whose delta just grew too large.
async fn build_csr_entries(
config: &GraphConfig,
sources: &HashMap,
schemas: &HashMap,
previous: Option<&EngineState>,
- fold_deltas: bool,
+ rebuild_deltas: bool,
+ persist: bool,
) -> Result> {
let mut entries = HashMap::new();
for (_, mapping) in config.iter_relationship_mappings() {
@@ -767,7 +820,7 @@ async fn build_csr_entries(
};
if let Some(old) = previous.and_then(|p| p.csr.get(rel_type)) {
let delta = old.outgoing.delta_edges() + old.outgoing.tombstones() > 0;
- if old.source.same_data(source) && !(fold_deltas && delta) {
+ if old.source.same_data(source) && !(rebuild_deltas && delta) {
entries.insert(rel_type.clone(), old.clone());
continue;
}
@@ -790,18 +843,36 @@ async fn build_csr_entries(
}
let previous_entry = previous.and_then(|p| p.csr.get(rel_type));
- if let Some(old) = previous_entry.filter(|_| !fold_deltas) {
+ if let Some(old) = previous_entry.filter(|_| !rebuild_deltas) {
if let Some(entry) = try_apply_csr_delta(old, source, src.name(), dst.name()).await {
entries.insert(rel_type.clone(), entry);
continue;
}
}
+ if persist && previous_entry.is_none() {
+ if let Some(entry) = load_stored_csr(source, src.name(), dst.name()).await {
+ entries.insert(rel_type.clone(), entry);
+ continue;
+ }
+ }
+
+ // The one full build: the first, a refresh whose delta would pass the threshold,
+ // and optimize_csr() (`rebuild_deltas`) all end here.
let started = std::time::Instant::now();
// Two projected scans of the endpoint columns, one per direction.
let scan = || source.edge_stream(src.name(), dst.name());
let outgoing = Adjacency::build(scan().await?, src.name(), dst.name()).await?;
let incoming = Adjacency::build(scan().await?, dst.name(), src.name()).await?;
+ let build_ms = started.elapsed().as_secs_f64() * 1e3;
+ if let (true, EngineSource::Lance { dataset, .. }) = (persist, source) {
+ // A failed write (read-only dataset or credentials) only costs the next start a build.
+ if let Err(e) =
+ csr_store::save(dataset, src.name(), dst.name(), &outgoing, &incoming).await
+ {
+ log::warn!("Could not store the CSR of {rel_type}: {e}");
+ }
+ }
entries.insert(
rel_type.clone(),
CsrEntry {
@@ -810,7 +881,7 @@ async fn build_csr_entries(
incoming: Arc::new(incoming),
edges: source.edge_rows(),
base_version: source.lance_version(),
- build_ms: started.elapsed().as_secs_f64() * 1e3,
+ build_ms,
update: if previous_entry.is_some() {
"rebuild"
} else {
@@ -829,20 +900,35 @@ async fn build_csr_entries(
///
/// Lance datasets are read lazily from storage, pinned to the version they were
/// opened at: writes made later (by this or another process) are not visible until
-/// [`refresh`](Self::refresh). Each dataset is pinned separately; versions across
+/// [`checkout_latest`](Self::checkout_latest). Each dataset is pinned separately; versions across
/// tables are not a transactional multi-table snapshot. Arrow batches are held in
/// memory and never change; build a new engine to replace them.
///
/// Building the CSR (`use_csr`) is CPU-bound work inside an async function: in an
-/// application with its own async runtime, construct and refresh the engine on a
+/// application with its own async runtime, construct the engine and call `checkout_latest()` on a
/// blocking thread (for example `tokio::task::spawn_blocking` plus `block_on`).
+/// Options for [`CypherEngine::with_options`].
+#[derive(Debug, Clone, Default, PartialEq)]
+pub struct EngineOptions {
+ /// Build CSR adjacencies and node indexes; see [`CypherEngine::new`].
+ pub use_csr: bool,
+ /// DataFusion `target_partitions`; see [`CypherEngine::with_target_partitions`].
+ pub target_partitions: Option,
+ /// Store each fully built CSR inside its Lance relationship dataset, at
+ /// `_lance_csr/.arrow`, and load it instead of building when an engine
+ /// starts. Changes since the stored version apply as a delta, under
+ /// the same 25% threshold as `checkout_latest()`. Ignored without `use_csr` and for Arrow
+ /// input. A failed write is logged and doesn't fail the engine.
+ pub persist_csr: bool,
+}
+
pub struct CypherEngine {
config: GraphConfig,
state: RwLock>,
- use_csr: bool,
- target_partitions: Option,
- /// Held for a whole refresh, so a slower refresh can't swap in older data over a newer one.
- refresh_lock: futures::lock::Mutex<()>,
+ options: EngineOptions,
+ /// Held for a whole update of the snapshot, so a slower `checkout_latest()` can't swap in older
+ /// data over a newer one.
+ update_lock: futures::lock::Mutex<()>,
}
impl CypherEngine {
@@ -868,7 +954,7 @@ impl CypherEngine {
/// Like [`new`](Self::new), with a target for DataFusion query partitioning.
///
/// `Some(n)` sets DataFusion's `target_partitions` for every query, kept across
- /// [`refresh`](Self::refresh); `None` keeps DataFusion's default (the number of
+ /// [`checkout_latest`](Self::checkout_latest); `None` keeps DataFusion's default (the number of
/// cores). It is not a thread or memory limit. In-memory Arrow inputs are still
/// sliced into one partition per core.
///
@@ -880,29 +966,44 @@ impl CypherEngine {
sources: HashMap,
use_csr: bool,
target_partitions: Option,
+ ) -> Result {
+ let options = EngineOptions {
+ use_csr,
+ target_partitions,
+ ..Default::default()
+ };
+ Self::with_options(config, sources, options).await
+ }
+
+ /// Like [`new`](Self::new), with every [`EngineOptions`] setting.
+ ///
+ /// # Errors
+ /// As [`with_target_partitions`](Self::with_target_partitions).
+ pub async fn with_options(
+ config: GraphConfig,
+ sources: HashMap,
+ options: EngineOptions,
) -> Result {
if sources.is_empty() {
return Err(config_error("No input datasets provided".to_string()));
}
- if let Some(partitions) = target_partitions {
+ if let Some(partitions) = options.target_partitions {
if !(1..=MAX_TARGET_PARTITIONS).contains(&partitions) {
return Err(config_error(format!(
"target_partitions must be between 1 and {MAX_TARGET_PARTITIONS}"
)));
}
}
- let state =
- build_engine_state(&config, sources, use_csr, target_partitions, None, false).await?;
+ let state = build_engine_state(&config, sources, &options, None, false).await?;
Ok(Self {
config,
state: RwLock::new(Arc::new(state)),
- use_csr,
- target_partitions,
- refresh_lock: Default::default(),
+ options,
+ update_lock: Default::default(),
})
}
- /// The current snapshot. Cloning the `Arc` lets a query outlive a concurrent refresh.
+ /// The current snapshot. Cloning the `Arc` lets a query outlive a concurrent `checkout_latest()`.
fn snapshot(&self) -> Arc {
self.state
.read()
@@ -940,7 +1041,7 @@ impl CypherEngine {
.with_parameters(parameters)
.with_config(self.config.clone())
.with_fan_out(state.fan_out());
- let mut query = if self.use_csr {
+ let mut query = if self.options.use_csr {
query.with_adjacency(state.adjacency_catalog())
} else {
query
@@ -987,7 +1088,7 @@ impl CypherEngine {
///
/// Batches are computed as the stream is polled. Scans, filters and projections
/// stream with bounded memory; joins, sorts and aggregations may still buffer.
- /// The stream keeps what it reads alive, so a concurrent `refresh()` does not
+ /// The stream keeps what it reads alive, so a concurrent `checkout_latest()` does not
/// affect it. Planning errors return here; execution errors arrive in the stream.
/// Dropping the stream cancels the remaining work.
pub async fn execute_stream(
@@ -1023,15 +1124,16 @@ impl CypherEngine {
vector_search.search(&candidates).await
}
- /// Move every Lance dataset to its latest version.
+ /// Move every Lance dataset to its latest version: the engine's counterpart of
+ /// Lance's `Dataset::checkout_latest()`.
///
/// Datasets whose version and commit time are unchanged keep their open handle.
/// The replacement is validated in full before it is swapped in; on error nothing
/// changes and the previous snapshot stays usable. Queries already running finish
/// on the snapshot they started with. Arrow inputs are unchanged. Concurrent calls
/// run one at a time.
- pub async fn refresh(&self) -> Result<()> {
- let _refreshing = self.refresh_lock.lock().await;
+ pub async fn checkout_latest(&self) -> Result<()> {
+ let _updating = self.update_lock.lock().await;
let current = self.snapshot();
let mut sources = current.sources.clone();
for source in sources.values_mut() {
@@ -1047,35 +1149,33 @@ impl CypherEngine {
}
}
}
- let state = build_engine_state(
- &self.config,
- sources,
- self.use_csr,
- self.target_partitions,
- Some(¤t),
- false,
- )
- .await?;
+ let state =
+ build_engine_state(&self.config, sources, &self.options, Some(¤t), false).await?;
*self.state.write().expect("engine state lock poisoned") = Arc::new(state);
Ok(())
}
- /// Fold each CSR's delta into a new base, at the snapshot the engine is pinned to: the
+ /// Former name of [`checkout_latest`](Self::checkout_latest).
+ #[deprecated(note = "renamed to `checkout_latest`, after Lance's `Dataset::checkout_latest`")]
+ pub async fn refresh(&self) -> Result<()> {
+ self.checkout_latest().await
+ }
+
+ /// Rebuild each CSR that carries a delta, at the snapshot the engine is pinned to: the
/// CSR's counterpart of Lance's `optimize_indices()`.
///
- /// `refresh()` applies changes as a delta over the base CSR and rebuilds on its own
+ /// `checkout_latest()` applies changes as a delta over the base CSR and rebuilds on its own
/// only once they pass 25% of the base. This rebuilds every CSR that carries a delta
/// now, so lookups stop carrying it; one without a delta, and everything else, is
/// reused. Results never change. The replacement is validated in full before it is
- /// swapped in, as for `refresh()`, and the two run one at a time.
+ /// swapped in, as for `checkout_latest()`, and the two run one at a time.
pub async fn optimize_csr(&self) -> Result<()> {
- let _refreshing = self.refresh_lock.lock().await;
+ let _updating = self.update_lock.lock().await;
let current = self.snapshot();
let state = build_engine_state(
&self.config,
current.sources.clone(),
- self.use_csr,
- self.target_partitions,
+ &self.options,
Some(¤t),
true,
)
@@ -1084,9 +1184,32 @@ impl CypherEngine {
Ok(())
}
+ /// Delete the CSR files stored for this engine's Lance relationship datasets
+ /// (see [`EngineOptions::persist_csr`]), returning how many files were deleted.
+ ///
+ /// The engine keeps the CSR it has in memory, so queries are unaffected. An
+ /// engine with `persist_csr` stores a new file at its next full build; create
+ /// engines without it to stay in memory only. Runs one at a time with `checkout_latest()`
+ /// and `optimize_csr()`.
+ pub async fn delete_stored_csr(&self) -> Result {
+ let _updating = self.update_lock.lock().await;
+ let current = self.snapshot();
+ let mut deleted = 0;
+ for (_, mapping) in self.config.iter_relationship_mappings() {
+ let source = current
+ .sources
+ .iter()
+ .find(|(name, _)| name.eq_ignore_ascii_case(&mapping.relationship_type));
+ if let Some((_, EngineSource::Lance { dataset, .. })) = source {
+ deleted += csr_store::delete_all(dataset).await?;
+ }
+ }
+ Ok(deleted)
+ }
+
/// CSR adjacency statistics per relationship type (experimental, `use_csr`).
///
- /// An identical map after `refresh()` means the snapshot itself was unchanged.
+ /// An identical map after `checkout_latest()` means the snapshot itself was unchanged.
pub fn csr_stats(&self) -> HashMap {
self.snapshot()
.csr
@@ -1165,7 +1288,7 @@ mod tests {
}
#[tokio::test]
- async fn target_partitions_survive_refresh_without_changing_other_defaults() {
+ async fn target_partitions_survive_checkout_latest_without_changing_other_defaults() {
let defaults = SessionConfig::new();
for target_partitions in [None, Some(1), Some(2)] {
let engine = CypherEngine::with_target_partitions(
@@ -1177,7 +1300,7 @@ mod tests {
.await
.unwrap();
let before = engine.snapshot();
- engine.refresh().await.unwrap();
+ engine.checkout_latest().await.unwrap();
for snapshot in [before, engine.snapshot()] {
let state = snapshot.context.state();
assert_eq!(
@@ -1203,7 +1326,7 @@ mod tests {
}
#[tokio::test]
- async fn plan_cache_reuses_bounds_and_resets_on_refresh() {
+ async fn plan_cache_reuses_bounds_and_resets_on_checkout_latest() {
let engine = CypherEngine::new(person_config(), people_engine_sources(), false)
.await
.unwrap();
@@ -1270,7 +1393,7 @@ mod tests {
assert_eq!(first.num_rows(), 1);
// A refresh publishes a new snapshot with an empty cache; the old one is untouched.
- engine.refresh().await.unwrap();
+ engine.checkout_latest().await.unwrap();
assert!(engine.snapshot().plans.lock().unwrap().is_empty());
assert_eq!(cached(), PLAN_CACHE_ENTRIES);
}
@@ -1319,10 +1442,10 @@ mod tests {
.unwrap();
}
- /// A refresh that starts while another is in flight waits for it, then publishes
+ /// A `checkout_latest()` that starts while another is in flight waits for it, then publishes
/// the newest version: an older snapshot can never land after a newer one.
#[tokio::test]
- async fn refresh_waits_for_an_in_flight_refresh() {
+ async fn checkout_latest_waits_for_an_in_flight_checkout() {
let dir = tempfile::tempdir().unwrap();
let uri = dir.path().join("person.lance").display().to_string();
write_people(&uri, vec![1], WriteMode::Create).await;
@@ -1337,13 +1460,13 @@ mod tests {
let engine = Arc::new(CypherEngine::new(config, sources, true).await.unwrap());
// Stand in for refresh A, still building its replacement.
- let in_flight = engine.refresh_lock.lock().await;
+ let in_flight = engine.update_lock.lock().await;
let b = tokio::spawn({
let engine = engine.clone();
- async move { engine.refresh().await }
+ async move { engine.checkout_latest().await }
});
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
- assert!(!b.is_finished(), "refresh B ran while A was in flight");
+ assert!(!b.is_finished(), "checkout B ran while A was in flight");
// A newer version lands before A finishes; B must publish it, not anything older.
write_people(&uri, vec![2], WriteMode::Append).await;
@@ -1352,4 +1475,197 @@ mod tests {
assert_eq!(engine.dataset_versions()["Person"], 2);
assert_eq!(engine.node_index_stats()["Person"].rows, 2);
}
+
+ async fn write_knows(uri: &str, edges: &[(i64, i64)], mode: WriteMode) {
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("src", DataType::Int64, false),
+ Field::new("dst", DataType::Int64, false),
+ ]));
+ let batch = RecordBatch::try_new(
+ schema.clone(),
+ vec![
+ Arc::new(Int64Array::from_iter_values(edges.iter().map(|e| e.0))),
+ Arc::new(Int64Array::from_iter_values(edges.iter().map(|e| e.1))),
+ ],
+ )
+ .unwrap();
+ let params = WriteParams {
+ mode,
+ max_rows_per_file: 40,
+ ..Default::default()
+ };
+ Dataset::write(
+ RecordBatchIterator::new([Ok(batch)], schema),
+ uri,
+ Some(params),
+ )
+ .await
+ .unwrap();
+ }
+
+ /// Every `(a, b)` reachable in two hops, sorted, from an engine over `uri`.
+ async fn two_hops(uri: &str, persist_csr: bool) -> (Vec<(i64, i64)>, CsrStats) {
+ let config = GraphConfig::builder()
+ .node_table("Person", "id")
+ .rel_table("KNOWS", "Person", "Person", "src", "dst")
+ .build()
+ .unwrap();
+ let people = RecordBatch::try_from_iter([(
+ "id",
+ Arc::new(Int64Array::from_iter_values(0..60)) as arrow_array::ArrayRef,
+ )])
+ .unwrap();
+ let knows = EngineSource::open_lance(uri, None, HashMap::new())
+ .await
+ .unwrap();
+ let sources = HashMap::from([
+ ("Person".to_string(), EngineSource::Arrow(people)),
+ ("KNOWS".to_string(), knows),
+ ]);
+ let options = EngineOptions {
+ use_csr: true,
+ persist_csr,
+ ..Default::default()
+ };
+ let engine = CypherEngine::with_options(config, sources, options)
+ .await
+ .unwrap();
+ let batch = engine
+ .execute(
+ "MATCH (a:Person)-[:KNOWS]->(:Person)<-[:KNOWS]-(b:Person) \
+ RETURN a.id AS a, b.id AS b",
+ HashMap::new(),
+ )
+ .await
+ .unwrap();
+ let column = |i: usize| {
+ batch
+ .column(i)
+ .as_any()
+ .downcast_ref::()
+ .unwrap()
+ .clone()
+ };
+ let (a, b) = (column(0), column(1));
+ let mut rows: Vec<_> = a
+ .values()
+ .iter()
+ .copied()
+ .zip(b.values().iter().copied())
+ .collect();
+ rows.sort_unstable();
+ (rows, engine.csr_stats()["KNOWS"].clone())
+ }
+
+ fn stored_files(uri: &str) -> Vec {
+ let mut names: Vec =
+ std::fs::read_dir(std::path::Path::new(uri).join("_lance_csr"))
+ .map(|entries| {
+ entries
+ .map(|e| e.unwrap().file_name().to_string_lossy().into_owned())
+ .collect()
+ })
+ .unwrap_or_default();
+ names.sort();
+ names
+ }
+
+ /// A stored CSR is loaded in place of a build, brought up to date with later
+ /// appends and deletes, ignored when corrupt, and replaced by `optimize_csr()`;
+ /// every answer matches an engine that builds from scratch.
+ #[tokio::test]
+ async fn persisted_csr_loads_and_catches_up() {
+ let dir = tempfile::tempdir().unwrap();
+ let uri = dir.path().join("knows.lance").display().to_string();
+ // Duplicate edges and a self-loop, over three fragments.
+ let mut edges: Vec<(i64, i64)> = (0..100).map(|i| (i % 50, (i * 7) % 60)).collect();
+ edges.extend([(3, 3), (3, 21), (3, 21)]);
+ write_knows(&uri, &edges, WriteMode::Create).await;
+
+ let (expected, stats) = two_hops(&uri, false).await;
+ assert_eq!(stats.update, "build");
+ assert!(
+ stored_files(&uri).is_empty(),
+ "nothing is stored without persist_csr"
+ );
+
+ let (rows, stats) = two_hops(&uri, true).await;
+ assert_eq!((rows == expected, stats.update), (true, "build"));
+ assert_eq!(stored_files(&uri), ["1.arrow"]);
+
+ let (rows, stats) = two_hops(&uri, true).await;
+ assert_eq!((rows == expected, stats.update), (true, "loaded"));
+ assert_eq!((stats.edges, stats.base_version), (103, Some(1)));
+
+ // An append and a delete since the stored version become a delta.
+ write_knows(&uri, &[(1, 2), (2, 1), (40, 3)], WriteMode::Append).await;
+ let mut ds = Dataset::open(&uri).await.unwrap();
+ ds.delete("src = 3 AND dst = 21").await.unwrap();
+ let (expected, _) = two_hops(&uri, false).await;
+ let (rows, stats) = two_hops(&uri, true).await;
+ assert_eq!((rows == expected, stats.update), (true, "loaded"));
+ assert_eq!(
+ (
+ stats.delta_edges,
+ stats.tombstones,
+ stats.version,
+ stats.base_version
+ ),
+ (3, 3, Some(3), Some(1))
+ );
+ assert_eq!(
+ stored_files(&uri),
+ ["1.arrow"],
+ "a delta doesn't store a new base"
+ );
+
+ // A corrupt file is ignored: the engine builds and stores the current version.
+ std::fs::write(
+ dir.path().join("knows.lance/_lance_csr/1.arrow"),
+ b"not arrow",
+ )
+ .unwrap();
+ let (rows, stats) = two_hops(&uri, true).await;
+ assert_eq!((rows == expected, stats.update), (true, "build"));
+ assert_eq!(stored_files(&uri), ["3.arrow"], "older files are pruned");
+
+ // optimize_csr() stores the rebuilt base of a CSR loaded with a delta.
+ write_knows(&uri, &[(5, 6)], WriteMode::Append).await;
+ let config = GraphConfig::builder()
+ .node_table("Person", "id")
+ .rel_table("KNOWS", "Person", "Person", "src", "dst")
+ .build()
+ .unwrap();
+ let knows = EngineSource::open_lance(&uri, None, HashMap::new())
+ .await
+ .unwrap();
+ let options = EngineOptions {
+ use_csr: true,
+ persist_csr: true,
+ ..Default::default()
+ };
+ let engine = CypherEngine::with_options(
+ config,
+ HashMap::from([("KNOWS".to_string(), knows)]),
+ options,
+ )
+ .await
+ .unwrap();
+ assert_eq!(engine.csr_stats()["KNOWS"].delta_edges, 1);
+ engine.optimize_csr().await.unwrap();
+ let stats = &engine.csr_stats()["KNOWS"];
+ assert_eq!((stats.update, stats.base_version), ("rebuild", Some(4)));
+ assert_eq!(stored_files(&uri), ["4.arrow"]);
+ let (rows, stats) = two_hops(&uri, true).await;
+ assert_eq!(stats.update, "loaded");
+ assert_eq!(rows, two_hops(&uri, false).await.0);
+
+ // Deleting the stored files leaves the engine's CSR in place.
+ assert_eq!(engine.delete_stored_csr().await.unwrap(), 1);
+ assert!(stored_files(&uri).is_empty());
+ assert!(!dir.path().join("knows.lance/_lance_csr").exists());
+ assert_eq!(engine.csr_stats()["KNOWS"].edges, 104);
+ assert_eq!(engine.delete_stored_csr().await.unwrap(), 0);
+ assert_eq!(two_hops(&uri, true).await.1.update, "build");
+ }
}
diff --git a/crates/lance-csr/src/lib.rs b/crates/lance-csr/src/lib.rs
index 5c9b399..7e86159 100644
--- a/crates/lance-csr/src/lib.rs
+++ b/crates/lance-csr/src/lib.rs
@@ -52,6 +52,7 @@ pub mod case_insensitive;
pub mod config;
pub mod csr_expand;
pub mod csr_index;
+mod csr_store;
mod csr_traversal;
pub mod datafusion_planner;
pub mod engine;
diff --git a/crates/lance-csr/tests/test_engine.rs b/crates/lance-csr/tests/test_engine.rs
index fe1b18d..5e5adcc 100644
--- a/crates/lance-csr/tests/test_engine.rs
+++ b/crates/lance-csr/tests/test_engine.rs
@@ -116,7 +116,7 @@ async fn parameters_and_streaming() {
}
#[tokio::test]
-async fn lance_input_is_pinned_until_refresh() {
+async fn lance_input_is_pinned_until_checkout_latest() {
let dir = tempfile::tempdir().unwrap();
let person_uri = dir.path().join("person.lance");
let knows_uri = dir.path().join("knows.lance");
@@ -144,14 +144,14 @@ async fn lance_input_is_pinned_until_refresh() {
["Carol"]
);
- // A write after construction stays invisible until refresh().
+ // A write after construction stays invisible until checkout_latest().
write(knows(&[(2, 4)]), knows_uri, WriteMode::Append).await;
assert_eq!(
names(&engine.execute(TWO_HOPS, HashMap::new()).await.unwrap()),
["Carol"]
);
- engine.refresh().await.unwrap();
+ engine.checkout_latest().await.unwrap();
assert_eq!(
names(&engine.execute(TWO_HOPS, HashMap::new()).await.unwrap()),
["Carol", "David"]
@@ -179,7 +179,7 @@ async fn optimize_csr_folds_the_delta_into_the_base() {
let engine = CypherEngine::new(config(), sources, true).await.unwrap();
write(knows(&[(2, 4)]), knows_uri, WriteMode::Append).await;
- engine.refresh().await.unwrap();
+ engine.checkout_latest().await.unwrap();
let csr = engine.csr_stats()["KNOWS"].clone();
assert_eq!(
(csr.update, csr.delta_edges, csr.unindexed_edges),
diff --git a/docs/AGENTS.md b/docs/AGENTS.md
index 3da48c0..13327bf 100644
--- a/docs/AGENTS.md
+++ b/docs/AGENTS.md
@@ -31,7 +31,7 @@ them.
Build from the repository root with the locked build command above and publish `site/`.
- Files that aren't pages (this file, `check_snippets.py`, `sync_benchmarks.py`, the uv project, `overrides/`, the
brand README and previews) are listed under `exclude_docs` in `zensical.toml`. Add new non-page files there.
-- Benchmark tables between `` markers in `benchmarks.md` and the root `README.md` are
+- Benchmark tables between `` markers in `benchmarks.md` are
generated; never edit them by hand. The runners and results live in graph-benchmark-ldbc-complex and
graph-benchmark-ldbc. To publish a new run, merge it there, pin its commit in `sync_benchmarks.py`, and run
`uv run --project docs --locked python docs/sync_benchmarks.py`.
@@ -76,14 +76,20 @@ them.
## Style
-- Lead with what the reader wants to do, then show it. Mix short prose with code; a concept that takes more
- than two paragraphs probably needs an example or a diagram.
+- Explain what something is and why it exists before how to use it. Don't open a page or section with an
+ instruction ("Call it after…", "Use this when…"): the reader needs the what and the why to judge the how. Mix
+ short prose with code; a concept that takes more than two paragraphs probably needs an example or a diagram.
+- Don't assume the reader knows the background. Give the context a newcomer needs (for example, that Lance
+ datasets gain new versions as they're written, so the engine has to be told about them), and define a term,
+ such as the CSR's delta, before relying on it.
+- Say each thing once. A section shouldn't restate what the page already said above it; keep sentences short and
+ cut words that add no information.
- Use Zensical components where they help the reader, not for decoration:
- admonitions (`!!! tip`, `note`, `warning`, `danger`, `info`, `example`, `failure`, `bug`) for advice,
caveats and known issues; `???` makes a collapsible one for long detail;
- content tabs (`=== "Tab"`) for alternatives, such as install methods or input types;
- code annotations (`# (1)!` plus a numbered list after the block) to explain specific lines;
- - Mermaid diagrams (```` ```mermaid ````) for flows: query pipelines, refresh decisions, snapshots;
+ - Mermaid diagrams (```` ```mermaid ````) for flows: query pipelines, `checkout_latest()` decisions, snapshots;
- tables for options and comparisons, definition lists for method references;
- grid cards on the home page only.
- Plain, direct sentences. Use "you". Avoid filler and marketing adjectives. Spell out a term the first time
diff --git a/docs/benchmarks.md b/docs/benchmarks.md
index 6b3a416..ebc825d 100644
--- a/docs/benchmarks.md
+++ b/docs/benchmarks.md
@@ -163,10 +163,10 @@ uv run --no-sync python benchmarks/csr_expand.py --data /tmp/csr-bench --edges 1
| Script | Measures |
| --- | --- |
-| `csr_expand.py` | CSR build cost, single-hop latency against the join path, refresh cost and peak RSS, per edge count |
-| `csr_delta.py` | `refresh()` with a delta against a full rebuild, after one append or delete commit (uses `csr_expand.py`'s data) |
+| `csr_expand.py` | CSR build cost, single-hop latency against the join path, `checkout_latest()` cost and peak RSS, per edge count |
+| `csr_delta.py` | `checkout_latest()` with a delta against a full rebuild, after one append or delete commit (uses `csr_expand.py`'s data) |
| `csr_delta_accumulated.py` | Refresh cost as deltas accumulate over many small commits (the basis of the 25% rebuild threshold) |
-| `engine_cache.py` | Engine construction, first-query, warm-query and refresh timings; can compare against upstream `lance_graph` |
+| `engine_cache.py` | Engine construction, first-query, warm-query and `checkout_latest()` timings; can compare against upstream `lance_graph` |
| `bounded_memory.py` | Peak memory of streamed scans over Lance data larger than a memory cap (`generate`, then `run`) |
Each script's docstring lists its options. When you change performance-sensitive code, include before and
diff --git a/docs/csr.md b/docs/csr.md
index d6ed058..f37a788 100644
--- a/docs/csr.md
+++ b/docs/csr.md
@@ -90,14 +90,16 @@ Node index
!!! info "Memory cost"
Expect about **95 bytes per edge** of extra peak memory while the CSR is built, for both directions
- together (measured at 10M edges with `python/benchmarks/csr_expand.py`). About 32 bytes per edge stay
- resident afterwards: each direction holds the neighbour key and the edge's row for every edge. The index
- lives in memory even when the tables are read lazily from Lance.
+ together (measured at 10M edges with `python/benchmarks/csr_expand.py`). Afterwards, each direction keeps
+ 8 bytes per edge for the neighbour key and about 2 for the edge's row, which is stored
+ [packed](internals/indexes.md#packed-edge-rows), plus about 37 bytes per source node for its offset and its
+ entry in the map from key to position. On the GDC SF1 graph that's 732 MiB for 17.3 million edges, both
+ directions. The index lives in memory even when the tables are read lazily from Lance.
During query execution, each CSR expansion partition keeps one input batch and collects at most one
output chunk of neighbour positions before fetching properties. It yields that chunk before visiting
the remaining neighbours, even for a single high-degree node. Deleted edge positions are read from the
- shared snapshot rather than copied for each query; refresh resolves those positions by scanning the
+ shared snapshot rather than copied for each query; `checkout_latest()` resolves those positions by scanning the
affected sources' neighbour lists.
Cancellation checkpoints count source rows and frontiers with no surviving edges,
@@ -271,7 +273,7 @@ print(engine.csr_stats()["KNOWS"])
| `delta_edges` | Appended edges held outside the base CSR since its last full build. |
| `tombstones` | Removed edges masked out since the last full build. |
| `unindexed_edges` | Live edges outside the base CSR: appended since its last full build and not since removed. Lance index statistics call this `num_unindexed_rows`. |
-| `update` | How the last build, `refresh()` or [`optimize_csr()`](#folding-the-delta-in) produced it: `build` (first), `rebuild` (full), `delta` (changes applied) or `reused` (no edge change). |
+| `update` | How the last build, `checkout_latest()` or [`optimize_csr()`](#rebuilding-the-csr) produced it: `build` (first), `rebuild` (full), `loaded` (from a [stored file](#storing-the-csr), with any changes since applied as a delta), `delta` (changes applied) or `reused` (no edge change). |
| `version` | The relationship dataset's Lance version, or `None` for PyArrow input. |
| `base_version` | The version of the last full build, or `None` for PyArrow input. It trails `version` while deltas apply. |
| `build_ms` | Wall time of that build or delta, both directions. |
@@ -288,47 +290,18 @@ print(sorted(engine.node_index_stats())) # ['Person']
## Keeping the CSR current
-In-memory PyArrow tables never change, so their CSR never needs updating. For Lance datasets, the engine
-is pinned to a version until you call [`refresh()`](lance.md#refresh), which updates the CSR incrementally:
+A Lance dataset gets a new version every time someone writes to it, whether that's appending rows, deleting them
+or compacting its files. The engine doesn't switch versions on its own. It answers every query from the versions it
+opened, so that queries stay consistent with each other, which means new writes stay invisible until you ask for
+them.
-```mermaid
-flowchart TD
- R["engine.refresh()"] --> V{"Relationship dataset
changed?"}
- V -- no --> U["reused"]
- V -- yes --> D{"Same endpoint
columns?"}
- D -- no --> RB["rebuild"]
- D -- yes --> S{"Pending changes > 25%
of the base CSR?"}
- S -- yes --> RB
- S -- no --> DL["delta: appended edges +
tombstones for deleted ones"]
-```
-
-- **Appends** become delta edges and **deletes** become tombstones, both applied on top of the base CSR.
- Deleted and rewritten rows are matched by the row each CSR entry records, so only added fragments are
- read. Compaction and `cleanup_old_versions()` don't force a rebuild: nothing is read from the old version.
-- Once the accumulated delta would exceed **25%** of the base, or the dataset was overwritten, the CSR is
- **rebuilt** and the delta folded in.
-- **Node-table writes never touch the CSR**: it stores raw target keys, not row positions. A changed node
- table rebuilds only its node index.
-- The replacement is validated in full before it's swapped in. If anything fails, nothing changes and the
- previous snapshot stays usable. Queries already running finish on the snapshot they started with.
-
-The 25% threshold is fixed: lookups pay a little for every pending delta edge, and a rebuild clears them.
-`python/benchmarks/csr_delta_accumulated.py` measures that trade-off. To fold the delta in sooner, call
-[`optimize_csr()`](#folding-the-delta-in).
-
-### How to keep the CSR fresh
-
-The engine doesn't watch your datasets for changes. Keeping the CSR in step with the data is up to you:
-
-1. **Write with any Lance writer**: `pylance`, LanceDB or another process.
-2. **Call `refresh()` after the writes your queries should see.** When a writer updates several tables, call it
- after all of them commit (see [snapshots are per dataset](lance.md#lazy-version-pinned-reads)). A refresh
- with nothing new reuses the existing CSR, so calling it on a timer or before each batch of queries is fine.
-3. **Check what it did with [`csr_stats()`](#csr_stats)**: `update` says whether the CSR was `reused`, updated
- with a `delta` or fully rebuilt (`rebuild`). `base_version` and `unindexed_edges` say how far the base
- CSR trails the data. Answers are exact either way: delta edges are part of every lookup.
-4. **Optionally, fold the delta in with [`optimize_csr()`](#folding-the-delta-in)** at a quiet moment, so
- lookups stop carrying it.
+`checkout_latest()` is how you ask. It moves the engine to the newest version of every Lance dataset it reads, the
+way Lance's own `checkout_latest()` does for a single dataset (see Lance's
+[time travel](https://lance.org/quickstart/#time-travel)), and it updates everything the engine built from those
+datasets, the CSR included. Afterwards, queries see the new data and their answers are exact. When nothing has
+changed, it returns almost immediately, so it's safe to call often, even on a timer. If a writer updates several
+tables, call it once they've all committed, since each Lance dataset is
+[pinned on its own](lance.md#lazy-version-pinned-reads).
=== "Python"
@@ -343,9 +316,9 @@ The engine doesn't watch your datasets for changes. Keeping the CSR in step with
use_csr=True,
)
- # David now knows Alice. The engine doesn't see the write until refresh().
+ # David now knows Alice. The engine doesn't see the write until checkout_latest().
lance.write_dataset(pa.table({"src": [4], "dst": [1]}), "knows.lance", mode="append")
- engine.refresh()
+ engine.checkout_latest()
stats = engine.csr_stats()["KNOWS"]
print(stats["edges"], stats["delta_edges"], stats["update"]) # 5 1 delta
@@ -356,52 +329,145 @@ The engine doesn't watch your datasets for changes. Keeping the CSR in step with
This continues the on-disk example from [Using from Rust](rust.md), where Bob gains an edge to Alice.
```rust
- --8<-- "crates/lance-csr/examples/embedded_query.rs:refresh"
+ --8<-- "crates/lance-csr/examples/embedded_query.rs:checkout_latest"
```
-A refresh can only apply a delta when it can compare the version the engine is pinned to with the latest one.
-If you overwrite a relationship dataset, or remove the pinned version's files (for example with
-`cleanup_old_versions()`) before refreshing, the next refresh rebuilds that CSR in full. Refresh more often
-than you clean up old versions to keep updates incremental.
+!!! caution "In-memory PyArrow tables"
-In-memory PyArrow tables never change. To query new in-memory data, build a new engine.
+ If you gave the engine in-memory PyArrow tables instead of Lance datasets, there's nothing to update: those
+ tables can't change underneath the engine, so their CSR is never out of date. To query new in-memory data,
+ create a new engine.
-### Folding the delta in
+### How the CSR keeps up with new writes { #checkout-latest }
-The CSR follows Lance's own index design. A Lance index covers the fragments it was built from, and data
-written after that stays unindexed: searches combine indexed results with a scan of the unindexed rows, so
-answers stay correct while latency grows, until you call `optimize_indices()` (see Lance's
-[index maintenance guide](https://lance.org/guide/indexing/#maintain-indexes-after-appends) and the
-[index format](https://lance.org/format/index/#basic-concepts)). The CSR maps onto that one to one:
+Rebuilding a CSR means reading every edge again, which takes about a second for 10 million edges. Most writes only
+add or remove a few edges, so `checkout_latest()` doesn't rebuild the CSR every time. It reads just the new data,
+keeps the added edges in a short list next to the CSR, called the *delta*, and marks deleted edges so that lookups
+skip them. Every lookup checks the delta as well as the CSR, so the new edges show up in answers straight away.
-| Lance index | CSR |
-| --- | --- |
-| The fragments an index was built from | The base CSR, at `base_version` |
-| Unindexed rows, scanned alongside the index | The delta (appended edges and tombstones), read by every lookup |
-| `num_unindexed_rows` in index statistics | `unindexed_edges` in `csr_stats()` |
-| `optimize_indices()` | [`optimize_csr()`][lance_csr.CypherEngine.optimize_csr] |
+The delta can't grow forever. Once it reaches 25% of the CSR's size, `checkout_latest()` rebuilds the CSR
+from scratch instead, which empties the delta. You don't have to manage any of this. To see what a call did, look
+at `update` in [`csr_stats()`](#csr_stats): `reused` (nothing changed), `delta` (new edges went into the delta) or
+`rebuild` (the CSR was rebuilt), as in the example above.
+
+??? info "Details: when `checkout_latest()` rebuilds"
+
+ ```mermaid
+ flowchart LR
+ R["checkout_latest()"] -- "no edge changes" --> U["reused"]
+ R -- "delta under 25% of the CSR" --> DL["delta"]
+ R -- "25% or more, overwritten, or new endpoint columns" --> RB["rebuild"]
+ ```
+
+ - Compaction and `cleanup_old_versions()` don't force a rebuild: deleted rows are matched by the edge row each
+ CSR entry records.
+ - Writes to a node table never touch the CSR, which stores node keys rather than row positions. Only that
+ label's node index is rebuilt.
-`optimize_csr()` rebuilds every CSR that carries a delta, at the version the engine is pinned to, so the base
-covers all the data again: `unindexed_edges`, `delta_edges` and `tombstones` drop to 0, `base_version` catches
-up with `version` and `update` says `rebuild`. A CSR without a delta is kept as it is, and answers never change.
+### Optional: rebuilding early with `optimize_csr()` { #rebuilding-the-csr }
+
+Most applications never need this method. `checkout_latest()` only rebuilds the CSR once the delta reaches 25%
+of the CSR's size, and until then the delta stays in place. In some situations, however, it's worth rebuilding
+sooner than that:
+
+- **Large traversals.** Checking the delta on every lookup makes big traversals a little slower (in one of our experiments, we measured 5-13% slower
+ with a delta of 20% of the CSR, on a graph with 10 million edges). Point lookups aren't affected.
+- **Stored CSRs.** With [`persist_csr=True`](#storing-the-csr), the stored file holds the CSR without its delta, so
+ every new engine re-applies the delta when it starts. A rebuild stores a fresh file, so later starts are fast
+ again.
+
+`optimize_csr()` rebuilds every CSR that has a delta, right away, instead of waiting for `checkout_latest()` to do
+it. A rebuild takes about as long as building the CSR the first time, and it uses the versions the engine already
+has, so new writes need a `checkout_latest()` first:
```python
-engine.refresh() # applies the new edges as a delta
-engine.optimize_csr() # folds it into a new base
+engine.checkout_latest() # new edges go into the delta
+engine.optimize_csr() # rebuilds the CSR with those edges included
stats = engine.csr_stats()["KNOWS"]
print(stats["update"], stats["unindexed_edges"]) # rebuild 0
```
-!!! tip "`refresh()` or `optimize_csr()`?"
+It's modelled on Lance's `optimize_indices()`, which brings Lance's own indexes up to date after new writes in the
+same way (see Lance's [indexing docs](https://lance.org/guide/indexing/#maintain-indexes-after-appends)).
+
+### Which one to call
+
+!!! tip "`checkout_latest()` or `optimize_csr()`?"
+
+ | | `checkout_latest()` | `optimize_csr()` |
+ | --- | --- | --- |
+ | What it's for | Showing new writes to your queries | Rebuilding the CSR now instead of later |
+ | When you need it | Every time your data changes | Rarely: for large traversals with a big delta, or with `persist_csr` |
+ | Cost | Usually milliseconds; occasionally a full rebuild | About as long as the first build |
+
+ A rebuild gives the same CSR whichever method does it. The two never run at the same time: if you call one
+ while the other is running, the second waits for the first to finish, and queries keep running meanwhile.
+
+## Storing the CSR
+
+Every engine builds its CSR from a full scan of the relationship tables, even when the data hasn't changed since
+the last run. With `persist_csr=True`, an engine over Lance datasets stores each CSR it builds and loads it on the
+next start instead:
+
+```python
+def open_engine():
+ lance_datasets = {"Person": lance.dataset("person.lance"), "KNOWS": lance.dataset("knows.lance")}
+ return CypherEngine(config, lance_datasets, use_csr=True, persist_csr=True)
+
+print(open_engine().csr_stats()["KNOWS"]["update"]) # build: built, then stored
+print(open_engine().csr_stats()["KNOWS"]["update"]) # loaded: read from the file
+```
+
+Each relationship dataset keeps its CSR in its own directory, one file per dataset version:
+
+```text
+knows.lance/
+├── _versions/ data/ _deletions/ _indices/ _transactions/
+└── _lance_csr/
+ └── 42.arrow
+```
+
+The file belongs to the table: dropping or copying the dataset drops or copies it, and Lance's
+`cleanup_old_versions()` leaves the folder alone, since it only deletes the data, index, deletion and transaction
+files it recognises.
+
+- **Changes since the stored version** apply as a delta, read from the new fragments only, exactly as
+ [`checkout_latest()`](#keeping-the-csr-current) applies them. `update` says `loaded` and `base_version` names the
+ stored version. That holds after compaction and `cleanup_old_versions()` too.
+- **Past the 25% threshold**, the engine rebuilds the CSR and stores the new version. Every full build is stored,
+ including those from `checkout_latest()` and [`optimize_csr()`](#rebuilding-the-csr), and older files are deleted
+ once a newer one is written.
+- **A file that can't be used** (corrupt, written for other endpoint columns or by another format) is ignored and
+ rebuilt, so a bad file costs a build, never a wrong answer.
+- **A dataset that can't be written**, because of read-only storage or credentials, is built as without the option,
+ and nothing is stored. Rust applications see the failed write as a `log` warning.
+- **PyArrow inputs** are never stored. That includes Lance data loaded into memory with `to_table()` first.
+
+To go back to building in memory only, create engines without `persist_csr` and delete the stored files once with
+[`delete_stored_csr()`][lance_csr.CypherEngine.delete_stored_csr]. It removes the `_lance_csr` folder of every
+relationship dataset the engine reads and returns how many files it deleted; the engine keeps the CSR it already
+holds, so queries carry on as before:
+
+```python
+print(open_engine().delete_stored_csr()) # 1
+```
+
+!!! note "Cost"
+
+ The file holds the CSR's arrays as they sit in memory, compressed with zstd: on SF1 about 8 bytes per edge,
+ 134 MiB in all. A load decompresses them and rebuilds only the map from key to position; node indexes are
+ still built at start-up. On the GDC SF1 graph read lazily (17.3 million edges over 23 relationship types),
+ measured on an Apple M5:
- Call `refresh()` whenever your queries should see new writes. It's cheap and incremental, and it already
- rebuilds a CSR on its own once the delta passes 25% of the base. Call `optimize_csr()` only when
- `csr_stats()` shows a delta built up below that threshold and you want lookups to stop carrying it: after a
- bulk load, before a latency-sensitive run of queries, or on a schedule at a quiet time. It scans every edge
- of the relationship dataset again, like the first build, so don't call it after every refresh.
+ | | Engine start | Peak memory |
+ | --- | ---: | ---: |
+ | Without `persist_csr` | 1.7 s | 1.37 to 1.42 GiB |
+ | First start, building and storing 134 MiB | 2.6 s | 1.25 to 1.46 GiB |
+ | Later starts, loading | 0.46 s | 1.06 to 1.12 GiB |
-`optimize_csr()` doesn't move to newer data, so call `refresh()` first. Like `refresh()`, it builds the
-replacement in full before swapping it in, and the two run one at a time.
+ Compressing adds about 0.9 s to the first start; decompression takes most of a load. Query times don't
+ change. `python/benchmarks/csr_persist.py` measures the same cases on a synthetic graph, including a load
+ with a delta.
## Should I turn it on?
diff --git a/docs/cypher.md b/docs/cypher.md
index d47a1c5..f58bfc9 100644
--- a/docs/cypher.md
+++ b/docs/cypher.md
@@ -6,7 +6,7 @@ There are two ways to run Cypher, and most code wants the first:
| --- | --- | --- |
| Use it for | Many queries over the same data | One query, once |
| Setup cost | Paid once, in the constructor | Paid on every `execute()` |
-| Lance datasets | Read lazily, pinned to a version, `refresh()` | Read into memory |
+| Lance datasets | Read lazily, pinned to a version, `checkout_latest()` | Read into memory |
| CSR index | `use_csr=True` | Not available |
| Streaming results | `execute_batches()` | No |
@@ -126,7 +126,7 @@ print(rows) # 4
- CSR expansion visits neighbours and fetches their properties a chunk at a time, including when one node
has many neighbours. See [CSR memory cost](csr.md#turning-it-on) for what it keeps in memory.
- Planning errors (a syntax error, an unknown label) raise immediately. Execution errors raise while reading.
-- The reader keeps its snapshot alive, so a concurrent [`refresh()`](lance.md#refresh) doesn't affect it.
+- The reader keeps its snapshot alive, so a concurrent [`checkout_latest()`](lance.md#checkout-latest) doesn't affect it.
- Dropping the reader cancels the remaining work.
### Query deadlines
@@ -163,7 +163,7 @@ print("CsrExpandExec" in plan) # True
| Method | Returns |
| --- | --- |
| `config()` | The engine's `GraphConfig`. |
-| `refresh()` | `None`. Moves every Lance dataset to its latest version. See [Lance datasets](lance.md#refresh). |
+| `checkout_latest()` | `None`. Moves every Lance dataset to its latest version. See [Lance datasets](lance.md#checkout-latest). |
| `dataset_versions()` | `dict[str, int]`: the pinned version of each Lance dataset. |
| `csr_stats()`, `node_index_stats()` | CSR statistics. See [The CSR index](csr.md#inspecting-the-csr). |
| `execute_with_vector_rerank(query, vector_search)` | A table reranked by vector similarity. See [Vector search](vector-search.md). |
@@ -372,7 +372,7 @@ in `__lance_csr_identity`. Relationship records also carry the node records `__l
`__lance_csr_end`. These metadata fields are distinct from user properties, even if a property has the
same name. `keys(entity)` lists only non-null user properties; `keys(map)` includes null-valued keys.
Prefer the Cypher property and endpoint functions when writing queries. Relationship identities identify actual occurrences within this query's inventory, not persistent
-IDs that can be compared across queries or refreshes. Rich paths support the same signed integer and
+IDs that can be compared across queries or snapshots. Rich paths support the same signed integer and
`uint8`/`uint16`/`uint32` keys as the CSR. Mixed labels may share compatible properties; conflicting Arrow
property types fail explicitly. Non-null node IDs must be unique within each participating label so
relationship endpoints identify one node; duplicate IDs raise an error. Expanding a graph pattern from
@@ -382,7 +382,7 @@ A path value needs a query-local inventory of the participating node and relatio
properties. It therefore materializes those tables, even with `use_csr=True`. Traversal output is emitted
in batches and named ordinary paths keep a cursor per hop, but inventory memory grows with the graph.
Rich path queries bypass the query-template cache: each execution binds its current parameters and
-creates an inventory from its own snapshot. A reader opened before `refresh()` retains the old snapshot.
+creates an inventory from its own snapshot. A reader opened before `checkout_latest()` retains the old snapshot.
Queries asking only for shortest-path length, node keys or path null checks, including homogeneous node-property
comprehensions and paths carried through `WITH` aliases, retain the usual CSR execution route when
the relationship joins the same node label. Cross-label shortest paths use typed identities even for
diff --git a/docs/execution.md b/docs/execution.md
index c4df547..4f8ca27 100644
--- a/docs/execution.md
+++ b/docs/execution.md
@@ -71,7 +71,7 @@ make it build a multi-million-row intermediate as one single-threaded hash table
lance-csr computes distinct counts for every node key and edge endpoint column (from the CSR when there is
one, otherwise with one scan of the endpoint columns) and reports them to DataFusion's join planner. The
-counts are cached per dataset version, so an unchanged `refresh()` costs nothing.
+counts are cached per dataset version, so an unchanged `checkout_latest()` costs nothing.
## Parallelism
@@ -93,14 +93,14 @@ sequenceDiagram
participant E as CypherEngine
participant Q2 as Query 2
Q1->>E: execute() takes snapshot v1
- Note over E: refresh() builds snapshot v2
(reusing unchanged parts), then swaps it in
+ Note over E: checkout_latest() builds snapshot v2
(reusing unchanged parts), then swaps it in
Q2->>E: execute() takes snapshot v2
E-->>Q1: finishes on v1
E-->>Q2: finishes on v2
```
-`refresh()` builds a complete replacement next to the current snapshot, reusing everything whose source
-dataset hasn't changed, and swaps it in atomically. A failed refresh leaves the old snapshot in place.
+`checkout_latest()` builds a complete replacement next to the current snapshot, reusing everything whose source
+dataset hasn't changed, and swaps it in atomically. A failed call leaves the old snapshot in place.
## Single-machine scope
diff --git a/docs/getting-started.md b/docs/getting-started.md
index d6028ba..3234d79 100644
--- a/docs/getting-started.md
+++ b/docs/getting-started.md
@@ -214,7 +214,7 @@ than one query against the same data.
- [Graph schema](graph-schema.md): every option of `GraphConfig`.
- [Querying with Cypher](cypher.md): parameters, streaming, `explain()` and the supported Cypher.
-- [Lance datasets](lance.md): query data on disk without loading it, and pick up new writes with `refresh()`.
+- [Lance datasets](lance.md): query data on disk without loading it, and pick up new writes with `checkout_latest()`.
- [Using from Rust](rust.md): the same engine as a Rust library.
- The [`examples/`](https://github.com/hyperdimensionalcomputingai/lance-csr/tree/main/examples) directory has
runnable scripts.
diff --git a/docs/index.md b/docs/index.md
index a27133b..ae01b96 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -78,7 +78,7 @@ each relationship hop from a hash join into an array lookup, which is what makes
---
- Lazy, version-pinned reads, `refresh()`, streaming results and LanceDB tables.
+ Lazy, version-pinned reads, `checkout_latest()`, streaming results and LanceDB tables.
[:octicons-arrow-right-24: Lance datasets](lance.md)
diff --git a/docs/internals/index.md b/docs/internals/index.md
index 2532ed4..abb3059 100644
--- a/docs/internals/index.md
+++ b/docs/internals/index.md
@@ -53,7 +53,7 @@ matters.
- **Bounded, streaming operators.** The extension operators yield bounded output batches and resume where they left
off, even inside a single high-degree node's neighbour list, and they check for cancellation as they go.
- **Immutable snapshots.** An engine holds a snapshot of pinned dataset versions plus the indexes built for them.
- `refresh()` builds a replacement next to it, reusing whatever didn't change, and swaps it in atomically, so running
+ `checkout_latest()` builds a replacement next to it, reusing whatever didn't change, and swaps it in atomically, so running
queries never see a half-updated index.
## Where the extensions plug in
diff --git a/docs/internals/indexes.md b/docs/internals/indexes.md
index 5131e99..43aa2d9 100644
--- a/docs/internals/indexes.md
+++ b/docs/internals/indexes.md
@@ -2,7 +2,7 @@
The [custom operators](operators.md) read two in-memory indexes. Both are built from the same underlying Lance dataset and point back into
it, so neither holds a copy of node or relationship properties. [The CSR index](../csr.md) covers what users see:
-how to turn it on, `csr_stats()`, and when to call `refresh()` and `optimize_csr()`. This page covers how the indexes
+how to turn it on, `csr_stats()`, and when to call `checkout_latest()` and `optimize_csr()`. This page covers how the indexes
are laid out and maintained.
The compact store of offsets and neighbours came from [lance-graph](https://github.com/lance-format/lance-graph),
@@ -39,9 +39,24 @@ each type has an outgoing and an incoming one. It is built from the relationship
- each neighbour also records the row its edge came from: its Lance row address (fragment id in the high 32 bits,
offset in the low 32), or its row number for in-memory input.
-That makes 16 bytes per edge in each direction. The edge row is what lets a hop fetch relationship properties with
-`take`, and what lets a refresh find deleted edges without reading the old dataset version. Edges with a null
-endpoint are skipped: they could never satisfy the equality join the CSR replaces.
+The edge row is what lets a hop fetch relationship properties with `take`, and what lets a refresh find deleted edges
+without reading the old dataset version. Edges with a null endpoint are skipped: they could never satisfy the equality
+join the CSR replaces.
+
+### Packed edge rows
+
+Unpacked, the neighbour and the edge row take 16 bytes per edge in each direction. Edge rows are stored packed, by
+`PackedU64`: in blocks of 128 values, a block keeps its minimum and a bit width, and its values as bit-packed offsets
+from that minimum. Neighbouring slots mostly come from nearby rows of the same fragment, so a block needs few bits; on
+GDC SF1 edge rows take about 2 bytes per edge in each direction instead of 8, which cut the CSR's arrays from 613 to
+419 MiB. Any position stays directly readable,
+which traversal needs to resume part-way through a list, and reading a range loads each block's header once. Plain
+hops never unpack edge rows; only hops that name their relationship, and refresh deletes, do.
+
+Neighbours stay a plain `u64` array. Packing them the same way roughly halved them on SF1 (their keys are sparse:
+Comment keys span 2.2 trillion for 2 million rows, so a single width for the whole array, as a `u32` offset, wouldn't
+fit at all), but every hop reads neighbours, and lookup-heavy queries got slower: custom Q30, which makes one lookup per comment
+into lists of one neighbour, by about 22%. Each lookup read a block base, a header and a word instead of one value.
## The node index
@@ -52,7 +67,7 @@ table, so edge commits update only the adjacency and node commits rebuild only t
## How a refresh diffs Lance versions
-`refresh()` compares the pinned and latest versions of each relationship dataset fragment by fragment, from their
+`checkout_latest()` compares the pinned and latest versions of each relationship dataset fragment by fragment, from their
manifests:
| Fragment | Result |
@@ -93,5 +108,34 @@ for a full rebuild.
is raised.
The base and delta map onto Lance's own index design: the delta plays the part of Lance's unindexed rows, and
-`optimize_csr()` the part of `optimize_indices()`. [Folding the delta in](../csr.md#folding-the-delta-in) has the
+`optimize_csr()` the part of `optimize_indices()`. [Rebuilding the CSR with `optimize_csr()`](../csr.md#rebuilding-the-csr) has the
full comparison.
+
+## Stored CSR files
+
+With `persist_csr=True` (see [Storing the CSR](../csr.md#storing-the-csr)), `csr_store.rs` writes the base of both
+adjacencies after every full build of a Lance relationship dataset, to `_lance_csr/.arrow` at the dataset's
+root. The file holds the CSR exactly as it sits in memory, so a load neither builds nor packs anything. It's an
+Arrow IPC file with one `UInt64` column, `values`, and twelve record batches of different lengths, one array each,
+six per direction (outgoing first):
+
+| Batch | Holds |
+| --- | --- |
+| 1 | Source keys (`i64` bit-cast) in CSR position order |
+| 2 | Offsets |
+| 3 | Neighbours |
+| 4 to 6 | Packed edge rows: block bases, block headers (start bit and width), bit-packed words |
+
+The arrays only make sense to `csr_store.rs`, which validates every one before use; reading the file with pyarrow gives
+the raw parts, not neighbour lists. The schema metadata records the format, the dataset version, the endpoint column
+names, their Lance field ids and types, and the fragments the base was built from. The delta is never stored.
+
+When an engine starts, it reads the newest file at or before the pinned version. It then diffs that file's fragments against the pinned version, as a refresh diffs two versions (see
+[How a refresh diffs Lance versions](#how-a-refresh-diffs-lance-versions)), so the file's own version doesn't need to
+exist any more. An empty diff uses the file as it is; otherwise the changes become the delta, under the same 25%
+threshold. Rebuilds after start-up (from `checkout_latest()` or `optimize_csr()`) don't look for a file, since the stored base
+is the one the engine already holds, but they do store the new one.
+
+Every batch is compressed with zstd, Arrow IPC's own buffer compression, on top of the packing. A load decompresses
+the arrays, checks them, and rebuilds only the source-key map, one insert per source. Writes go through the dataset's object store with `put_if_absent`, which
+publishes the whole file at once on local disk and object storage; the compressed file is encoded in memory first.
diff --git a/docs/internals/optimizer.md b/docs/internals/optimizer.md
index 1bec696..5fe00af 100644
--- a/docs/internals/optimizer.md
+++ b/docs/internals/optimizer.md
@@ -114,4 +114,4 @@ graph join chain, that turns a multi-million-row intermediate into an estimate o
`KeyStatisticsProvider` wraps each table and reports distinct counts for its node-id and edge-endpoint columns, taken
from the CSR when there is one and otherwise from one scan of the endpoint columns. Every other statistic comes from
-the inner table. The counts are cached per dataset version, so a `refresh()` with no changes costs nothing.
+the inner table. The counts are cached per dataset version, so a `checkout_latest()` with no changes costs nothing.
diff --git a/docs/lance.md b/docs/lance.md
index 7989c7c..dbe8fe6 100644
--- a/docs/lance.md
+++ b/docs/lance.md
@@ -36,21 +36,24 @@ print(engine.dataset_versions()) # {'Person': 1, 'KNOWS': 1}
```
Each dataset is **pinned to the version you passed in**. Writes made later, by your process or another one,
-stay invisible until you call `refresh()`. That gives every query a stable snapshot.
+stay invisible until you call `checkout_latest()`. That gives every query a stable snapshot.
!!! note "Snapshots are per dataset"
Each dataset is pinned separately. The versions across tables are not a transactional multi-table
- snapshot; if a writer updates `Person` and `KNOWS` one after the other, refresh after both commits.
+ snapshot; if a writer updates `Person` and `KNOWS` one after the other, call `checkout_latest()` after both commits.
!!! warning "Use pylance 13.x to write"
The engine is built against Lance 13. Datasets written by an older or newer major version of `pylance`
may use a file format it can't read. Installing `lance-csr` pins a compatible `pylance` for you.
-## `refresh()` { #refresh }
+## `checkout_latest()` { #checkout-latest }
-`refresh()` moves every Lance dataset to its latest version:
+`checkout_latest()` moves every Lance dataset to its latest version. It's named after Lance's own
+`LanceDataset.checkout_latest()`, which does the same for a single dataset, and it updates everything the engine
+built from that data along with it: node indexes, join statistics and, with `use_csr=True`, the CSR (see
+[Keeping the CSR current](csr.md#keeping-the-csr-current) for how the CSR keeps up):
```python
lance.write_dataset(
@@ -59,27 +62,32 @@ lance.write_dataset(
lance.write_dataset(pa.table({"src": [3], "dst": [4]}), "knows.lance", mode="append")
print(engine.execute("MATCH (p:Person) RETURN count(*) AS n").to_pylist()) # [{'n': 3}]
-engine.refresh()
+engine.checkout_latest()
print(engine.execute("MATCH (p:Person) RETURN count(*) AS n").to_pylist()) # [{'n': 4}]
print(engine.dataset_versions()) # {'Person': 2, 'KNOWS': 2}
```
-What `refresh()` guarantees:
+!!! note "Formerly `refresh()`"
+
+ This method was called `refresh()` before. `refresh()` still works, and does exactly the same, but it
+ warns with `DeprecationWarning` and will be removed in a later release.
+
+What `checkout_latest()` guarantees:
- **Unchanged datasets keep their handle.** A dataset whose version and commit time are unchanged is reused,
- along with its CSR, node index and statistics, so a no-op refresh is cheap.
-- **Schema changes are picked up.** New columns are queryable after the refresh.
+ along with its CSR, node index and statistics, so calling it when nothing changed is cheap.
+- **Schema changes are picked up.** New columns are queryable afterwards.
- **All or nothing.** The replacement is validated in full before it's swapped in. On error, nothing changes
and the previous snapshot keeps working.
-- **Concurrent refreshes cannot overwrite a newer published snapshot with an older candidate.** If another
- refresh publishes first, the overlapping refresh retries from that snapshot and reopens the latest datasets.
- The retry can add work when refreshes overlap; it does not block queries while the replacement is built.
-- **Running queries are unaffected.** Queries and `execute_batches()` readers that started before the refresh
+- **Concurrent calls cannot overwrite a newer published snapshot with an older candidate.** If another call
+ publishes first, the overlapping one retries from that snapshot and reopens the latest datasets. The retry
+ can add work when calls overlap; it does not block queries while the replacement is built.
+- **Running queries are unaffected.** Queries and `execute_batches()` readers that started before the call
finish on their original snapshot.
- **The CSR updates incrementally**, as described in [Keeping the CSR current](csr.md#keeping-the-csr-current).
- **PyArrow inputs don't change.** To replace in-memory data, build a new engine.
-A refresh reopens each dataset by its URI with the same `storage_options` it was opened with, so object-store
+`checkout_latest()` reopens each dataset by its URI with the same `storage_options` it was opened with, so object-store
credentials carry over.
## Repeated queries
@@ -91,8 +99,8 @@ rows and results are never cached. Query text and parameter names/types identify
separate from non-null values. Least recently used templates are evicted at the limit.
Lists, maps and parameterized `SKIP`/`LIMIT` counts use ordinary planning because their shape or value can
-change the plan. Queries whose operators cannot produce a reusable template fall back to ordinary planning. `explain()` plans the current values directly. A successful `refresh()` starts an empty cache for the
-new snapshot, including a refresh whose datasets did not change; active readers retain their existing plans.
+change the plan. Queries whose operators cannot produce a reusable template fall back to ordinary planning. `explain()` plans the current values directly. A successful `checkout_latest()` starts an empty cache for the
+new snapshot, even when no dataset changed; active readers retain their existing plans.
These rules apply with and without CSR. They do not promise a speedup for a particular workload.
## Query parallelism
@@ -108,13 +116,13 @@ serial_engine = CypherEngine(
target_partitions=1,
)
print(serial_engine.execute("MATCH (p:Person) RETURN count(*) AS n").to_pylist()) # [{'n': 4}]
-serial_engine.refresh() # retains target_partitions=1
+serial_engine.checkout_latest() # retains target_partitions=1
```
The target is a planning preference, not a thread or memory limit; operators may use a different number
of partitions. A lower target can help short queries while making larger joins and scans slower. Compare
your full workload in the storage mode you use before choosing a value. The default remains unchanged,
-and `refresh()` preserves your choice. Booleans and non-integers other than `None` raise `TypeError`;
+and `checkout_latest()` preserves your choice. Booleans and non-integers other than `None` raise `TypeError`;
zero, negative and out-of-range integers raise `ValueError`.
## Streaming large results
@@ -137,12 +145,12 @@ memory. Joins, sorts and aggregations may still buffer their inputs.
| --- | --- |
| Lance dataset rows | On disk; read per query, only the needed columns and rows |
| PyArrow inputs | In memory for the engine's lifetime, sliced across cores without copying |
-| CSR adjacency (`use_csr=True`) | In memory, about 70 bytes per edge for both directions |
+| CSR adjacency (`use_csr=True`) | In memory, about 20 bytes per edge for both directions plus about 37 per source node; see [the CSR's memory cost](csr.md#turning-it-on) |
| Node index (`use_csr=True`) | In memory, one entry per node row |
| Join-key statistics | In memory, a few numbers per key column |
| Query results | In memory with `execute()`; streamed with `execute_batches()` |
-The CSR and node index are built from the Lance data once and updated on `refresh()`; queries don't rescan
+The CSR and node index are built from the Lance data once and updated on `checkout_latest()`; queries don't rescan
the edge table to traverse it.
## Tables created with LanceDB
@@ -166,12 +174,12 @@ config = (
engine = CypherEngine(config, {"Person": people.to_lance(), "KNOWS": knows.to_lance()})
knows.add(new_edges) # keep writing with the LanceDB API
-engine.refresh() # the engine sees the new version
+engine.checkout_latest() # the engine sees the new version
```
- Reads are lazy and version-pinned exactly as with `lance.dataset(...)`.
- Passing the LanceDB `Table` object itself raises a `TypeError`. `table.to_arrow()` works, but loads the
- whole table into memory and never refreshes.
+ whole table into memory and never sees later versions.
- Tested with LanceDB 0.40.0 on local tables (Lance file format 2.2). A LanceDB release bundling a newer Lance
major may write files this engine can't read.
@@ -205,4 +213,4 @@ print(namespace.base_uri) # graph
```
`CypherQuery` (with a namespace or a dict) reads the data for each `execute()`; use `CypherEngine` for lazy
-reads, refresh and the CSR.
+reads, `checkout_latest()` and the CSR.
diff --git a/docs/rust.md b/docs/rust.md
index 32bcd14..a39aa19 100644
--- a/docs/rust.md
+++ b/docs/rust.md
@@ -58,7 +58,7 @@ Each input is an `EngineSource`:
`EngineSource::open_lance(uri, version, storage_options)`
: Opens a Lance dataset, pinned to `version` (`None` means the latest). The dataset is read lazily as
- queries need it, and `storage_options` are kept so `refresh()` can reopen it with the same credentials.
+ queries need it, and `storage_options` are kept so `checkout_latest()` can reopen it with the same credentials.
See [Lance datasets](lance.md).
`EngineSource::Arrow(batch)`
@@ -92,11 +92,11 @@ Each query runs on the snapshot that was current when it started.
## Pick up new writes
-`refresh()` moves every Lance dataset to its latest version and updates the CSR, applying the new edges as a
+`checkout_latest()` moves every Lance dataset to its latest version and updates the CSR, applying the new edges as a
delta when it can (see [Keeping the CSR current](csr.md#keeping-the-csr-current)):
```rust
---8<-- "crates/lance-csr/examples/embedded_query.rs:refresh"
+--8<-- "crates/lance-csr/examples/embedded_query.rs:checkout_latest"
```
`csr_stats()` and `node_index_stats()` return the same fields as in Python, as `CsrStats` and `NodeIndexStats`
@@ -118,7 +118,7 @@ different exception depending on the variant:
## Async runtimes and cancellation
-- Building the CSR in `new()` and `refresh()` is CPU-heavy work inside an async function, so it holds a
+- Building the CSR in `new()` and `checkout_latest()` is CPU-heavy work inside an async function, so it holds a
runtime worker thread until it finishes. In an application with its own runtime (a desktop or web backend),
run those two calls with `tokio::task::spawn_blocking` so other tasks keep running.
- Dropping a query's future, or the stream from `execute_stream`, cancels the rest of that query.
@@ -147,7 +147,7 @@ The deadline applies only to work inside the call; it isn't stored in the engine
| a PyArrow table in `datasets` | `EngineSource::Arrow(batch)` |
| `engine.execute(query, params)` → `pyarrow.Table` | `engine.execute(query, params).await` → `RecordBatch` |
| `engine.execute_batches(...)` → `RecordBatchReader` | `engine.execute_stream(...).await` → stream |
-| `engine.refresh()` | `engine.refresh().await` |
+| `engine.checkout_latest()` | `engine.checkout_latest().await` |
| `engine.optimize_csr()` | `engine.optimize_csr().await` |
| `engine.execute(query, timeout=2.0)` | `timeout_at(deadline, with_execution_deadline(Some(deadline), engine.execute(...)))` |
| `CypherEngine(..., target_partitions=4)` | `CypherEngine::with_target_partitions(config, sources, use_csr, Some(4)).await` |
diff --git a/docs/sql.md b/docs/sql.md
index fd5bf8d..483453c 100644
--- a/docs/sql.md
+++ b/docs/sql.md
@@ -55,7 +55,7 @@ print(friends.to_pylist())
The SQL dialect is DataFusion's; see the
[DataFusion SQL reference](https://datafusion.apache.org/user-guide/sql/index.html) for what it supports.
-`SqlEngine` has no `refresh()` or CSR: those are `CypherEngine` features.
+`SqlEngine` has no `checkout_latest()` or CSR: those are `CypherEngine` features.
## From Cypher to SQL
diff --git a/docs/sync_benchmarks.py b/docs/sync_benchmarks.py
index 3ba1e94..5d4717c 100644
--- a/docs/sync_benchmarks.py
+++ b/docs/sync_benchmarks.py
@@ -1,4 +1,4 @@
-"""Rewrite the benchmark tables in README.md and docs/benchmarks.md from the benchmark repositories.
+"""Rewrite the benchmark tables in docs/benchmarks.md from the benchmark repositories.
Each benchmark repository commits pytest-benchmark's output for every engine to `results/`, with
the machine and date in `results/environment.yml`. This script downloads those files at the
@@ -25,7 +25,7 @@
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
-TARGETS = [ROOT / "README.md", ROOT / "docs" / "benchmarks.md"]
+TARGETS = [ROOT / "docs" / "benchmarks.md"]
# Columns: (results file stem, column title). The first two are lance-csr's modes; the rest are
# the systems it is compared with.
diff --git a/examples/README.md b/examples/README.md
index de2e6d9..6d61a34 100644
--- a/examples/README.md
+++ b/examples/README.md
@@ -8,7 +8,7 @@ dependencies):
```bash
uv run --with lance-csr python examples/basic_cypher.py
uv run --with lance-csr python examples/kg_traversal.py
-uv run --with lance-csr python examples/lance_snapshot_refresh.py
+uv run --with lance-csr python examples/lance_snapshots.py
```
With a local build, so the examples use your changes, set up `python/.venv` as in
@@ -35,8 +35,8 @@ uv run --with lance-csr python examples/basic_cypher.py
uv run --with lance-csr python examples/kg_traversal.py
```
-- `lance_snapshot_refresh.py`: lazy Lance datasets, snapshots and `refresh()`.
+- `lance_snapshots.py`: lazy Lance datasets, snapshots and `checkout_latest()`.
```bash
-uv run --with lance-csr python examples/lance_snapshot_refresh.py
+uv run --with lance-csr python examples/lance_snapshots.py
```
diff --git a/examples/lance_snapshot_refresh.py b/examples/lance_snapshots.py
similarity index 84%
rename from examples/lance_snapshot_refresh.py
rename to examples/lance_snapshots.py
index efae887..333afa0 100644
--- a/examples/lance_snapshot_refresh.py
+++ b/examples/lance_snapshots.py
@@ -1,11 +1,11 @@
"""
-Disk-backed CypherEngine: pinned snapshots, refresh, and streamed results.
+Disk-backed CypherEngine: pinned snapshots, checkout_latest(), and streamed results.
Requirements: an installed lance-csr wheel (or `maturin develop` in python/), which
pulls in a compatible pylance 13.x.
Run:
- python examples/lance_snapshot_refresh.py
+ python examples/lance_snapshots.py
"""
from __future__ import annotations
@@ -46,7 +46,7 @@ def main() -> None:
"Person": lance.dataset(person_uri),
"KNOWS": lance.dataset(root / "knows.lance"),
},
- # Optional: choose this target after measuring your own queries; refresh keeps it.
+ # Optional: choose this target after measuring your own queries; checkout_latest() keeps it.
target_partitions=2,
)
count = "MATCH (p:Person) RETURN count(*) AS n"
@@ -59,13 +59,13 @@ def main() -> None:
).to_pylist(),
)
- # Writes are not visible until refresh(): the engine is pinned to version 1.
+ # Writes are not visible until checkout_latest(): the engine is pinned to version 1.
lance.write_dataset(people(ROWS, ROWS + 10), person_uri, mode="append")
lance.dataset(person_uri).add_columns({"age": "id % 100"}) # schema change
- print("after writes, before refresh:", engine.execute(count).column(0)[0].as_py())
- engine.refresh()
+ print("after writes, before checkout_latest():", engine.execute(count).column(0)[0].as_py())
+ engine.checkout_latest()
print(
- "after refresh:",
+ "after checkout_latest():",
engine.execute(count).column(0)[0].as_py(),
engine.dataset_versions(),
)
diff --git a/python/README.md b/python/README.md
index 8db3c86..471d200 100644
--- a/python/README.md
+++ b/python/README.md
@@ -110,11 +110,11 @@ lets relationship-type alternation (`[:likes_post|likes_comment]`) pick the righ
`with_node_label` and `with_relationship`, the names in lance-csr 0.1, were removed in
0.5.0; use `node_table` and `rel_table` instead.
-### 3. Lance datasets: snapshots, refresh and streaming
+### 3. Lance datasets: snapshots, checking out new versions and streaming
`CypherEngine` reads Lance datasets lazily from storage (projections and filters are pushed into the
scan). Each dataset is pinned to the version you pass in; later writes are visible only after
-`refresh()`. Use `execute_batches()` to stream large results instead of materializing a table.
+`checkout_latest()`. Use `execute_batches()` to stream large results instead of materializing a table.
```python
import lance
@@ -124,15 +124,15 @@ config = GraphConfig.builder().node_table("Person", "id").build()
engine = CypherEngine(config, {"Person": lance.dataset("people.lance")})
engine.execute("MATCH (p:Person) RETURN count(*)") # sees the pinned version
-engine.refresh() # move to the latest version
+engine.checkout_latest() # move to the latest version
for batch in engine.execute_batches("MATCH (p:Person) RETURN p.id"):
... # pyarrow.RecordBatch, one at a time
```
Experimental: `CypherEngine(config, datasets, use_csr=True)` answers single-hop patterns over integer ids
with an in-memory CSR adjacency, and fetches target rows by position instead of joining. The CSR is
-rebuilt on `refresh()` when the relationship table changes, and the per-label id index when its node
-table changes. Results are identical; peak memory grows by about 70 bytes per edge (both directions).
+rebuilt on `checkout_latest()` when the relationship table changes, and the per-label id index when its node
+table changes. Results are identical; peak memory grows by about 95 bytes per edge while the CSR is built (both directions).
Write Lance data with `pylance` 13.x: the engine is built against Lance 13 and older or newer
majors may write files it cannot read.
@@ -158,11 +158,11 @@ config = (
engine = CypherEngine(config, {"Person": people.to_lance(), "KNOWS": knows.to_lance()})
knows.add(new_edges) # keep writing with the LanceDB API
-engine.refresh() # the engine sees the new version
+engine.checkout_latest() # the engine sees the new version
```
- Reads are lazy and pinned to the version you pass in, exactly as with `lance.dataset(...)`.
-- `table.to_arrow()` also works, but loads the whole table into memory and never refreshes.
+- `table.to_arrow()` also works, but loads the whole table into memory and never sees later versions.
- Tested with LanceDB 0.40.0 on local tables, which write Lance file format 2.2. A LanceDB release that
bundles a newer Lance major may write files this engine cannot read.
- Not supported: LanceDB Cloud (remote tables have no `to_lance()`), and tables opened through a
diff --git a/python/benchmarks/csr_delta.py b/python/benchmarks/csr_delta.py
index e74ccbe..427bab7 100644
--- a/python/benchmarks/csr_delta.py
+++ b/python/benchmarks/csr_delta.py
@@ -1,14 +1,14 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The Lance Authors
-"""Incremental CSR maintenance: refresh-with-delta cost against a full rebuild.
+"""Incremental CSR maintenance: ``checkout_latest()``-with-delta cost against a full rebuild.
Uses the graphs written by ``csr_expand.py`` (``--data`` must point at the same
directory). For each scenario, on a fresh copy of the edge table (1M-row
fragments, plus a ``seq`` row-number column for targeted deletes):
- apply one commit (append or delete);
-- time ``refresh()`` on a CSR engine built before the commit (delta path);
+- time ``checkout_latest()`` on a CSR engine built before the commit (delta path);
- time a full CSR build on the new version (what a rebuild costs);
- compare warm query latency with the delta in place against the clean build,
checking both return identical results.
@@ -116,7 +116,7 @@ def main() -> None:
engine = CypherEngine(CONFIG, datasets, use_csr=True)
change(uri, nodes)
- refresh_ms, _ = timed(engine.refresh)
+ refresh_ms, _ = timed(engine.checkout_latest)
stats = engine.csr_stats()["KNOWS"]
latest = {"Person": person, "KNOWS": lance.dataset(uri)}
diff --git a/python/benchmarks/csr_delta_accumulated.py b/python/benchmarks/csr_delta_accumulated.py
index e2dcf73..5dbfac2 100644
--- a/python/benchmarks/csr_delta_accumulated.py
+++ b/python/benchmarks/csr_delta_accumulated.py
@@ -4,7 +4,7 @@
"""Refresh cost as CSR deltas accumulate over many small commits.
Uses a graph written by ``csr_expand.py``. On a fresh copy of the edge table,
-appends ``--step`` of the edges ``--commits`` times, calling ``refresh()`` after
+appends ``--step`` of the edges ``--commits`` times, calling ``checkout_latest()`` after
each, and prints each refresh's time with the delta size so far. With deltas that
are copied on every refresh, the time grows with the accumulated delta; with
shared delta layers it should stay roughly proportional to the step.
@@ -59,7 +59,7 @@ def main() -> None:
)
lance.write_dataset(new, uri, mode="append")
start = time.perf_counter()
- engine.refresh()
+ engine.checkout_latest()
refresh_ms = (time.perf_counter() - start) * 1e3
stats = engine.csr_stats()["KNOWS"]
print(
diff --git a/python/benchmarks/csr_expand.py b/python/benchmarks/csr_expand.py
index 6ef4e92..5134b51 100644
--- a/python/benchmarks/csr_expand.py
+++ b/python/benchmarks/csr_expand.py
@@ -124,7 +124,7 @@ def open_engine():
out[name] = {**check, **summary(samples)}
t = time.perf_counter()
- engine.refresh()
+ engine.checkout_latest()
out["refresh_unchanged_ms"] = (time.perf_counter() - t) * 1e3
# Recorded before the copy below, which materializes the edge table.
out["peak_rss_mib"] = peak_rss_mib()
@@ -149,7 +149,7 @@ def open_engine():
)
lance.write_dataset(new_edges, knows, mode="append")
t = time.perf_counter()
- engine.refresh()
+ engine.checkout_latest()
out["refresh_after_append_ms"] = (time.perf_counter() - t) * 1e3
if mode == "csr":
out["csr_rebuild_ms"] = engine.csr_stats()["KNOWS"]["build_ms"]
diff --git a/python/benchmarks/csr_persist.py b/python/benchmarks/csr_persist.py
new file mode 100644
index 0000000..844069e
--- /dev/null
+++ b/python/benchmarks/csr_persist.py
@@ -0,0 +1,152 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The Lance Authors
+
+"""Engine start-up with a stored CSR (``persist_csr=True``) against building it.
+
+For each edge count, writes a deterministic random graph to Lance (nodes =
+edges / AVG_DEGREE), then runs each case in a fresh subprocess on its own copy
+of the relationship dataset:
+
+- ``off``: ``persist_csr=False``, today's build;
+- ``first``: ``persist_csr=True`` with nothing stored, so a build plus the write;
+- ``loaded``: ``persist_csr=True`` with the file from ``first``;
+- ``loaded_delta``: as ``loaded``, after appending 1% more edges and deleting about
+ 1% of the rows, so the changes since the stored version apply as a delta.
+
+Each case reports construction time, the CSR's ``build_ms`` and ``update``, peak
+RSS, the size of ``_lance_csr/`` and warm latency of a few queries, whose row
+counts and checksums must match across cases.
+
+ python benchmarks/csr_persist.py --data /tmp/csr-persist --edges 1000000 10000000
+"""
+
+import argparse
+import json
+import resource
+import shutil
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+from csr_expand import AVG_DEGREE, generate, summary
+
+CASES = ("off", "first", "loaded", "loaded_delta")
+QUERIES = {
+ "point": "MATCH (a:Person)-[:KNOWS]->(b:Person) WHERE a.id = 42 RETURN b.id",
+ "two_hop": (
+ "MATCH (a:Person)-[:KNOWS]->(:Person)-[:KNOWS]->(c:Person) "
+ "WHERE a.id < 100 RETURN count(c) AS n"
+ ),
+ "all_edges": "MATCH (a:Person)-[:KNOWS]->(b:Person) RETURN count(*) AS n",
+}
+
+
+def peak_rss_mib() -> float:
+ peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
+ return (peak if sys.platform == "darwin" else peak * 1024) / 2**20
+
+
+def folder_mib(path: Path) -> float:
+ return sum(f.stat().st_size for f in path.glob("*")) / 2**20 if path.exists() else 0
+
+
+def child(root: Path, knows: Path, case: str, repeats: int) -> dict:
+ import lance
+ import pyarrow.compute as pc
+ from lance_csr import CypherEngine, GraphConfig
+
+ config = (
+ GraphConfig.builder()
+ .node_table("Person", "id")
+ .rel_table("KNOWS", "Person", "Person", "src", "dst")
+ .build()
+ )
+ datasets = {
+ "Person": lance.dataset(root / "person.lance"),
+ "KNOWS": lance.dataset(knows),
+ }
+ kwargs = {} if case == "off" else {"persist_csr": True}
+ start = time.perf_counter()
+ engine = CypherEngine(config, datasets, use_csr=True, **kwargs)
+ out = {"case": case, "construct_ms": (time.perf_counter() - start) * 1e3}
+ stats = engine.csr_stats()["KNOWS"]
+ out |= {k: stats[k] for k in ("update", "build_ms", "edges", "delta_edges")}
+ out["node_index_build_ms"] = engine.node_index_stats()["Person"]["build_ms"]
+ out["peak_rss_mib"] = peak_rss_mib()
+ out["stored_mib"] = folder_mib(knows / "_lance_csr")
+ for name, query in QUERIES.items():
+ result = engine.execute(query)
+ check = {"rows": result.num_rows, "checksum": pc.sum(result.column(0)).as_py()}
+ samples = []
+ for _ in range(repeats):
+ t = time.perf_counter()
+ engine.execute(query)
+ samples.append(time.perf_counter() - t)
+ out[name] = {**check, **summary(samples)}
+ return out
+
+
+def change(knows: Path) -> None:
+ """Append 1% more edges and delete about 1% of the rows."""
+ import lance
+ import numpy as np
+ import pyarrow as pa
+
+ ds = lance.dataset(knows)
+ rows = ds.count_rows()
+ nodes = rows // AVG_DEGREE
+ rng = np.random.default_rng(11)
+ added = pa.table(
+ {
+ "src": rng.integers(0, nodes, rows // 100, dtype=np.int64),
+ "dst": rng.integers(0, nodes, rows // 100, dtype=np.int64),
+ }
+ )
+ lance.write_dataset(added, knows, mode="append")
+ lance.dataset(knows).delete(f"src < {nodes // 100}")
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--data", type=Path, required=True)
+ parser.add_argument("--edges", type=int, nargs="+", default=[1_000_000])
+ parser.add_argument("--repeats", type=int, default=20)
+ parser.add_argument("--cases", nargs="+", choices=CASES, default=list(CASES))
+ parser.add_argument("--child", nargs=3, metavar=("ROOT", "KNOWS", "CASE"))
+ args = parser.parse_args()
+
+ if args.child:
+ root, knows, case = args.child
+ print(json.dumps(child(Path(root), Path(knows), case, args.repeats)))
+ return
+
+ for edges in args.edges:
+ root = args.data / f"e{edges}"
+ root.mkdir(parents=True, exist_ok=True)
+ generate(root, edges)
+ # `first` writes into this copy; `loaded` and `loaded_delta` read it back.
+ knows = root / "knows-persist.lance"
+ shutil.rmtree(knows, ignore_errors=True)
+ shutil.copytree(root / "knows.lance", knows)
+ results = {}
+ for case in args.cases:
+ if case == "loaded_delta":
+ change(knows)
+ target = root / "knows.lance" if case == "off" else knows
+ cmd = [sys.executable, __file__, "--data", str(args.data)]
+ cmd += ["--repeats", str(args.repeats), "--child", str(root), str(target), case]
+ proc = subprocess.run(cmd, capture_output=True, text=True)
+ if proc.returncode != 0:
+ print(json.dumps({"edges": edges, "case": case, "error": proc.stderr[-2000:]}))
+ continue
+ results[case] = json.loads(proc.stdout.strip().splitlines()[-1])
+ print(json.dumps({"edges": edges, **results[case]}), flush=True)
+ same = [c for c in ("off", "first", "loaded") if c in results]
+ for name in QUERIES:
+ answers = {c: (results[c][name]["rows"], results[c][name]["checksum"]) for c in same}
+ assert len(set(answers.values())) <= 1, (edges, name, answers)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/python/benchmarks/engine_cache.py b/python/benchmarks/engine_cache.py
index 05a1ff2..ab32a06 100644
--- a/python/benchmarks/engine_cache.py
+++ b/python/benchmarks/engine_cache.py
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The Lance Authors
-"""Engine construction, first-query, warm-query and refresh timings.
+"""Engine construction, first-query, warm-query and checkout_latest() timings.
Runs against either the upstream build (``--module lance_graph``) or this fork
(``--module lance_csr``) on the same on-disk Lance data, so the two can be compared
@@ -112,11 +112,13 @@ def emit(case, **values):
warm.append(t)
emit("warm_query", **summary(warm))
- # Upstream has no refresh(); its equivalent is building a new engine.
+ # Upstream has no checkout_latest() (refresh() in older lance-csr); its equivalent
+ # is building a new engine.
refresh = []
+ checkout = getattr(engine, "checkout_latest", None) or getattr(engine, "refresh", None)
for _ in range(min(args.repeats, 10)):
- if hasattr(engine, "refresh"):
- t, _ = timed(engine.refresh)
+ if checkout is not None:
+ t, _ = timed(checkout)
else:
t, engine = timed(lambda: mod.CypherEngine(config, open_datasets()))
refresh.append(t)
diff --git a/python/python/lance_csr/_internal/graph.pyi b/python/python/lance_csr/_internal/graph.pyi
index a82cc84..64b74f0 100644
--- a/python/python/lance_csr/_internal/graph.pyi
+++ b/python/python/lance_csr/_internal/graph.pyi
@@ -34,7 +34,7 @@ class _CsrStats(TypedDict):
delta_edges: int
tombstones: int
unindexed_edges: int
- update: Literal["build", "rebuild", "delta", "reused"]
+ update: Literal["build", "rebuild", "loaded", "delta", "reused"]
version: int | None
base_version: int | None
build_ms: float
@@ -114,6 +114,7 @@ class CypherEngine:
use_csr: bool = False,
*,
target_partitions: int | None = None,
+ persist_csr: bool = False,
) -> None: ...
def execute(
self,
@@ -132,8 +133,10 @@ class CypherEngine:
def explain(
self, query: str, params: dict[str, _ParameterT] | None = None
) -> str: ...
+ def checkout_latest(self) -> None: ...
def refresh(self) -> None: ...
def optimize_csr(self) -> None: ...
+ def delete_stored_csr(self) -> int: ...
def node_index_stats(self) -> dict[str, _NodeIndexStats]: ...
def csr_stats(self) -> dict[str, _CsrStats]: ...
def dataset_versions(self) -> dict[str, int]: ...
diff --git a/python/python/lance_csr/_types.py b/python/python/lance_csr/_types.py
index 34a9c5a..6dcfdfe 100644
--- a/python/python/lance_csr/_types.py
+++ b/python/python/lance_csr/_types.py
@@ -54,7 +54,7 @@ class CsrStats(TypedDict):
delta_edges: int
tombstones: int
unindexed_edges: int
- update: Literal["build", "rebuild", "delta", "reused"]
+ update: Literal["build", "rebuild", "loaded", "delta", "reused"]
version: int | None
base_version: int | None
build_ms: float
diff --git a/python/python/lance_csr/cypher.py b/python/python/lance_csr/cypher.py
index 0130897..a2b795a 100644
--- a/python/python/lance_csr/cypher.py
+++ b/python/python/lance_csr/cypher.py
@@ -2,6 +2,7 @@
from __future__ import annotations
+import warnings
from enum import IntEnum
from typing import TYPE_CHECKING, Any, Self, final
@@ -47,9 +48,9 @@ class CypherEngine:
Lance datasets are read lazily, pinned to the version passed in: later writes, by
this or another process, aren't visible until
- [`refresh`][lance_csr.CypherEngine.refresh]. Each dataset is pinned separately, so
- versions across tables are not a transactional snapshot. PyArrow inputs are held in
- memory and never change; create a new engine to replace them.
+ [`checkout_latest`][lance_csr.CypherEngine.checkout_latest]. Each dataset is pinned
+ separately, so versions across tables are not a transactional snapshot. PyArrow
+ inputs are held in memory and never change; create a new engine to replace them.
[`execute`][lance_csr.CypherEngine.execute] and
[`execute_batches`][lance_csr.CypherEngine.execute_batches] reuse up to 128
@@ -58,7 +59,8 @@ class CypherEngine:
call. Container parameters, rich path values, `CALL` subqueries, parameterized
`SKIP`/`LIMIT` and unsupported template shapes use ordinary planning.
[`explain`][lance_csr.CypherEngine.explain] always plans directly, and a successful
- `refresh` clears the cache. See [Repeated queries](lance.md#repeated-queries).
+ `checkout_latest` clears the cache. See [Repeated
+ queries](lance.md#repeated-queries).
Parameters
----------
@@ -73,15 +75,25 @@ class CypherEngine:
endpoints, and node indexes for labels with unique integer keys.
target_partitions : int or None
Keyword-only. A target from 1 to 1024 for DataFusion query partitioning; `None`
- keeps DataFusion's default. `refresh` preserves it. It is not a thread or memory
- limit. See [Query parallelism](lance.md#query-parallelism).
+ keeps DataFusion's default. `checkout_latest` preserves it. It is not a thread
+ or memory limit. See [Query parallelism](lance.md#query-parallelism).
+ persist_csr : bool
+ Keyword-only, experimental, needs `use_csr=True`. Store each relationship
+ dataset's CSR inside the dataset's own directory, at
+ `_lance_csr/.arrow`, and load it instead of building when an engine
+ starts. Changes since the stored version apply as a delta under the same 25%
+ threshold as `checkout_latest`; past it, the CSR is rebuilt and stored again.
+ Lance's `cleanup_old_versions` leaves the folder alone. PyArrow inputs aren't
+ stored, and a dataset that can't be written (read-only storage or credentials)
+ is built as without it. See [Storing the CSR](csr.md#storing-the-csr).
Raises
------
ValueError
If `datasets` is empty, a configured key or endpoint column is missing, a Lance
- dataset was opened through a namespace client and can't be reopened by URI, or
- `target_partitions` is outside 1 to 1024.
+ dataset was opened through a namespace client and can't be reopened by URI,
+ `target_partitions` is outside 1 to 1024, or `persist_csr` is set without
+ `use_csr`.
TypeError
If `config` isn't a `GraphConfig`, a dataset has an unsupported type, or
`target_partitions` is a boolean or another non-integer other than `None`.
@@ -118,12 +130,14 @@ def __init__[DatasetT: Dataset](
use_csr: bool = False,
*,
target_partitions: int | None = None,
+ persist_csr: bool = False,
) -> None:
self._inner = _graph.CypherEngine(
inner(config, GraphConfig, "config"),
datasets,
use_csr,
target_partitions=target_partitions,
+ persist_csr=persist_csr,
)
def __repr__(self) -> str:
@@ -188,8 +202,8 @@ def execute_batches[ParameterT: Parameter](
Batches are computed as the reader is consumed. Scans, filters and projections
stream with bounded memory; joins, sorts and aggregations may still buffer. The
reader keeps its snapshot alive, so a concurrent
- [`refresh`][lance_csr.CypherEngine.refresh] doesn't affect it. Planning errors
- raise immediately; execution errors raise while reading.
+ [`checkout_latest`][lance_csr.CypherEngine.checkout_latest] doesn't affect it.
+ Planning errors raise immediately; execution errors raise while reading.
Parameters
----------
@@ -245,39 +259,73 @@ def explain[ParameterT: Parameter](
"""
return self._inner.explain(query, params)
- def refresh(self) -> None:
+ def checkout_latest(self) -> None:
"""Move every Lance dataset to its latest version.
- Updates the CSR, node indexes and statistics to match. Datasets whose version
- and commit time are unchanged keep their open handle. The replacement is
- validated in full before it's swapped in: on error nothing changes and the
- previous snapshot stays usable. Queries already running finish on the snapshot
- they started with. PyArrow inputs are unchanged. Concurrent calls run one at a
- time, and `target_partitions` is preserved. See
- [Lance datasets](lance.md#refresh).
+ The engine's counterpart of Lance's `LanceDataset.checkout_latest()`, for every
+ dataset at once. Updates the CSR, node indexes and statistics to match. Datasets
+ whose version and commit time are unchanged keep their open handle. The
+ replacement is validated in full before it's swapped in: on error nothing
+ changes and the previous snapshot stays usable. Queries already running finish
+ on the snapshot they started with. PyArrow inputs are unchanged. Concurrent
+ calls run one at a time, and `target_partitions` is preserved. See [Lance
+ datasets](lance.md#checkout-latest).
+ """
+ self._inner.checkout_latest()
+
+ def refresh(self) -> None:
+ """Deprecated alias of `checkout_latest`.
+
+ Behaves exactly like
+ [`checkout_latest`][lance_csr.CypherEngine.checkout_latest], and warns with
+ `DeprecationWarning`.
"""
- self._inner.refresh()
+ warnings.warn(
+ "CypherEngine.refresh() is deprecated; use checkout_latest(), named after "
+ "Lance's LanceDataset.checkout_latest()",
+ DeprecationWarning,
+ stacklevel=2,
+ )
+ self.checkout_latest()
def optimize_csr(self) -> None:
- """Fold each CSR's delta into a new base (experimental; needs `use_csr=True`).
+ """Rebuild each CSR that carries a delta (experimental; needs `use_csr=True`).
The CSR's counterpart of Lance's `optimize_indices()`.
- [`refresh`][lance_csr.CypherEngine.refresh] applies Lance writes as a delta
- over the base CSR, and rebuilds on its own only once pending changes pass 25%
- of the base. This rebuilds every CSR that carries a delta now, at the version
- the engine is pinned to, so lookups stop carrying it. It scans every edge
- again, so call it after a bulk load or at a quiet time, not after every
- refresh. It doesn't move to
- newer versions: call `refresh` first for that. CSRs without a delta, and
- PyArrow inputs, are left as they are, and query results never change.
- Afterwards [`csr_stats`][lance_csr.CypherEngine.csr_stats] reports `update`
- `"rebuild"`, `unindexed_edges` 0 and `base_version` equal to `version`. As for
- `refresh`, the replacement is validated in full before it's swapped in, and the
- two run one at a time. See
- [Folding the delta in](csr.md#folding-the-delta-in).
+ [`checkout_latest`][lance_csr.CypherEngine.checkout_latest] applies Lance writes
+ as a delta over the base CSR, and rebuilds on its own only once pending changes
+ pass 25% of the base. This rebuilds every CSR that carries a delta now, at the
+ version the engine is pinned to, so lookups stop carrying it. It scans every
+ edge again, so call it after a bulk load or at a quiet time, not after every
+ `checkout_latest`. It doesn't move to newer versions: call `checkout_latest`
+ first for that. CSRs without a delta, and PyArrow inputs, are left as they are,
+ and query results never change. Afterwards
+ [`csr_stats`][lance_csr.CypherEngine.csr_stats] reports `update` `"rebuild"`,
+ `unindexed_edges` 0 and `base_version` equal to `version`. As for
+ `checkout_latest`, the replacement is validated in full before it's swapped in,
+ and the two run one at a time. See [Rebuilding the
+ CSR](csr.md#rebuilding-the-csr).
"""
self._inner.optimize_csr()
+ def delete_stored_csr(self) -> int:
+ """Delete the CSR files stored with `persist_csr=True` (experimental).
+
+ Removes the `_lance_csr` folder of every Lance relationship dataset this
+ engine reads, whether or not the engine itself was created with
+ `persist_csr`. The engine keeps the CSR it already holds in memory, so queries
+ aren't affected. To go back to in-memory builds only, create later engines
+ without `persist_csr`: an engine with it stores a new file at its next full
+ build. Runs one at a time with `checkout_latest` and `optimize_csr`. See
+ [Storing the CSR](csr.md#storing-the-csr).
+
+ Returns
+ -------
+ int
+ The number of files deleted; 0 when nothing was stored.
+ """
+ return self._inner.delete_stored_csr()
+
def node_index_stats(self) -> dict[str, NodeIndexStats]:
"""Statistics for each node index (experimental; needs `use_csr=True`).
@@ -295,9 +343,9 @@ def node_index_stats(self) -> dict[str, NodeIndexStats]:
def csr_stats(self) -> dict[str, CsrStats]:
"""Statistics for each CSR adjacency index (experimental; needs `use_csr=True`).
- An identical result after [`refresh`][lance_csr.CypherEngine.refresh] means the
- snapshot itself didn't change. See
- [Inspecting the CSR](csr.md#inspecting-the-csr).
+ An identical result after
+ [`checkout_latest`][lance_csr.CypherEngine.checkout_latest] means the snapshot
+ itself didn't change. See [Inspecting the CSR](csr.md#inspecting-the-csr).
Returns
-------
@@ -309,10 +357,11 @@ def csr_stats(self) -> dict[str, CsrStats]:
full build, held outside the base CSR.
- `unindexed_edges`: live edges outside the base CSR, appended since its
last full build and not since removed.
- - `update`: how the last refresh or
+ - `update`: how the last build, `checkout_latest` or
[`optimize_csr`][lance_csr.CypherEngine.optimize_csr] produced it:
- `"build"` (first), `"rebuild"` (full), `"delta"` (changes applied) or
- `"reused"` (no edge change).
+ `"build"` (first), `"rebuild"` (full), `"loaded"` (from a file stored with
+ `persist_csr=True`, with any changes since applied as a delta), `"delta"`
+ (changes applied) or `"reused"` (no edge change).
- `version`: the relationship dataset version it reflects (`None` for
PyArrow).
- `base_version`: the relationship dataset version of the last full
diff --git a/python/python/tests/test_call_subqueries.py b/python/python/tests/test_call_subqueries.py
index 9d4a498..454dfe8 100644
--- a/python/python/tests/test_call_subqueries.py
+++ b/python/python/tests/test_call_subqueries.py
@@ -413,7 +413,7 @@ def test_rich_path_reader_retains_snapshot_after_refresh(graph, tmp_path):
lance.write_dataset(
pa.table({"src": [20], "dst": [30]}), paths["KNOWS"], mode="append"
)
- engine.refresh()
+ engine.checkout_latest()
assert engine.execute(query, {"start": 20}).column(0).to_pylist() == [
[20, 30],
[20, 40],
diff --git a/python/python/tests/test_engine_csr.py b/python/python/tests/test_engine_csr.py
index 28aa28e..edb3c59 100644
--- a/python/python/tests/test_engine_csr.py
+++ b/python/python/tests/test_engine_csr.py
@@ -285,7 +285,7 @@ def test_csr_tracks_edge_changes_but_not_node_changes(tmp_path):
tmp_path / "person.lance",
mode="append",
)
- engine.refresh()
+ engine.checkout_latest()
assert engine.csr_stats() == before
new_edge = pa.table(
@@ -297,7 +297,7 @@ def test_csr_tracks_edge_changes_but_not_node_changes(tmp_path):
)
lance.write_dataset(new_edge, tmp_path / "knows.lance", mode="append")
assert engine.execute(query).num_rows == 0 # pinned snapshot
- engine.refresh()
+ engine.checkout_latest()
after = engine.csr_stats()
assert after["KNOWS"]["version"] == 2 and after["KNOWS"]["edges"] == 8
assert engine.execute(query).column(0).to_pylist() == [1]
@@ -337,7 +337,7 @@ def test_high_degree_stream_keeps_its_snapshot_across_deletions(tmp_path):
assert 0 < first.num_rows < n
# A live reader retains the old adjacency, including both copies of 0->1.
knows.delete("dst = 1")
- engine.refresh()
+ engine.checkout_latest()
assert engine.csr_stats()["KNOWS"]["tombstones"] == 2
counts = {}
for batch in chain([first], reader):
@@ -429,7 +429,7 @@ def test_node_index_rebuilt_only_when_node_version_changes(tmp_path):
}
)
lance.write_dataset(edge, tmp_path / "knows.lance", mode="append")
- engine.refresh()
+ engine.checkout_latest()
assert engine.node_index_stats() == before
query = "MATCH (a:Person)-[:KNOWS]->(b:Person) WHERE a.id = 6 RETURN b.name"
assert engine.execute(query).num_rows == 0 # node 7 does not exist yet
@@ -439,7 +439,7 @@ def test_node_index_rebuilt_only_when_node_version_changes(tmp_path):
tmp_path / "person.lance",
mode="append",
)
- engine.refresh()
+ engine.checkout_latest()
after = engine.node_index_stats()
assert after["Person"]["version"] == 2 and after["Person"]["rows"] == 7
assert engine.execute(query).column(0).to_pylist() == ["g"]
diff --git a/python/python/tests/test_engine_csr_delta.py b/python/python/tests/test_engine_csr_delta.py
index 6c928eb..345b222 100644
--- a/python/python/tests/test_engine_csr_delta.py
+++ b/python/python/tests/test_engine_csr_delta.py
@@ -3,7 +3,7 @@
"""Incremental CSR maintenance: edge commits applied as deltas and tombstones.
-After each change and ``refresh()``, the CSR engine must return exactly what a
+After each change and ``checkout_latest()``, the CSR engine must return exactly what a
fresh join-path engine and a freshly built CSR engine return on the new data.
"""
@@ -68,7 +68,7 @@ def make(**write_kwargs):
def check(engine, knows_uri):
- """CSR engine (after refresh) == fresh join engine == fresh CSR engine."""
+ """CSR engine (after checkout_latest) == fresh join engine == fresh CSR engine."""
datasets = {
"Person": lance.dataset(knows_uri.parent / "person.lance"),
"KNOWS": lance.dataset(knows_uri),
@@ -95,7 +95,7 @@ def test_initial_build_reports_build(graph):
def test_append_applied_as_delta(graph):
engine, uri = graph()
lance.write_dataset(edges([(1, 2), (39, 0), (5, 5)], 1000), uri, mode="append")
- engine.refresh()
+ engine.checkout_latest()
s = stats(engine)
assert (s["update"], s["delta_edges"], s["tombstones"]) == ("delta", 3, 0)
assert s["edges"] == 203 and s["unindexed_edges"] == 3
@@ -106,7 +106,7 @@ def test_append_applied_as_delta(graph):
def test_delete_applied_as_tombstones(graph):
engine, uri = graph()
lance.dataset(uri).delete("since % 9 = 0")
- engine.refresh()
+ engine.checkout_latest()
s = stats(engine)
assert s["update"] == "delta" and s["tombstones"] == 23 and s["delta_edges"] == 0
check(engine, uri)
@@ -116,7 +116,7 @@ def test_delete_one_of_duplicate_edges(graph):
engine, uri = graph()
# since 190..199 are the duplicated i -> i+1 edges; delete one copy of 2 -> 3.
lance.dataset(uri).delete("since = 192")
- engine.refresh()
+ engine.checkout_latest()
query = "MATCH (a:Person)-[:KNOWS]->(b:Person) WHERE a.id = 2 RETURN b.id"
expected = sorted(d for s, d in BASE if s == 2)
expected.remove(3)
@@ -127,7 +127,7 @@ def test_delete_one_of_duplicate_edges(graph):
def test_delete_whole_fragment(graph):
engine, uri = graph()
lance.dataset(uri).delete("since >= 40 AND since < 80")
- engine.refresh()
+ engine.checkout_latest()
assert stats(engine)["tombstones"] == 40
check(engine, uri)
@@ -135,7 +135,7 @@ def test_delete_whole_fragment(graph):
def test_update_endpoint_values(graph):
engine, uri = graph()
lance.dataset(uri).update({"dst": "0"}, where="since < 15")
- engine.refresh()
+ engine.checkout_latest()
assert stats(engine)["update"] == "delta"
check(engine, uri)
@@ -144,7 +144,7 @@ def test_non_endpoint_column_change_reuses_csr(graph):
engine, uri = graph()
before = stats(engine)
lance.dataset(uri).add_columns({"weight": "since * 2"})
- engine.refresh()
+ engine.checkout_latest()
after = stats(engine)
assert after["update"] == "reused" and after["edges"] == before["edges"]
assert after["version"] > before["version"]
@@ -154,11 +154,11 @@ def test_non_endpoint_column_change_reuses_csr(graph):
def test_sequential_deltas_accumulate(graph):
engine, uri = graph()
lance.write_dataset(edges([(7, 8), (7, 9)], 2000), uri, mode="append")
- engine.refresh()
+ engine.checkout_latest()
lance.dataset(uri).delete("since = 2000 OR since = 0")
- engine.refresh()
+ engine.checkout_latest()
lance.write_dataset(edges([(7, 8)], 3000), uri, mode="append")
- engine.refresh()
+ engine.checkout_latest()
s = stats(engine)
assert s["update"] == "delta" and s["delta_edges"] == 3 and s["tombstones"] == 2
# One tombstone removes an appended edge, so two of three appends are still live.
@@ -170,7 +170,7 @@ def test_optimize_folds_delta_into_base(graph):
engine, uri = graph()
lance.write_dataset(edges([(7, 8), (7, 9)], 2000), uri, mode="append")
lance.dataset(uri).delete("since = 2000 OR since = 0")
- engine.refresh()
+ engine.checkout_latest()
before = stats(engine)
assert before["update"] == "delta" and before["unindexed_edges"] == 1
@@ -210,7 +210,7 @@ def test_compaction_keeps_results(graph):
ds = lance.dataset(uri)
ds.delete("since < 30")
ds.optimize.compact_files(target_rows_per_fragment=1000)
- engine.refresh()
+ engine.checkout_latest()
check(engine, uri)
@@ -219,14 +219,14 @@ def test_deltas_survive_cleanup_of_old_versions(graph):
engine, uri = graph()
lance.write_dataset(edges([(3, 4), (4, 3)], 4000), uri, mode="append")
lance.write_dataset(edges([(5, 6), (6, 5)], 4002), uri, mode="append")
- engine.refresh()
+ engine.checkout_latest()
# Compaction rewrites the two small fragments; cleanup deletes their files.
lance.dataset(uri).optimize.compact_files(target_rows_per_fragment=40)
lance.dataset(uri).delete("since = 0 OR since = 4003")
lance.dataset(uri).cleanup_old_versions(
older_than=timedelta(0), delete_unverified=True
)
- engine.refresh()
+ engine.checkout_latest()
s = stats(engine)
assert (s["update"], s["delta_edges"], s["tombstones"]) == ("delta", 7, 5)
check(engine, uri)
@@ -235,7 +235,7 @@ def test_deltas_survive_cleanup_of_old_versions(graph):
def test_non_endpoint_column_change_is_visible_through_relationship(graph):
engine, uri = graph()
lance.dataset(uri).add_columns({"weight": "since * 2"})
- engine.refresh()
+ engine.checkout_latest()
assert stats(engine)["update"] == "reused"
got = engine.execute(
"MATCH (a:Person)-[r:KNOWS]->(b:Person) RETURN r.since, r.weight"
@@ -247,7 +247,7 @@ def test_non_endpoint_column_change_is_visible_through_relationship(graph):
def test_overwrite_triggers_rebuild(graph):
engine, uri = graph()
lance.write_dataset(edges([(1, 2), (2, 3)]), uri, mode="overwrite")
- engine.refresh()
+ engine.checkout_latest()
s = stats(engine)
assert (s["update"], s["edges"], s["delta_edges"]) == ("rebuild", 2, 0)
check(engine, uri)
@@ -257,7 +257,7 @@ def test_large_delta_triggers_rebuild(graph):
engine, uri = graph()
many = [(i % N, (i + 3) % N) for i in range(100)] # 50% of the base
lance.write_dataset(edges(many, 5000), uri, mode="append")
- engine.refresh()
+ engine.checkout_latest()
s = stats(engine)
assert (s["update"], s["edges"], s["delta_edges"]) == ("rebuild", 300, 0)
assert s["unindexed_edges"] == 0 and s["base_version"] == s["version"]
@@ -269,6 +269,6 @@ def test_stable_row_ids_with_deletes_and_appends(graph):
ds = lance.dataset(uri)
ds.delete("since % 5 = 1")
lance.write_dataset(edges([(0, 39)], 9000), uri, mode="append")
- engine.refresh()
+ engine.checkout_latest()
assert stats(engine)["update"] == "delta"
check(engine, uri)
diff --git a/python/python/tests/test_engine_csr_persist.py b/python/python/tests/test_engine_csr_persist.py
new file mode 100644
index 0000000..b45bbd6
--- /dev/null
+++ b/python/python/tests/test_engine_csr_persist.py
@@ -0,0 +1,153 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The Lance Authors
+
+"""Stored CSR files (`persist_csr=True`).
+
+An engine that loads a stored CSR must return exactly what a join-path engine
+returns, after appends, deletes, compaction and cleanup of old versions.
+"""
+
+import os
+import sys
+from datetime import timedelta
+
+import lance
+import pyarrow as pa
+import pytest
+from lance_csr import CypherEngine, GraphConfig
+
+CONFIG = (
+ GraphConfig.builder()
+ .node_table("Person", "id")
+ .rel_table("KNOWS", "Person", "Person", "src", "dst")
+ .build()
+)
+N = 40
+PEOPLE = pa.table({"id": list(range(N)), "name": [f"p{i}" for i in range(N)]})
+QUERIES = [
+ "MATCH (a:Person)-[:KNOWS]->(b:Person) RETURN a.id, b.id",
+ "MATCH (a:Person)<-[:KNOWS]-(b:Person) RETURN a.id, b.name",
+ "MATCH (a:Person)-[:KNOWS]->(b:Person)-[:KNOWS]->(c:Person) RETURN count(*) AS n",
+ "MATCH (a:Person)-[r:KNOWS]->(b:Person) RETURN a.id, b.id, r.since",
+]
+
+
+def edges(pairs, start=0):
+ src, dst = zip(*pairs)
+ return pa.table(
+ {
+ "src": pa.array(src, pa.int64()),
+ "dst": pa.array(dst, pa.int64()),
+ "since": list(range(start, start + len(pairs))),
+ }
+ )
+
+
+# 200 edges in 5 fragments, with duplicates.
+BASE = [(i % N, (i * 7 + 1) % N) for i in range(190)] + [(i, i + 1) for i in range(10)]
+
+
+def rows(table):
+ return sorted(table.to_pylist(), key=lambda r: [str(v) for v in r.values()])
+
+
+def stored(uri):
+ folder = uri / "_lance_csr"
+ return sorted(os.listdir(folder)) if folder.exists() else []
+
+
+@pytest.fixture
+def knows(tmp_path):
+ uri = tmp_path / "knows.lance"
+ lance.write_dataset(edges(BASE), uri, max_rows_per_file=40)
+ return uri
+
+
+def engine(uri, **kwargs):
+ datasets = {"Person": PEOPLE, "KNOWS": lance.dataset(uri)}
+ return CypherEngine(CONFIG, datasets, use_csr=True, persist_csr=True, **kwargs)
+
+
+def check(persisted, uri):
+ join = CypherEngine(CONFIG, {"Person": PEOPLE, "KNOWS": lance.dataset(uri)})
+ for query in QUERIES:
+ assert rows(persisted.execute(query)) == rows(join.execute(query)), query
+
+
+def test_stored_csr_is_loaded(knows):
+ first = engine(knows)
+ assert first.csr_stats()["KNOWS"]["update"] == "build"
+ assert stored(knows) == ["1.arrow"]
+ second = engine(knows)
+ s = second.csr_stats()["KNOWS"]
+ assert (s["update"], s["base_version"], s["edges"]) == ("loaded", 1, len(BASE))
+ check(second, knows)
+
+
+def test_stored_csr_survives_compaction_and_cleanup(knows):
+ engine(knows)
+ lance.write_dataset(edges([(3, 4), (4, 3)], 4000), knows, mode="append")
+ lance.write_dataset(edges([(5, 6), (6, 5)], 4002), knows, mode="append")
+ lance.dataset(knows).optimize.compact_files(target_rows_per_fragment=40)
+ lance.dataset(knows).delete("since = 0 OR since = 4003")
+ # Version 1, the stored file's version, is gone after this.
+ lance.dataset(knows).cleanup_old_versions(
+ older_than=timedelta(0), delete_unverified=True
+ )
+ assert stored(knows) == ["1.arrow"]
+ loaded = engine(knows)
+ s = loaded.csr_stats()["KNOWS"]
+ assert (s["update"], s["base_version"]) == ("loaded", 1)
+ assert s["delta_edges"] > 0 and s["tombstones"] > 0
+ check(loaded, knows)
+
+
+def test_rebuild_stores_a_new_base(knows):
+ engine(knows)
+ # More than 25% of the base appended: the next start rebuilds and stores again.
+ lance.write_dataset(edges(BASE[:60], 5000), knows, mode="append")
+ rebuilt = engine(knows)
+ assert rebuilt.csr_stats()["KNOWS"]["update"] == "build"
+ assert stored(knows) == ["2.arrow"]
+ check(rebuilt, knows)
+ assert engine(knows).csr_stats()["KNOWS"]["update"] == "loaded"
+
+
+@pytest.mark.skipif(
+ sys.platform == "win32" or os.geteuid() == 0,
+ reason="needs POSIX permissions",
+)
+def test_read_only_dataset_still_builds(knows):
+ os.chmod(knows, 0o555)
+ try:
+ read_only = engine(knows)
+ finally:
+ os.chmod(knows, 0o755)
+ assert read_only.csr_stats()["KNOWS"]["update"] == "build"
+ assert stored(knows) == []
+ check(read_only, knows)
+
+
+def test_delete_stored_csr(knows):
+ persisted = engine(knows)
+ assert stored(knows) == ["1.arrow"]
+ assert persisted.delete_stored_csr() == 1
+ assert not (knows / "_lance_csr").exists()
+ check(persisted, knows) # the in-memory CSR stays
+ assert persisted.delete_stored_csr() == 0
+ in_memory = CypherEngine(
+ CONFIG, {"Person": PEOPLE, "KNOWS": lance.dataset(knows)}, use_csr=True
+ )
+ assert in_memory.csr_stats()["KNOWS"]["update"] == "build"
+ assert stored(knows) == []
+
+
+def test_pyarrow_input_is_not_stored():
+ datasets = {"Person": PEOPLE, "KNOWS": edges(BASE)}
+ e = CypherEngine(CONFIG, datasets, use_csr=True, persist_csr=True)
+ assert e.csr_stats()["KNOWS"]["update"] == "build"
+
+
+def test_persist_csr_needs_use_csr(knows):
+ with pytest.raises(ValueError, match="persist_csr requires use_csr"):
+ CypherEngine(CONFIG, {"KNOWS": lance.dataset(knows)}, persist_csr=True)
diff --git a/python/python/tests/test_engine_lance.py b/python/python/tests/test_engine_lance.py
index 0c81356..7c14962 100644
--- a/python/python/tests/test_engine_lance.py
+++ b/python/python/tests/test_engine_lance.py
@@ -1,10 +1,10 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The Lance Authors
-"""CypherEngine over on-disk Lance datasets: snapshots, refresh and streaming.
+"""CypherEngine over on-disk Lance datasets: snapshots, checkout_latest() and streaming.
The engine pins the dataset version it was given; writes become visible only
-after ``refresh()``. Expected values are computed independently of the engine.
+after ``checkout_latest()``. Expected values are computed independently of the engine.
"""
import re
@@ -109,16 +109,25 @@ def test_snapshot_until_refresh_after_append(graph):
lance.write_dataset(people(N, N + 5), path / "person.lance", mode="append")
assert count_people(engine) == N
- engine.refresh()
+ engine.checkout_latest()
assert count_people(engine) == N + 5
assert engine.dataset_versions()["Person"] == 2
+def test_refresh_is_a_deprecated_alias_of_checkout_latest(graph):
+ path, person, knows = graph
+ engine = CypherEngine(CONFIG, {"Person": person, "KNOWS": knows})
+ lance.write_dataset(people(N, N + 5), path / "person.lance", mode="append")
+ with pytest.warns(DeprecationWarning, match="checkout_latest"):
+ engine.refresh()
+ assert count_people(engine) == N + 5
+
+
def test_refresh_without_changes_keeps_results(graph):
_, person, knows = graph
engine = CypherEngine(CONFIG, {"Person": person, "KNOWS": knows})
before = engine.execute("MATCH (p:Person) WHERE p.id < 3 RETURN p.name")
- engine.refresh()
+ engine.checkout_latest()
after = engine.execute("MATCH (p:Person) WHERE p.id < 3 RETURN p.name")
assert before.equals(after)
assert engine.dataset_versions() == {"Person": 1, "KNOWS": 1}
@@ -136,7 +145,7 @@ def test_refresh_sees_overwrite_with_added_and_retyped_columns(graph):
lance.write_dataset(replacement, path / "person.lance", mode="overwrite")
assert count_people(engine) == N # still pinned
- engine.refresh()
+ engine.checkout_latest()
out = engine.execute("MATCH (p:Person) RETURN p.name, p.age")
assert out.column(0).type == pa.int64()
assert sorted(out.column(1).to_pylist()) == [30, 40]
@@ -149,7 +158,7 @@ def test_refresh_sees_table_replaced_at_same_location(graph):
# Delete and recreate: the new dataset is version 1 again, like the old one.
shutil.rmtree(path / "person.lance")
lance.write_dataset(people(0, 3), path / "person.lance")
- engine.refresh()
+ engine.checkout_latest()
assert count_people(engine) == 3
@@ -163,7 +172,7 @@ def test_failed_refresh_keeps_previous_snapshot(graph, use_csr):
pa.table({"name": ["x"]}), path / "person.lance", mode="overwrite"
)
with pytest.raises(ValueError, match="id"):
- engine.refresh()
+ engine.checkout_latest()
assert count_people(engine) == N
assert engine.dataset_versions()["Person"] == 1
@@ -185,8 +194,8 @@ def test_historical_version_is_pinned_and_engines_independent(graph):
assert count_people(old) == N
assert count_people(new) == N + 5
- # refresh() moves a historical snapshot to the latest version.
- old.refresh()
+ # checkout_latest() moves a historical snapshot to the latest version.
+ old.checkout_latest()
assert count_people(old) == N + 5
@@ -201,7 +210,7 @@ class Wrapped(lance.LanceDataset):
# Treated as Lance (pinned, refreshable), not materialized as Arrow.
assert engine.dataset_versions()["Person"] == 1
lance.write_dataset(people(N, N + 1), path / "person.lance", mode="append")
- engine.refresh()
+ engine.checkout_latest()
assert count_people(engine) == N + 1
@@ -224,7 +233,7 @@ def reader():
lance.write_dataset(
people(N + 5 * (i - 1), N + 5 * i), path / "person.lance", mode="append"
)
- engine.refresh()
+ engine.checkout_latest()
for t in threads:
t.join()
@@ -264,7 +273,7 @@ def test_execute_batches_keeps_snapshot_across_refresh(graph, use_csr):
first = reader.read_next_batch()
lance.write_dataset(people(N, N + 5), path / "person.lance", mode="append")
- engine.refresh()
+ engine.checkout_latest()
remaining = reader.read_all()
assert first.num_rows + remaining.num_rows == N
diff --git a/python/python/tests/test_path_values.py b/python/python/tests/test_path_values.py
index 6a4baf4..c21e391 100644
--- a/python/python/tests/test_path_values.py
+++ b/python/python/tests/test_path_values.py
@@ -274,7 +274,7 @@ def test_path_inventory_keeps_lance_snapshot(tmp_path):
lance.write_dataset(
pa.table({"src": [1], "dst": [2], "weight": [7]}), edges_path, mode="overwrite"
)
- engine.refresh()
+ engine.checkout_latest()
assert reader.read_all().to_pylist() == [
{"names": ["old", "target"], "weights": [5]}
]
@@ -599,7 +599,7 @@ def test_cross_label_shortest_inventory_isolates_refresh(cross_label_graph, tmp_
lance.write_dataset(
pa.table({"src": [1], "dst": [2]}), paths["LIVES"], mode="overwrite"
)
- engine.refresh()
+ engine.checkout_latest()
assert reader.read_all().to_pylist() == [{"ids": [1, 1]}]
for start in [1, 2, 1]:
assert engine.execute(query, {"start": start}).to_pylist() == (
diff --git a/python/python/tests/test_target_partitions.py b/python/python/tests/test_target_partitions.py
index 2bc0f1e..676ad61 100644
--- a/python/python/tests/test_target_partitions.py
+++ b/python/python/tests/test_target_partitions.py
@@ -58,7 +58,7 @@ def test_partition_targets_preserve_queries_and_refresh(
lance.write_dataset(
pa.table({"id": [5]}), tmp_path / "Person.lance", mode="append"
)
- engine.refresh()
+ engine.checkout_latest()
assert reader.read_all().to_pylist() == [{"id": value} for value in range(1, 5)]
count = 5 if storage == "lance" else 4
assert engine.execute(scan).to_pylist() == [
diff --git a/python/python/tests/test_typing.py b/python/python/tests/test_typing.py
index 87b21cc..21378ea 100644
--- a/python/python/tests/test_typing.py
+++ b/python/python/tests/test_typing.py
@@ -190,7 +190,7 @@ def test_stub_check_detects_missing_method():
node.body = [
item
for item in node.body
- if not isinstance(item, ast.FunctionDef) or item.name != "refresh"
+ if not isinstance(item, ast.FunctionDef) or item.name != "checkout_latest"
]
with pytest.raises(AssertionError, match="public members differ"):
check_class(node, _internal.graph.CypherEngine)
diff --git a/python/python/tests/typing/public_api.py b/python/python/tests/typing/public_api.py
index 539614f..b2b3c36 100644
--- a/python/python/tests/typing/public_api.py
+++ b/python/python/tests/typing/public_api.py
@@ -54,7 +54,7 @@ def public_api(
assert_type(engine.execute(query, params), Table)
assert_type(engine.execute_batches(query, params), RecordBatchReader)
assert_type(engine.explain(query, params), str)
- assert_type(engine.refresh(), None)
+ assert_type(engine.checkout_latest(), None)
assert_type(engine.config(), GraphConfig)
assert_type(engine.dataset_versions(), dict[str, int])
assert_type(engine.node_index_stats()["person"]["rows"], int)