Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
cc02ac1
commit
touch-of-grey Jan 20, 2026
fc3b5e2
stage
touch-of-grey Jan 22, 2026
c49d5d7
feat(mem_wal): unify time and size-based WAL flush triggers
touch-of-grey Jan 22, 2026
4895ac4
fix(mem_wal): reset WAL buffer in MemTableFlushHandler::freeze_memtable
touch-of-grey Jan 22, 2026
7547851
fix(mem_wal): use saturating subtraction for pending_bytes to prevent…
touch-of-grey Jan 22, 2026
fe45b22
refactor: remove pending_bytes tracking from WalBuffer
touch-of-grey Jan 22, 2026
eceb2bb
refactor: replace IVF partition overflow with 90% soft capacity thres…
touch-of-grey Jan 22, 2026
3b4305b
refactor: simplify IvfPqMemIndex to use SkipMap instead of pre-transp…
touch-of-grey Jan 22, 2026
1277042
feat(mem_wal): add HNSW as configurable in-memory vector index option
touch-of-grey Jan 22, 2026
5f33efb
refactor(mem_wal): remove HNSW, keep only IVF-PQ for vector indexing
touch-of-grey Jan 22, 2026
0c2038f
test(mem_wal): add BTree and FTS indexes to S3 integration test
touch-of-grey Jan 22, 2026
4b7241a
fix(mem_wal): use eprintln! for integration test output
touch-of-grey Jan 22, 2026
02635bf
fix(mem_wal): restore eprintln! debug statements for visibility
touch-of-grey Jan 22, 2026
bdb742d
perf(mem_wal): run all index updates in parallel during WAL flush
touch-of-grey Jan 22, 2026
ca2f2f1
feat(mem_wal): add per-index timing logs during WAL flush
touch-of-grey Jan 22, 2026
ef9819d
refactor(mem_wal): per-memtable WAL watermarks and instant freeze
touch-of-grey Jan 23, 2026
051619c
better stats
touch-of-grey Jan 23, 2026
6d2145b
fix: set memtable batch capacity dynamically in read benchmark
touch-of-grey Jan 23, 2026
1e89047
feat(mem_wal): optimize IVF-PQ index with hybrid storage and MVCC vis…
touch-of-grey Jan 23, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
607 changes: 456 additions & 151 deletions Cargo.lock

Large diffs are not rendered by default.

2 changes: 2 additions & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,7 @@ criterion = { version = "0.5", features = [
"html_reports",
] }
crossbeam-queue = "0.3"
crossbeam-skiplist = "0.1"
datafusion = { version = "51.0.0", default-features = false, features = [
"nested_expressions",
"regex_expressions",
Expand Down Expand Up @@ -173,6 +174,7 @@ serde = { version = "^1" }
serde_json = { version = "1" }
semver = "1.0"
shellexpand = "3.0"
slatedb = "0.3"
snafu = "0.8"
strum = "0.26"
tantivy = { version = "0.24.1", features = ["stopwords"] }
Expand Down
499 changes: 64 additions & 435 deletions docs/src/format/table/mem_wal.md

Large diffs are not rendered by default.

Binary file modified docs/src/images/mem_wal_regional.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
7,447 changes: 7,447 additions & 0 deletions result.log

Large diffs are not rendered by default.

11 changes: 11 additions & 0 deletions rust/lance/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ byteorder.workspace = true
bytes.workspace = true
chrono.workspace = true
clap = { version = "4.1.1", features = ["derive"], optional = true }
crossbeam-skiplist.workspace = true
# This is already used by datafusion
dashmap = "6"
deepsize.workspace = true
Expand Down Expand Up @@ -81,6 +82,7 @@ humantime = { workspace = true }
async_cell = "0.2.2"
semver.workspace = true
tokio-stream = { workspace = true }
tokio-util = { workspace = true }

[target.'cfg(target_os = "linux")'.dev-dependencies]
pprof.workspace = true
Expand Down Expand Up @@ -113,6 +115,7 @@ aws-sdk-s3 = { workspace = true }
geoarrow-array = { workspace = true }
geoarrow-schema = { workspace = true }
geo-types = { workspace = true }
slatedb = { workspace = true }


[features]
Expand Down Expand Up @@ -165,5 +168,13 @@ harness = false
name = "random_access"
harness = false

[[bench]]
name = "mem_wal_write"
harness = false

[[bench]]
name = "memtable_read"
harness = false

[lints]
workspace = true
Loading
Loading