diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 372e2639bb0..5ec74a05396 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -64,6 +64,17 @@ jobs:
-c .yamllint.yaml \
.github/
+ docs-conformance:
+ name: "Docs conformance"
+ runs-on: ubuntu-latest
+ timeout-minutes: 10
+ steps:
+ - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
+ - name: Check docs facts match their source of truth
+ run: python3 scripts/check-docs-conformance.py
+ - name: Self-test the conformance checker (catch matcher regressions)
+ run: python3 scripts/check-docs-conformance.py --self-test
+
python-lint:
name: "Python (lint)"
runs-on: >-
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
index cee26f919d6..c31c56b44bc 100644
--- a/.github/workflows/docs.yml
+++ b/.github/workflows/docs.yml
@@ -20,6 +20,12 @@ jobs:
timeout-minutes: 30
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+ - name: Check docs conformance (fail before publishing stale docs)
+ # Runs FIRST, before the changelog generator writes generated release-note pages into docs/
+ # (their historical text would otherwise be scanned and could fail the current-docs lint).
+ run: |
+ python3 scripts/check-docs-conformance.py
+ python3 scripts/check-docs-conformance.py --self-test
- uses: ./.github/actions/setup-rust
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
diff --git a/README.md b/README.md
index 8279e9651ca..893a6a1cb47 100644
--- a/README.md
+++ b/README.md
@@ -6,7 +6,7 @@
[](https://codspeed.io/vortex-data/vortex)
[](https://crates.io/crates/vortex)
[](https://pypi.org/project/vortex-data/)
-[](https://central.sonatype.com/artifact/dev.vortex/vortex-spark)
+[](https://central.sonatype.com/artifact/dev.vortex/vortex-spark_2.13)
[](https://codecov.io/github/vortex-data/vortex)
[](CITATION.cff)
diff --git a/docs/_extra/_redirects b/docs/_extra/_redirects
new file mode 100644
index 00000000000..2591023c08c
--- /dev/null
+++ b/docs/_extra/_redirects
@@ -0,0 +1,16 @@
+# Cloudflare Pages redirects for docs.vortex.dev.
+# Copied verbatim to the site root via `html_extra_path` in conf.py.
+# More specific rules must come before broader splat rules.
+
+# The internals/serialization page was split across several specification pages
+# (array-format, ipc-format, reading-a-file); send it to the Specification index, which
+# fans out to all of them.
+/developer-guide/internals/serialization /specification/ 301
+/developer-guide/internals/serialization.html /specification/ 301
+
+# The specs/ section was renamed to specification/. Cover the section root explicitly
+# (the splat does not match the bare /specs path) before the child-path splat rule.
+/specs /specification/ 301
+/specs/ /specification/ 301
+/specs/index.html /specification/ 301
+/specs/* /specification/:splat 301
diff --git a/docs/api/c/index.rst b/docs/api/c/index.rst
index 81d142d16c0..9d9ad30813e 100644
--- a/docs/api/c/index.rst
+++ b/docs/api/c/index.rst
@@ -1,9 +1,10 @@
C API
=====
-The Vortex C API provides a low-level FFI interface to the Vortex library. It is the foundation for
-other language bindings (including C++) and is suitable for embedding Vortex into C applications or
-building higher-level wrappers.
+The Vortex C API provides a low-level FFI interface to the Vortex library. It is the intended
+foundation for other language bindings and is suitable for embedding Vortex into C applications or
+building higher-level wrappers. (The C++ binding currently uses a direct ``cxx`` Rust bridge; a future
+migration to build it on top of this C FFI is planned.)
.. warning::
This API should be considered entirely unstable. It *will* change. Please reach out if a stable
diff --git a/docs/api/cpp/index.rst b/docs/api/cpp/index.rst
index befb980f6a9..704df12ad70 100644
--- a/docs/api/cpp/index.rst
+++ b/docs/api/cpp/index.rst
@@ -1,9 +1,10 @@
C++ API
=======
-The Vortex C++ API provides an idiomatic C++ wrapper around the Vortex C FFI, built using
-`cxx `_. It currently supports reading and writing Vortex files and integrates
-with the Arrow C Data Interface via `nanoarrow `_.
+The Vortex C++ API is an idiomatic C++ binding built with `cxx `_, which generates
+a direct bridge between C++ and Rust. It supports reading and writing Vortex files and integrates
+with the Arrow C Data Interface via `nanoarrow `_. A future
+migration to build the C++ API on top of the C FFI is planned.
In the future we will expand the C++ API to cover Vortex's plugin and extension points. Please
reach out if you are interested in extending Vortex from C++ so we can prioritize these features.
diff --git a/docs/api/java/index.rst b/docs/api/java/index.rst
index 9cb815a0aba..2414b38140d 100644
--- a/docs/api/java/index.rst
+++ b/docs/api/java/index.rst
@@ -39,21 +39,17 @@ They support any Linux distribution with a GLIBC version >= 2.31. This includes
* Ubuntu 20.04 or newer
-Usage Example
--------------
-
-Here's a basic example of using the Vortex Java API to read a Vortex file:
-
-.. code-block:: java
-
- import dev.vortex.api.File;
- import dev.vortex.api.Array;
+Core API
+--------
- // Open a Vortex file
- File vortexFile = File.open("path/to/file.vortex");
+The main entry points of the Vortex Java API are:
- // Read arrays from the file
- Array array = vortexFile.readArray();
+* ``Session`` — the top-level handle that owns data sources and writers.
+* ``DataSource`` — opens a Vortex file, glob, or table via ``DataSource.open(session, uri)`` and
+ exposes its schema and row count.
+* ``Scan`` / ``Partition`` — a scan over a ``DataSource`` iterates its partitions, yielding
+ Arrow-compatible batches.
+* ``VortexWriter`` — writes arrays to a Vortex file.
- // Work with the array data
- System.out.println("Array length: " + array.getLength());
+See the `Vortex JNI API <../../_static/vortex-jni/index.html>`_ above for the full,
+method-level reference.
diff --git a/docs/api/python/arrays.rst b/docs/api/python/arrays.rst
index 4d457c7108f..05f62e7c5d9 100644
--- a/docs/api/python/arrays.rst
+++ b/docs/api/python/arrays.rst
@@ -22,8 +22,9 @@ Base Class
Canonical Encodings
-------------------
-Each :class:`~vortex.DType` has a corresponding canonical encoding. These encodings represent the uncompressed version
-of the array, and are also zero-copy to Apache Arrow.
+Each :class:`~vortex.DType` has a corresponding canonical encoding, with the sole exception of ``Union`` (not yet
+canonicalized). These encodings represent the uncompressed version of the array, and are also zero-copy to
+Apache Arrow. The classes below are the canonical encodings currently exposed in Python.
.. autoclass:: vortex.NullArray
:members:
@@ -37,19 +38,15 @@ of the array, and are also zero-copy to Apache Arrow.
:members:
-.. autoclass:: vortex.VarBinArray
- :members:
-
-
.. autoclass:: vortex.VarBinViewArray
:members:
-.. autoclass:: vortex.StructArray
+.. autoclass:: vortex.FixedSizeListArray
:members:
-.. autoclass:: vortex.ListArray
+.. autoclass:: vortex.StructArray
:members:
@@ -72,6 +69,14 @@ Utility Encodings
:members:
+.. autoclass:: vortex.VarBinArray
+ :members:
+
+
+.. autoclass:: vortex.ListArray
+ :members:
+
+
Compressed Encodings
--------------------
diff --git a/docs/api/python/index.rst b/docs/api/python/index.rst
index 1ef85a4acb6..8f4c141b860 100644
--- a/docs/api/python/index.rst
+++ b/docs/api/python/index.rst
@@ -27,6 +27,7 @@ The Python bindings require Python 3.11 or newer. Pre-built wheels are available
* x86_64 Linux
* ARM64 Linux
* Apple Silicon macOS
+* Intel macOS
They support any Linux distribution with a GLIBC version >= 2.17. This includes
@@ -41,14 +42,17 @@ Here's a basic example of using the Vortex Python API to write and read a Vortex
.. code-block:: python
+ import pyarrow as pa
import vortex
# Write a Vortex file from a PyArrow table
- vortex.io.write_path(my_table, "data.vortex")
+ my_table = pa.table({"id": [1, 2, 3], "name": ["a", "b", "c"]})
+ vortex.io.write(my_table, "data.vortex")
# Read a Vortex file
- dataset = vortex.dataset("data.vortex")
- table = dataset.to_arrow()
+ vf = vortex.open("data.vortex")
+ reader = vf.to_arrow() # a pyarrow.RecordBatchReader
+ table = reader.read_all()
API Reference
diff --git a/docs/concepts/arrays.md b/docs/concepts/arrays.md
index 398b848f59c..8f00f3dc40c 100644
--- a/docs/concepts/arrays.md
+++ b/docs/concepts/arrays.md
@@ -41,19 +41,22 @@ define their own.
### Canonical Arrays
-In order to avoid having to implement logic for an exponential combination of encodings, Vortex defines one canonical
-encoding per logical data type. All arrays can eventually be decompressed one of these canonical encodings.
+In order to avoid having to implement logic for an exponential combination of encodings, Vortex defines a canonical
+encoding for each logical data type — every type except `Union`, which is not yet canonicalized. Arrays of those types
+can eventually be decompressed to one of the canonical encodings listed below.
| Data Type | Canonical Encoding |
|------------------------|----------------------|
| `DType::Null` | `NullArray` |
| `DType::Bool` | `BoolArray` |
| `DType::Primitive` | `PrimitiveArray` |
-| `DType::UTF8` | `VarBinViewArray` |
+| `DType::Decimal` | `DecimalArray` |
+| `DType::Utf8` | `VarBinViewArray` |
| `DType::Binary` | `VarBinViewArray` |
| `DType::Struct` | `StructArray` |
| `DType::List` | `ListViewArray` |
| `DType::FixedSizeList` | `FixedSizeListArray` |
+| `DType::Variant` | `VariantArray` |
| `DType::Extension` | `ExtensionArray` |
### Builtin Arrays
@@ -100,15 +103,15 @@ These can be found in the `encodings/` directory of the Vortex repository.
Arrays carry their own statistics with them, allowing many compute functions to short-circuit or optimize their
implementations. Currently, the available statistics are:
+* `is_constant`: Whether the array holds only a single unique value (treating nulls as equal).
+* `is_sorted`: Whether the array's values are in ascending order.
+* `is_strict_sorted`: Whether the array's values are in strictly ascending order, with no duplicates.
+* `max`: The maximum value in the array (ignoring nulls).
+* `min`: The minimum value in the array (ignoring nulls).
+* `sum`: The sum of the non-null values in the array.
* `null_count`: The number of null values in the array.
-* `true_count`: The number of `true` values in a boolean array.
-* `run_count`: The number of consecutive runs in an array.
-* `is_constant`: Whether the array only holds a single unique value
-* `is_sorted`: Whether the array values are sorted.
-* `is_strict_sorted`: Whether the array values are sorted and unique.
-* `min`: The minimum value in the array.
-* `max`: The maximum value in the array.
-* `uncompressed_size`: The size of the array in memory before any compression.
+* `uncompressed_size_in_bytes`: The size of the array in memory before any compression.
+* `nan_count`: The number of NaN values in the array.
## Execution
diff --git a/docs/concepts/dtypes.md b/docs/concepts/dtypes.md
index d0fd426fe20..ac6b2f448b3 100644
--- a/docs/concepts/dtypes.md
+++ b/docs/concepts/dtypes.md
@@ -19,7 +19,7 @@ with many columns.
## Logical Types
-The following table lists the built-in dtypes in Vortex, each of which can be marked as either nullable or non-nullable.
+The following table lists the built-in dtypes in Vortex, each of which can be marked as either nullable or non-nullable (the `Extension` type is the exception — it inherits nullability from its storage type).
| Name | Domain |
|-----------------|---------------------------------------------|
@@ -32,11 +32,14 @@ The following table lists the built-in dtypes in Vortex, each of which can be ma
| `List` | See [List](#list) |
| `FixedSizeList` | See [List](#list) |
| `Struct` | See [Struct](#struct) |
+| `Union` | Tagged union of variant types (partial) |
+| `Variant` | Semi-structured, JSON-like values (partial) |
| `Extension` | See [Extension](#extension) |
:::{note}
-There are additional logical types that Vortex does not yet support, for example fixed-length binary, maps, and variants.
-These may be added in future versions.
+Vortex's logical type system is still evolving. Some types (e.g. fixed-length binary and maps) are
+not yet supported, and others (notably `Variant` for semi-structured values, and `Union`) are only
+partially implemented — see the [roadmap](../project/roadmap.md).
:::
### Primitive
diff --git a/docs/concepts/file-format.md b/docs/concepts/file-format.md
index 36a6fe0a935..611a96a6248 100644
--- a/docs/concepts/file-format.md
+++ b/docs/concepts/file-format.md
@@ -7,7 +7,8 @@ and serializes the layout and its segments into a single file.
The bulk of the file format specification describes the representation of the footer bytes such that the
layout tree can be reconstructed for scans.
-See the [Vortex File Format Specification](../specs/file-format.md) for full details.
+See the [Vortex File Format Specification](../specification/file-format.md) for full details, or
+[Reading a File](../specification/reading-a-file.md) for a step-by-step walkthrough of the read path.
## Layout Strategy
@@ -15,15 +16,15 @@ The default layout strategy for Vortex files is roughly:
* Struct Layout at the top-level to partition by columns
* Zoned Layout to store pruning statistics for every 8k rows
- * Chunked Layout to partition the column into 2MB of uncompressed data
+ * Chunked Layout to partition the column into chunks
* Compressor Layout to apply a compression strategy
- * Buffered Layout to localize up to 1MB of compressed chunks per column.
+ * Buffered Layout to localize up to 2 MB of compressed chunks per column.
* Flat Layout to serialize each individual array chunk
This strategy optimizes for analytical query patterns: column pruning avoids reading unused columns,
zone statistics enable skipping irrelevant row ranges, and buffered chunks allow efficient I/O
-with parallel decompression. The 8k row zones and 2MB chunks balance pruning granularity against
-metadata overhead.
+with parallel decompression. The 8k row zones and buffered chunk locality balance pruning
+granularity against metadata overhead.
## Compression Strategies
diff --git a/docs/concepts/index.md b/docs/concepts/index.md
index 878296b3d09..f441bda4375 100644
--- a/docs/concepts/index.md
+++ b/docs/concepts/index.md
@@ -24,7 +24,8 @@ without dictating physical layout, allowing the same logical data to use differe
**[Arrays](arrays.md)** are the in-memory representation. Unlike Arrow, Vortex arrays can be
*compressed*—an integer array might be bit-packed rather than stored as a flat buffer. Arrays
-share the same representation on disk and over the wire, enabling zero-copy I/O.
+share the same representation in memory, on disk, and over the wire; the on-disk file layout adds
+alignment padding so reads can be zero-copy.
**[Compute](expressions.md)** functions operate directly on compressed arrays where possible,
dispatching to encoding-specific kernels or falling back to canonical implementations.
@@ -34,10 +35,11 @@ dispatching to encoding-specific kernels or falling back to canonical implementa
**[Layouts](layouts.md)** organize arrays into larger-than-memory datasets (e.g., chunked row groups)
and can read from any block storage: local disk, object stores, caches, etc.
-**[File Format](../specs/file-format.md)** (`.vortex` files) serialize layouts to disk with efficient
-segment retrieval, FlatBuffer metadata for O(1) schema access, and support for memory mapping.
+**[File Format](../specification/file-format.md)** (`.vortex` files) serialize layouts to disk with
+efficient segment retrieval, FlatBuffer metadata for O(1) schema access, and support for memory
+mapping.
-**[IPC Format](../specs/ipc-format.md)** provides streaming transfer of compressed arrays.
+**[IPC Format](../specification/ipc-format.md)** provides streaming transfer of compressed arrays.
## Integrations
diff --git a/docs/concepts/scanning.md b/docs/concepts/scanning.md
index 393b7d2d83c..4988da0d735 100644
--- a/docs/concepts/scanning.md
+++ b/docs/concepts/scanning.md
@@ -1,8 +1,8 @@
# Scan API
:::{note}
-The Scan API is on the roadmap and under active development. The core `Source` trait and scan pipeline
-are functional, but the full API surface is not yet fully defined or implemented.
+The Scan API is on the roadmap and under active development. The core `DataSource` trait and scan
+pipeline are functional, but the full API surface is not yet fully defined or implemented.
:::
The Vortex Scan API defines a standard interface between data storage and query engines. It solves the
@@ -18,11 +18,9 @@ interface that both sides can implement against.
Iceberg Tables ──► └──────────────┘ ──► Spark
```
-Storage backends implement the `Source` trait for reads. Query engines issue a scan request
+Storage backends implement the source interface for reads. Query engines issue a scan request
describing the filter and projection to push down, and the source returns a stream of
-independently-executable splits that can be run concurrently to produce result arrays. An
-equivalent `Sink` trait exists for the write path, accepting an array stream and writing it to
-the underlying storage.
+independently-executable splits that can be run concurrently to produce result arrays.
## Motivation
@@ -38,15 +36,13 @@ any decompression step.
## Source
-A **Source** represents any scannable tabular data. It accepts a scan request (filter, projection,
-limit) and returns a stream of independently-executable splits. An equivalent **Sink** interface
-exists for the write path, allowing query engines to both read from and write to any storage
-backend through a single pair of interfaces.
+A **source** (the `DataSource` trait) represents any scannable tabular data. It accepts a scan request
+(filter, projection, limit) and returns a stream of independently-executable splits.
### Splits
-A source produces splits, each representing an independent unit of work that can be executed in
-parallel. A split typically corresponds to a range of rows in a layout, such as a chunk or a set
+A source produces splits (the `Partition` trait), each representing an independent unit of work that
+can be executed in parallel. A split typically corresponds to a range of rows in a layout, such as a chunk or a set
of row-group partitions.
Each split carries size and row count estimates that query engines use for scheduling decisions.
@@ -56,7 +52,7 @@ Splits can also be serialized for distributed execution across remote workers.
A source may front remote storage rather than local files. In this case, the split's execution
issues a remote call and receives the result over the network. The
-[Vortex IPC format](../specs/ipc-format.md) can be used as the wire protocol for these calls, allowing
+[Vortex IPC format](../specification/ipc-format.md) can be used as the wire protocol for these calls, allowing
compressed arrays to be transferred without decompression. This gives remote sources the same
zero-decompression benefits as local scans -- the data stays in its compressed encoding end-to-end,
from remote storage through the network and into the query engine.
@@ -89,5 +85,5 @@ pipeline.
Query engines integrate with the Scan API by translating their internal plan representations into
scan requests and consuming the resulting array stream in their preferred format. Integrations
-exist for DuckDB, DataFusion, Spark, and Trino, with each engine converting its native filter
-and projection representations into Vortex [expressions](expressions.md).
+exist for DuckDB, DataFusion, and Spark, each engine converting its native filter and projection
+representations into Vortex [expressions](expressions.md).
diff --git a/docs/conf.py b/docs/conf.py
index edb6342aaee..f31dcd0240a 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -67,14 +67,62 @@
myst_enable_extensions = [
"colon_fence", # Use ::: for Sphinx directives
+ "substitution", # Allow {{ name }} substitutions sourced from code (see myst_substitutions)
]
myst_heading_anchors = 3
+
+def _read_rust_usize_const(rel_path: str, const_name: str) -> str:
+ """Read a ``pub const : usize = `` value from a Rust source file at build time, so the
+ docs can inline it via a MyST substitution instead of hard-coding (and drifting from) it. Fails
+ loudly if the constant moved, mirroring the doc-conformance lint's source-of-truth discipline.
+
+ This is the BUILD-TIME, source-from-code counterpart to ``scripts/check-docs-conformance.py``
+ (the LINT-TIME, value-is-present counterpart): substitutions auto-update the rendered value here,
+ while the conformance lint locks values that appear in prose/commands the substitution can't reach.
+ The two readers deliberately have separate semantics (this one parses Rust ``usize`` consts for
+ substitution; the script parses TOML/Rust/Java/C++ for presence locks) and separate tests
+ (the script's ``--self-test``); keep both fail-loud on a moved source."""
+ text = (git_root / rel_path).read_text(encoding="utf-8")
+ # Anchor to the start of a const statement and require the value to be the COMPLETE RHS — a plain
+ # integer literal terminated by `;` — so a non-literal RHS (`= 1_024;`, `= 64 * 3;`, `= 32usize;`)
+ # fails rather than partial-matching. Require EXACTLY ONE match: a commented-out/decoy copy of the
+ # const (e.g. an old value documented in a block comment) produces a second match and fails loudly
+ # rather than silently sourcing the wrong one. (A lone commented copy can't occur in code that
+ # compiles, since the const is referenced.)
+ matches = re.findall(
+ rf"^\s*pub const {re.escape(const_name)}: usize = (\d+)\s*;",
+ text,
+ re.MULTILINE,
+ )
+ if len(matches) != 1:
+ raise RuntimeError(
+ f"conf.py: expected exactly one plain-integer `pub const {const_name}: usize = N;` in "
+ f"{rel_path} for a MyST substitution, found {len(matches)} — the constant moved, is no "
+ f"longer a bare literal, or has a decoy copy; update conf.py."
+ )
+ return matches[0]
+
+
+# Substitutions sourced from code at build time so the rendered docs cannot drift from the
+# implementation. Used in prose/tables (MyST substitutions do not expand inside ``` code fences ```;
+# values that appear in commands/code blocks are covered by scripts/check-docs-conformance.py).
+myst_substitutions = {
+ "io_concurrency_local": _read_rust_usize_const(
+ "vortex-io/src/std_file/read_at.rs", "DEFAULT_CONCURRENCY"
+ ),
+ "io_concurrency_object": _read_rust_usize_const(
+ "vortex-io/src/object_store/read_at.rs", "DEFAULT_CONCURRENCY"
+ ),
+}
+
# -- Options for HTML output -------------------------------------------------
# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
html_theme = "pydata_sphinx_theme"
html_static_path = ["_static"]
+# Files copied verbatim to the site root (e.g. the Cloudflare Pages `_redirects` map).
+html_extra_path = ["_extra"]
html_css_files = ["style.css"]
html_favicon = "_static/vortex_logo.svg" # relative to _static/
diff --git a/docs/developer-guide/embedding/cxx.md b/docs/developer-guide/embedding/cxx.md
index 40238d36282..7cb18699747 100644
--- a/docs/developer-guide/embedding/cxx.md
+++ b/docs/developer-guide/embedding/cxx.md
@@ -6,7 +6,7 @@ This page is under construction.
Planned content:
-- The C++ wrapper around the C FFI
+- The C++ binding via the `cxx` direct Rust bridge (with a planned migration to build it on top of the C FFI)
- Building with CMake
- RAII wrappers for Vortex objects
- Working with arrays and dtypes from C++
diff --git a/docs/developer-guide/embedding/index.md b/docs/developer-guide/embedding/index.md
index 8c8154718ca..e023b21480c 100644
--- a/docs/developer-guide/embedding/index.md
+++ b/docs/developer-guide/embedding/index.md
@@ -6,24 +6,14 @@ This section is under construction. For guidance on embedding Vortex, please joi
or open a [GitHub Issue](https://github.com/vortex-data/vortex/issues/new/choose).
:::
-Vortex can be embedded into applications and services via its C FFI, C++ wrapper, or the Scan API.
+Vortex can be embedded into applications and services via its C FFI, C++ binding, or the Scan API.
The following topics are planned for this section:
- **C FFI** -- the Vortex C API, building and linking, session management, arrays, dtypes,
error handling, and memory ownership.
-- **C++** -- the C++ wrapper around the C FFI, CMake integration, and RAII wrappers.
+- **C++** -- the C++ binding (a direct `cxx` Rust bridge today, with a planned migration to build it
+ on top of the C FFI), CMake integration, and RAII wrappers.
- **Scan API** -- serving Vortex data to query engines, wire format serialization, filter and
projection pushdown, and custom scan providers.
- **GPU Acceleration** -- CUDA requirements, GPU-accelerated decompression and compute,
host/device memory management, and current limitations.
-
-```{toctree}
----
-hidden: true
----
-
-ffi
-cxx
-scan-api
-gpu
-```
diff --git a/docs/developer-guide/extending/index.md b/docs/developer-guide/extending/index.md
index c8e862c27be..f3c33df388c 100644
--- a/docs/developer-guide/extending/index.md
+++ b/docs/developer-guide/extending/index.md
@@ -17,14 +17,3 @@ The following topics are planned for this section:
on-disk data organizations.
- **Writing a Compute Function** -- the dispatch model, implementing kernels, vtable
registration, and testing.
-
-```{toctree}
----
-hidden: true
----
-
-extension-dtypes
-writing-an-encoding
-writing-a-layout
-writing-a-compute-fn
-```
diff --git a/docs/developer-guide/index.md b/docs/developer-guide/index.md
index e6eec4bfb70..79c9e12dfee 100644
--- a/docs/developer-guide/index.md
+++ b/docs/developer-guide/index.md
@@ -1,16 +1,40 @@
# Developer Guide
-Guide for extending, embedding, and contributing to the Vortex ecosystem.
+Guide for extending, embedding, and contributing to the Vortex ecosystem. See the
+[Overview](overview.md) for how the guide is organized and where to start.
+
+```{toctree}
+---
+maxdepth: 1
+---
+
+overview
+```
```{toctree}
---
maxdepth: 2
+caption: Extending
---
extending/index
+extending/writing-an-encoding
+extending/writing-a-layout
+extending/writing-a-compute-fn
+extending/extension-dtypes
+```
+
+```{toctree}
+---
+maxdepth: 2
+caption: Embedding
+---
+
embedding/index
-language-bindings
-benchmarking
+embedding/ffi
+embedding/cxx
+embedding/scan-api
+embedding/gpu
```
```{toctree}
@@ -26,7 +50,6 @@ internals/vtables
internals/execution
internals/stats-pruning
internals/io
-internals/serialization
internals/cuda
```
@@ -40,3 +63,13 @@ integrations/datafusion
integrations/duckdb
integrations/spark
```
+
+```{toctree}
+---
+maxdepth: 2
+caption: More
+---
+
+language-bindings
+benchmarking
+```
diff --git a/docs/developer-guide/integrations/datafusion.md b/docs/developer-guide/integrations/datafusion.md
index 93f45be0a8d..1bb21a4d741 100644
--- a/docs/developer-guide/integrations/datafusion.md
+++ b/docs/developer-guide/integrations/datafusion.md
@@ -64,7 +64,7 @@ DataFusion. Batches are sliced to respect DataFusion's configured batch size pre
## Future Work
The current integration builds directly on the `ScanBuilder` and layout reader APIs. Future work
-will migrate it to use the [Scan API](/concepts/scanning) `Source` trait, which will simplify
+will migrate it to use the [Scan API](/concepts/scanning) `DataSource` trait, which will simplify
the integration by providing a standard interface for file discovery, partitioning, and pushdown
that is shared across all engine integrations.
diff --git a/docs/developer-guide/integrations/duckdb.md b/docs/developer-guide/integrations/duckdb.md
index 060f5fca973..a61262135c8 100644
--- a/docs/developer-guide/integrations/duckdb.md
+++ b/docs/developer-guide/integrations/duckdb.md
@@ -64,6 +64,6 @@ vectorized execution model.
## Future Work
The current integration builds directly on the `ScanBuilder`, layout reader, and file APIs.
-Future work will migrate it to use the [Scan API](/concepts/scanning) `Source` trait, unifying
+Future work will migrate it to use the [Scan API](/concepts/scanning) `DataSource` trait, unifying
file discovery, multi-file coordination, and pushdown behind a single interface shared across
all engine integrations.
diff --git a/docs/developer-guide/integrations/spark.md b/docs/developer-guide/integrations/spark.md
index 2342f95063f..12ba8a7418d 100644
--- a/docs/developer-guide/integrations/spark.md
+++ b/docs/developer-guide/integrations/spark.md
@@ -39,8 +39,10 @@ Projection pushdown is supported through Spark's `SupportsPushDownRequiredColumn
The scan builder prunes the column list to only those referenced by the query, and the pruned
column set is passed to the native scan via `ScanOptions`.
-Filter pushdown infrastructure exists in the `ScanOptions` type but is not yet connected to
-Spark's `SupportsPushDownFilters` interface. This is planned future work.
+Filter pushdown is supported through Spark's `SupportsPushDownV2Filters` interface. The scan builder
+maps each pushable predicate to a Vortex expression (via `SparkPredicateToVortexExpression`) and pushes
+it into the native scan; predicates Spark cannot push (for example those referencing partition columns)
+are returned for post-scan evaluation.
## Data Export
@@ -52,6 +54,6 @@ through direct byte buffers.
## Future Work
The current integration builds directly on the native file and scan APIs via JNI. Future work
-will migrate it to use the [Scan API](/concepts/scanning) `Source` trait, which will provide a
-standard interface for file discovery, partitioning, and pushdown. This will also enable
-connecting Spark's filter pushdown to Vortex's expression-based filtering.
+will migrate it to use the [Scan API](/concepts/scanning) `DataSource` trait, which will provide a
+standard interface for file discovery, partitioning, and pushdown — unifying the existing column-
+pruning and filter-pushdown paths with the other engine integrations.
diff --git a/docs/developer-guide/internals/architecture.md b/docs/developer-guide/internals/architecture.md
index 3354c25cd85..21305bd8658 100644
--- a/docs/developer-guide/internals/architecture.md
+++ b/docs/developer-guide/internals/architecture.md
@@ -28,7 +28,7 @@ format, and I/O.
| ------------------------- | ----------------------------------------------------------------------------- |
| `vortex-error` | `VortexError` and `VortexResult` types, `vortex_err!` / `vortex_bail!` macros |
| `vortex-buffer` | Zero-copy aligned `Buffer` with guaranteed alignment |
-| `vortex-array/src/dtype` | `DType` enum: Null, Bool, Primitive, UTF8, Binary, Struct, List, Extension |
+| `vortex-array/src/dtype` | `DType` enum: Null, Bool, Primitive, Decimal, Utf8, Binary, List, FixedSizeList, Struct, Union, Variant, Extension |
| `vortex-array/src/scalar` | Single-value representations of each dtype |
| `vortex-mask` | Bitmask operations for validity and selection |
| `vortex-session` | Session object holding registries for encodings, layouts, and extension types |
@@ -38,7 +38,7 @@ format, and I/O.
| `vortex-ipc` | IPC format for inter-process communication |
| `vortex-file` | `.vortex` file reading and writing |
| `vortex-scan` | Table scan with filter and projection pushdown |
-| `vortex-expr` | Expression representation and optimization |
+| `vortex-array/src/expr` | Expression representation and optimization |
| `vortex-flatbuffers` | FlatBuffer schema definitions |
## Encodings
@@ -54,12 +54,13 @@ and registers itself with the session. The standard encodings are bundled into t
| `vortex-runend` | Run-end encoding for repetitive data |
| `vortex-sparse` | Sparse array encoding |
| `vortex-zigzag` | ZigZag encoding for signed integers |
-| `vortex-roaring` | Roaring bitmap encoding |
-| `vortex-dict` | Dictionary encoding |
| `vortex-bytebool` | Byte-per-boolean encoding |
| `vortex-datetime-parts` | DateTime field decomposition |
| `vortex-decimal-byte-parts` | Decimal byte decomposition |
| `vortex-sequence` | Arithmetic sequence encoding |
+| `vortex-pco` | Pcodec compression for numeric data |
+| `vortex-zstd` | Zstd block compression |
+| `vortex-parquet-variant` | Parquet Variant (semi-structured) encoding |
## Language Bindings
@@ -70,7 +71,7 @@ Language bindings expose Vortex to non-Rust environments.
| `vortex-python/` | Python bindings via PyO3 and Maturin |
| `java/vortex-jni/` | Java JNI bindings |
| `vortex-ffi/` | C FFI bindings (generates `vortex.h`) |
-| `vortex-cxx/` | C++ wrapper around the C FFI |
+| `vortex-cxx/` | C++ binding via the `cxx` Rust bridge |
## Integrations
@@ -80,8 +81,7 @@ Query engine integrations allow Vortex files to be queried through existing anal
|----------------------------------| ---------- |----------------------------------------------|
| `vortex-datafusion/` | DataFusion | `TableProvider` and `FileFormat` integration |
| `vortex-duckdb/` | DuckDB | Table function integration |
-| `java/vortex-spark_{2.12,2.13}/` | Spark | Spark DataSource V2 connector via JNI |
-| `java/vortex-trino/` | Trino | Trino connector (in development) |
+| `java/vortex-spark/` | Spark | Spark DataSource V2 connector via JNI (published as `vortex-spark_2.12` and `vortex-spark_2.13`) |
## Other Crates
diff --git a/docs/developer-guide/internals/io.md b/docs/developer-guide/internals/io.md
index 9143d203fb2..5a846a7c723 100644
--- a/docs/developer-guide/internals/io.md
+++ b/docs/developer-guide/internals/io.md
@@ -11,12 +11,12 @@ async `read_at(offset, length, alignment)` method that returns an aligned buffer
metadata methods that inform the I/O scheduler:
- **`concurrency()`** -- the maximum number of concurrent reads the backend can efficiently
- sustain. Local files default to 32; object stores default to 192.
+ sustain. Local files default to {{io_concurrency_local}}; object stores default to {{io_concurrency_object}}.
- **`coalesce_config()`** -- optional parameters that control read merging (see below).
- **`size()`** -- the total size of the source, used for footer reads and bounds checking.
-Implementations exist for local files (`FileReadAdapter`), object stores
-(`ObjectStoreSource` via the `object_store` crate), and in-memory buffers.
+Implementations exist for local files (`FileReadAt`), object stores
+(`ObjectStoreReadAt` via the `object_store` crate), and in-memory buffers.
## Read Coalescing
@@ -28,9 +28,14 @@ behaviour with two parameters:
more than this are issued independently.
- **`max_size`** -- the maximum span of a single coalesced read.
-Default configurations are tuned per backend. Local files use 8 KB for both distance and max
-size, reflecting the low cost of small NVMe reads. Object stores use 1 MB distance and 16 MB
-max size, reflecting the high per-request overhead of HTTP round-trips.
+`CoalesceConfig` provides presets tuned per backend:
+
+- Local files use a 1 MB distance and 4 MB max size.
+- Object stores use a 1 MB distance and 16 MB max size, reflecting the high per-request overhead of HTTP round-trips.
+- In-memory sources use an 8 KB distance and 8 KB max size.
+
+For in-memory sources, coalescing is opt-in: the default reader does not coalesce, while readers that
+opt in adopt the 8 KB preset above.
The coalescing algorithm runs inside an `IoRequestStream` that maintains a spatial index of
pending requests. When a request is polled, the stream scans for nearby requests within the
@@ -83,10 +88,13 @@ callers request the same segment simultaneously.
Each `VortexReadAt` implementation provides its own concurrency and coalescing parameters,
allowing the I/O scheduler to adapt automatically:
-| Backend | Concurrency | Coalesce Distance | Coalesce Max Size |
-|---------------|-------------|--------------------|--------------------|
-| Local file | 32 | 8 KB | 8 KB |
-| Object store | 192 | 1 MB | 16 MB |
+| Backend | Concurrency | Coalesce Distance | Coalesce Max Size |
+|---------------|----------------------------|--------------------|--------------------|
+| Local file | {{io_concurrency_local}} | 1 MB | 4 MB |
+| Object store | {{io_concurrency_object}} | 1 MB | 16 MB |
+
+In-memory readers coalesce only when opted in (see above); the table lists the two backends with
+fixed per-backend defaults.
Local file reads are dispatched via `spawn_blocking` to avoid blocking the async executor.
Object store reads are natively async and wrapped with `async_compat` for runtime
diff --git a/docs/developer-guide/internals/serialization.md b/docs/developer-guide/internals/serialization.md
deleted file mode 100644
index 11cfe720c47..00000000000
--- a/docs/developer-guide/internals/serialization.md
+++ /dev/null
@@ -1,114 +0,0 @@
-# Serialization
-
-Vortex uses the same binary representation for arrays in memory, on disk, and over the wire.
-Metadata is stored in FlatBuffers for O(1) field access without parsing, and data buffers are
-stored separately with alignment guarantees that enable zero-copy reads. Appropriate padding is
-written into Vortex files to ensure that segments can be memory-mapped with correct alignment.
-
-## Array Serialization
-
-A serialized array consists of two parts: a FlatBuffer describing the array tree, and a
-sequence of data buffers.
-
-The FlatBuffer contains an `ArrayNode` tree where each node records:
-- The array ID (as an interned u16 index).
-- Array-specific metadata bytes.
-- References to child `ArrayNode`s.
-- Indices into the buffer table.
-- Optional statistics (min, max, null count, sort order, etc.).
-
-The buffer table records each buffer's padding, alignment exponent, compression, and length.
-Buffers are laid out contiguously after the metadata, with padding inserted to satisfy each
-buffer's alignment requirement.
-
-On the wire, a serialized array is:
-
-```
-[padding] [buffer 0] [padding] [buffer 1] ... [flatbuffer] [u32 flatbuffer length]
-```
-
-Deserialization constructs an `ArrayParts` value that holds the FlatBuffer and buffer handles
-without copying. The array is then decoded by resolving the array ID through the session's
-registry and calling `build()` on the corresponding vtable.
-
-## IPC Format
-
-The IPC format wraps serialized arrays in a message-oriented protocol for streaming between
-processes. Each message consists of:
-
-```
-[u32 flatbuffer length] [flatbuffer Message] [body bytes]
-```
-
-Three message types are defined:
-
-- **ArrayMessage** -- a serialized array with its row count and an encoding context that maps
- dictionary indices to encoding IDs.
-- **BufferMessage** -- a raw buffer with an alignment exponent, used for transferring individual
- segments.
-- **DTypeMessage** -- a serialized dtype, used to communicate the schema before data transfer
- begins.
-
-The IPC format is used both for inter-process communication and as the wire protocol for remote
-source execution in the [Scan API](/concepts/scanning).
-
-:::{note}
-The IPC format is unstable and subject to change. It does not yet support shared arrays (e.g.
-a dictionary shared across multiple chunked arrays), which limits its efficiency for certain
-workloads. This is an area of active development.
-:::
-
-## Segment Storage
-
-In a Vortex file, data buffers are stored as segments -- contiguous byte ranges at known offsets.
-Each segment is described by a `SegmentSpec` containing:
-- **offset** -- byte position from the start of the file.
-- **length** -- size in bytes.
-- **alignment** -- required memory alignment (as a power-of-two exponent).
-
-Layouts reference segments by `SegmentId`, which is an index into the footer's segment table.
-This indirection allows the same layout tree to be backed by different segment sources (local
-file, object store, in-memory cache, etc.) without changing the layout structure.
-
-## File Footer
-
-The file footer is the entry point for reading a Vortex file. It is read from the end of the
-file and contains everything needed to reconstruct the layout tree and locate data segments.
-
-The last 8 bytes of the file contain:
-- A version number (2 bytes).
-- The postscript length (2 bytes).
-- A magic number (4 bytes).
-
-The postscript locates four regions by offset and length:
-- **DType** -- the schema, stored as a FlatBuffer (optional if embedded in the layout).
-- **Layout** -- the layout tree, stored as a FlatBuffer.
-- **Statistics** -- per-column file-level statistics (optional).
-- **Footer** -- dictionaries of encoding IDs, layout IDs, segment specs, and compression
- configs.
-
-The layout FlatBuffer is a tree of `Layout` nodes, each containing an encoding ID, row count,
-metadata, child layouts, and segment indices. This tree is deserialized and bound to a segment
-source to create a `LayoutReader` that can lazily fetch data on demand.
-
-## FlatBuffers
-
-Vortex uses FlatBuffers rather than Protocol Buffers or a custom binary format because
-FlatBuffers support O(1) random access into the serialized data without parsing the entire
-message. This is important for wide schemas where only a few columns are accessed per query --
-the reader can jump directly to the relevant layout node without deserializing the rest of the
-footer.
-
-All FlatBuffers in Vortex are aligned to 8 bytes. Schema definitions live in the
-`vortex-flatbuffers` crate and cover arrays, layouts, the file footer, and IPC messages.
-
-## Zero-Copy Design
-
-The alignment and padding system is designed so that serialized buffers can be used directly
-as in-memory arrays without copying. When a segment is read from disk or received over the
-network, the I/O subsystem allocates an aligned buffer matching the segment's alignment
-requirement. The resulting buffer handle can be used directly by the array without
-reallocating or copying the data.
-
-This property holds across all three contexts: in-memory arrays, on-disk file segments, and
-over-the-wire IPC messages all use the same layout and alignment conventions.
diff --git a/docs/developer-guide/language-bindings.md b/docs/developer-guide/language-bindings.md
index 8905364910b..e1943e10dea 100644
--- a/docs/developer-guide/language-bindings.md
+++ b/docs/developer-guide/language-bindings.md
@@ -20,7 +20,7 @@ support can reach this tier.
Filter and projection pushdown via expressions. Expressions can be serialized as protobuf bytes or
constructed natively in the host language. Results are still returned as Arrow streams, making this
-tier suitable for query engine integrations (e.g. DataFusion, DuckDB, Spark, Trino).
+tier suitable for query engine integrations (e.g. DataFusion, DuckDB, Spark).
**Capabilities:** everything in Tier 0, plus scan builder with filter, projection, limit, and row
range pushdown, and expression construction.
@@ -71,8 +71,8 @@ layout plugins, and extension DTypes.
|---------------|--------------|-------------|------------|-----------------------------------------------|
| Rust | 3 | 3 | Native | Future: stable plugin API via C ABI |
| Python | ~2 | 3 | PyO3 | Already has native expressions + array access |
-| C | ~1 | 2 | cbindgen | Foundation ABI for all non-Rust bindings |
-| C++ | ~1 | 2 | cxx -> C | Migrate from cxx to wrap C API |
+| C | ~1 | 2 | cbindgen | Intended foundation ABI for non-Rust bindings |
+| C++ | ~1 | 2 | cxx | Currently a direct cxx bridge; plan to wrap C |
| Java (JNI) | ~1 | 1 | JNI | Broad JDK compatibility, Arrow-based |
| Java (Panama) | — | 2 | Panama FFI | Direct C ABI access, requires JDK 22+ |
@@ -90,7 +90,12 @@ formalizing which APIs are stable vs experimental and exposing plugin registrati
### C
-The C API (generated via cbindgen) is the **foundation ABI** for non-Rust bindings. It currently
+
+The C API (generated via cbindgen) is the **intended foundation ABI** for non-Rust bindings (Python
+uses PyO3 and C++ uses a `cxx` bridge today; C++ is planned to migrate onto it, while Python continues
+to use PyO3). It currently
provides Tier ~1 capabilities (file I/O and basic scan). The C API is not yet ABI-stable — it
evolves with the project and should be statically linked. In the future, a subset of the API will
be flagged as stable for use via dynamic linking. The target is Tier 2, which requires array tree
@@ -104,13 +109,12 @@ wrapping the C API directly, providing RAII wrappers and CMake integration. Targ
### Java (JNI)
Java JNI bindings provide Tier ~1 capabilities today (Arrow I/O and basic scan). JNI will remain
-at Tier 1 for broad JDK compatibility. This is the current integration point for Spark and Trino
-connectors.
+at Tier 1 for broad JDK compatibility. This is the current integration point for the Spark connector,
+and the foundation any future JVM connector would build on.
### Java (Panama)
A new binding layer using Java's Foreign Function & Memory API (Panama) to call the C API
directly. Panama enables native array access without JNI overhead, targeting Tier 2. Requires
-JDK 22+. Trino already supports JDK 22 and can adopt Panama immediately. Spark targets older LTS
-releases and will not support Panama for some time, so the JNI path remains essential for Spark
-integration.
+JDK 22+. Spark targets older LTS releases and will not support Panama for some time, so the JNI path
+remains essential for Spark integration.
diff --git a/docs/developer-guide/overview.md b/docs/developer-guide/overview.md
new file mode 100644
index 00000000000..2cc2f029f4c
--- /dev/null
+++ b/docs/developer-guide/overview.md
@@ -0,0 +1,49 @@
+# Overview
+
+The Developer Guide is for people working *on* Vortex or building *against* its lower-level
+interfaces — as opposed to the [User Guide](../user-guide/index.md), which is for people querying
+Vortex data with engines and dataframe libraries. It is organized by what you are trying to do.
+
+## How this guide is organized
+
+- **Extending** — add new capabilities to Vortex: custom [encodings](extending/writing-an-encoding.md),
+ [layouts](extending/writing-a-layout.md), [compute functions](extending/writing-a-compute-fn.md),
+ and [extension dtypes](extending/extension-dtypes.md). Start here if you want Vortex to understand a
+ new compression scheme or logical type.
+- **Embedding** — drive Vortex from another system or language: the [C FFI](embedding/ffi.md),
+ [C++ binding](embedding/cxx.md), [Scan API](embedding/scan-api.md), and [GPU](embedding/gpu.md)
+ paths. Start here if you are building a query-engine connector or a data source.
+- **Internals** — how the implementation actually works: the [crate architecture](internals/architecture.md),
+ [vtables](internals/vtables.md), [session system](internals/session.md),
+ [async runtime](internals/async-runtime.md), [execution](internals/execution.md),
+ [I/O subsystem](internals/io.md), and [CUDA](internals/cuda.md). Start here if you are contributing
+ to the core.
+- **Integrations** — implementation notes for the engine connectors
+ ([DataFusion](integrations/datafusion.md), [DuckDB](integrations/duckdb.md),
+ [Spark](integrations/spark.md)). These complement the user-facing how-tos in the User Guide with
+ the *why* and *how* of each connector.
+- **[Language Bindings](language-bindings.md)** and **[Benchmarking](benchmarking.md)** round out the
+ guide.
+
+:::{note}
+Several pages in **Extending** and **Embedding** are still under construction — they currently list
+their planned contents rather than full walkthroughs. The **Internals** and **Integrations** sections
+are the most complete. If a page you need is a stub, the [API reference](../api/index.md) and the
+source are the best fallback, and the [Slack community](https://vortex.dev/slack) can help.
+:::
+
+## A note on where the difficulty lives
+
+A recurring theme across this guide — and a good lens for understanding Vortex's design — is that
+**reading the bytes of a Vortex file is not the hard part.** As the
+[Reading a File](../specification/reading-a-file.md) walkthrough shows, a correct reader is a magic
+number, a postscript, a footer, the root dtype, and a layout tree: an afternoon's work.
+
+**The hard part is getting *high performance* out of those bytes** — pruning irrelevant row ranges
+from zone statistics, pushing projections and filters down so you fetch and decompress as little as
+possible, and scheduling I/O so thousands of small segment reads become a few large coalesced ones.
+That is where the bulk of Vortex's engineering goes, and it is why the
+[concepts](../concepts/index.md) (layouts, the scan API) and the
+[internals](internals/io.md) sections exist. If you only ever need to read a file, you can stop at
+the [specification](../specification/index.md). If you need it to be *fast*, this guide is where the
+real work is documented.
diff --git a/docs/getting-started/convert.md b/docs/getting-started/convert.md
index cc4046422f9..394933e37a8 100644
--- a/docs/getting-started/convert.md
+++ b/docs/getting-started/convert.md
@@ -13,7 +13,7 @@ vx convert yellow_tripdata_2024-01.parquet
```
This produces `yellow_tripdata_2024-01.vortex` in the same directory. By default it uses
-BtrBlocks compression, chunking on Parquet row-group boundaries.
+BtrBlocks compression, chunking into 8192-row batches.
## Compression strategies
diff --git a/docs/getting-started/index.md b/docs/getting-started/index.md
index 72b88db57d4..62ec5e08db6 100644
--- a/docs/getting-started/index.md
+++ b/docs/getting-started/index.md
@@ -24,5 +24,5 @@ caption: Language Libraries
Python Quickstart
Rust Quickstart
-Java Quickstart
+Java (work in progress)
```
diff --git a/docs/getting-started/install.md b/docs/getting-started/install.md
index 9ff30a30fcd..acc4e31856c 100644
--- a/docs/getting-started/install.md
+++ b/docs/getting-started/install.md
@@ -47,7 +47,7 @@ vx --help
The `vx` CLI is the quickest way to get started, but Vortex can also be used as a library
for reading, writing, and manipulating compressed arrays programmatically. See the language
-quickstarts for [Python](python.rst), [Rust](rust.rst), and [Java](java.md).
+quickstarts for [Python](python.rst) and [Rust](rust.rst) (a [Java](java.md) guide is in progress).
## Sample data
diff --git a/docs/index.md b/docs/index.md
index b75a438f913..4cb016d9b45 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -16,64 +16,39 @@ sd_hide_title: true
:align: center
:::
-An extensible ecosystem for compressed columnar data. Spans in-memory arrays,
-on-disk file formats, over-the-wire protocols, and integrations with query engines — all built
-around the latest research from the database community.
+An extensible ecosystem for compressed columnar data. Vortex spans in-memory arrays,
+on-disk file formats, over-the-wire protocols, and query-engine integrations — all built around
+recent research from the database community.
-## Where to start
+These docs are organized by what you're trying to do. If you're **using** Vortex, start with the
+guides and API reference. If you're **building on or contributing to** Vortex, start with the
+concepts and the format specification.
-::::{grid} 1 2 2 3
-:gutter: 3
+## Using Vortex
-:::{grid-item-card} Read & write Vortex files
-:link: getting-started/index
-:link-type: doc
+**[Getting Started](getting-started/index)** — Install the `vx` command-line tool or a language
+binding, convert a Parquet file, and run your first query in Python or Rust.
-Get started with Vortex in **Python**, **Rust**, or **Java**. Convert from Parquet, compress
-your data, and query it.
-:::
-
-:::{grid-item-card} Use with a query engine
-:link: user-guide/index
-:link-type: doc
-
-Integrate Vortex with **DataFusion**, **DuckDB**, **Spark**, **Trino**, or **Ray** for
-accelerated queries over compressed data.
-:::
-
-:::{grid-item-card} Understand the architecture
-:link: concepts/index
-:link-type: doc
-
-Learn how **DTypes**, **Arrays**, **Encodings**, **Layouts**, and the **Scan API** fit together
-as building blocks.
-:::
+**[User Guide](user-guide/index)** — Query Vortex data with **DataFusion**, **DuckDB**, **Spark**,
+or **Ray**, and move data to and from **pandas**, **Polars**, and
+**PyArrow**.
-:::{grid-item-card} Extend Vortex
-:link: developer-guide/index
-:link-type: doc
+**[API Reference](api/index)** — Reference for the **Python**, **Rust**, **Java**, **C**, and **C++**
+interfaces. The Rust and Python APIs are the most complete; the C, C++, and Java bindings are still
+evolving.
-Write your own **encodings**, **layouts**, **compute functions**, or **extension types** from
-Rust or Python.
-:::
-
-:::{grid-item-card} Create an engine integration
-:link: developer-guide/index
-:link-type: doc
+## Understanding & extending Vortex
-Build a **query engine connector** or **data source** using the **Scan API**, **C FFI**, or
-**C++ wrapper**.
-:::
+**[Concepts](concepts/index)** — The mental model behind Vortex: how **DTypes**, **Arrays**,
+**Encodings**, **Layouts**, and the **Scan API** fit together as composable building blocks.
-:::{grid-item-card} Internals
-:link: developer-guide/index
-:link-type: doc
+**[Specification](specification/index)** — The on-disk file format (stable) and the over-the-wire IPC
+protocol (still unstable), plus a step-by-step walkthrough of how to **read a Vortex file** from
+scratch. The starting point for implementing a reader or porting Vortex to a new language.
-Explore the **crate architecture**, **async runtime**, **session system**, and integration
-internals. Build and benchmark locally.
-:::
-
-::::
+**[Developer Guide](developer-guide/index)** — **Extend** Vortex with your own encodings, layouts,
+compute functions, and types; **embed** it through the C FFI, C++ binding, or Scan API; or dig into
+the **internals**.
## Highlights
@@ -82,14 +57,14 @@ internals. Build and benchmark locally.
[FSST](https://github.com/spiraldb/fsst), and
[ALP](https://github.com/spiraldb/alp) — no decompression needed for many operations.
-- **Extensible file format**: Zero-allocation reads, FlatBuffer metadata for O(1) column access,
- and optional WASM decompression kernels for forward compatibility.
+- **Extensible file format**: Zero-allocation reads and FlatBuffer metadata for O(1) column access,
+ with a layout system designed to evolve without breaking existing readers.
- **Query engine integration**: Filter and projection pushdown through the Scan API, with native
- integrations for DataFusion, DuckDB, Spark, Trino, and Ray.
+ integrations for DataFusion, DuckDB, Spark, and Ray.
-- **Language bindings**: First-class support for Python (PyO3), Java (JNI + Spark/Trino connectors),
- and C/C++ (FFI).
+- **Language bindings**: First-class Python (PyO3) and Rust support, with Java (JNI), C (FFI), and
+ C++ (cxx) bindings evolving.
```{toctree}
---
@@ -97,10 +72,10 @@ hidden:
---
getting-started/index
-concepts/index
user-guide/index
+concepts/index
+specification/index
developer-guide/index
-specs/index
api/index
project/index
```
diff --git a/docs/project/bindings.md b/docs/project/bindings.md
index 3311a7a3663..f863b59d11b 100644
--- a/docs/project/bindings.md
+++ b/docs/project/bindings.md
@@ -25,6 +25,10 @@ libraries.
Each language chooses the binding strategy that makes the most sense:
+
+
- **Python** uses PyO3 to call Rust directly and will continue to do so. PyO3 provides the best
ergonomics and performance for Python.
- **Other languages** use Rust-specific bindings where it makes sense (e.g. cxx for C++), or wrap
@@ -66,7 +70,7 @@ This is a future concern — the current priority is stabilizing the C API for n
Java's Foreign Function & Memory API (Panama) graduated to a stable API in JDK 22. Key
considerations:
-- Trino already supports JDK 22 and can adopt Panama immediately.
+- Engines already on JDK 22+ can adopt Panama immediately.
- Spark targets older LTS releases and will not support Panama for some time.
- JNI must remain supported as the primary path for Spark and any other engine on older JDKs.
- Panama provides direct C ABI access, eliminating JNI overhead and enabling Tier 2 capabilities.
@@ -117,9 +121,9 @@ releases. JNI stays at Tier 1.
Build a new binding layer using Panama's Foreign Function & Memory API to call the C API directly.
This enables native array access, lower overhead, and a path to Tier 2 capabilities. Panama
-bindings are opt-in for environments running JDK 22+. Trino already supports JDK 22 and is the
-likely first adopter. Spark will not support Panama until it moves to a compatible JDK. Connectors
-should abstract over the binding layer so they can use JNI or Panama transparently.
+bindings are opt-in for environments running JDK 22+. Spark will not support Panama until it moves
+to a compatible JDK. Connectors should abstract over the binding layer so they can use JNI or Panama
+transparently.
### Rust
@@ -153,7 +157,7 @@ Remains native Tier 3. Future considerations:
- Design the plugin registration interface in the C API (Tier 3 capabilities).
- Formalize the Python plugin API (array plugins, compute plugins, extension DTypes).
- Investigate Rust stable plugin ABI for dynamic loading of encoding crates.
-- Evaluate Panama adoption timeline based on Spark/Trino JDK requirements.
+- Evaluate Panama adoption timeline based on Spark's JDK requirements.
## Expression Strategy
diff --git a/docs/project/community.md b/docs/project/community.md
index 2ef62e41dc5..4cade675ef7 100644
--- a/docs/project/community.md
+++ b/docs/project/community.md
@@ -33,6 +33,16 @@ technical direction, approving proposals, and establishing community norms and r
The TSC aims to reach consensus; when a vote is needed, decisions require a majority of those
present with at least 50% quorum.
+### Membership
+
+The TSC's voting members are the project **Maintainers**; its non-voting members are the
+**Committers** (see [Governance](#governance) above for the role definitions). The current roster —
+including the TSC chair — is maintained in
+[CONTRIBUTING.md](https://github.com/vortex-data/vortex/blob/develop/CONTRIBUTING.md#project-roles)
+so it stays in one place as membership changes.
+
+### Meetings
+
TSC meetings are open to the public and held monthly. Meeting details and agendas are posted
in the Slack workspace. See the
[public meeting calendar](https://zoom-lfx.platform.linuxfoundation.org/meetings/vortex)
diff --git a/docs/project/roadmap.md b/docs/project/roadmap.md
index f2a6d8ec7b7..ae5940b6c09 100644
--- a/docs/project/roadmap.md
+++ b/docs/project/roadmap.md
@@ -26,8 +26,9 @@ still benefiting from Vortex's encoding, compression, and I/O infrastructure.
### Scan API
-An abstract table-scan interface that positions Vortex as an interchange layer between data sources
-and query engines. The Scan API will support pluggable data sources and can be consumed over the
+The Scan API (the `vortex-scan` crate) already provides an abstract table-scan interface with
+pluggable data sources (the `DataSource` trait), positioning Vortex as an interchange layer between
+data sources and query engines. Upcoming work stabilizes that surface and adds consumption over the
C ABI for in-process integrations or over RPC for remote/distributed access.
### Language Bindings Overhaul
@@ -50,5 +51,6 @@ fixed-shape and variable-shape tensors within Vortex arrays and files.
### Variant DType
-A DType for representing arbitrarily nested, JSON-like data within Vortex arrays and files. This
-enables efficient columnar storage and querying of semi-structured data.
+The `Variant` DType already exists in the Vortex type system. It represents arbitrarily nested,
+JSON-like data within arrays and files, enabling efficient columnar storage and querying of
+semi-structured data; upcoming work completes its encoding and compute support.
diff --git a/docs/specification/array-format.md b/docs/specification/array-format.md
new file mode 100644
index 00000000000..e2819395389
--- /dev/null
+++ b/docs/specification/array-format.md
@@ -0,0 +1,243 @@
+# Array Format
+
+Vortex uses the **same binary representation for arrays in memory, on disk, and over the wire**. A
+serialized array is the unit that a [file](file-format.md) segment or an [IPC](ipc-format.md) message
+ultimately carries, and it is laid out so that its buffers can be used as in-memory arrays without
+copying.
+
+This page specifies that representation precisely enough to locate and decode any buffer from the
+bytes alone. What the bytes *inside* a node mean — encoding by encoding — is the separate
+[Encoding Format](encoding-format.md) reference; this page defines the container those encodings sit
+in.
+
+## Structure
+
+A serialized array consists of two parts: a FlatBuffer describing the array tree, and a sequence of
+data buffers.
+
+The FlatBuffer is a single `Array` message. It holds a tree of `ArrayNode`s (the `root` node plus
+its transitively-nested `children`) and one flat `buffers` table shared by the whole tree. Each
+`ArrayNode` records:
+
+- The **encoding** — an interned `u16` index into the encoding registry (see
+ [Decoding a node](#decoding-a-node)).
+- Array-specific **metadata** bytes, interpreted according to the node's encoding.
+- Its **children**, embedded inline as further `ArrayNode`s.
+- Its **buffers** — a list of `u16` indices into the shared buffer table.
+- Optional **statistics** (min, max, null count, sort order, …).
+
+A node stores **neither its length nor its dtype** — both are supplied top-down by the parent,
+ultimately from the file or IPC schema. This is why decoding always takes the logical length `n` and
+the `DType` as inputs rather than parsing them from the node.
+
+The data buffers are written **first**; the `Array` FlatBuffer is appended as a suffix after them,
+followed by a little-endian `u32` giving the FlatBuffer's length (see [Wire layout](#wire-layout)).
+Serialization takes a `SerializeOptions`: when zero-copy padding is enabled (the default for files),
+padding is inserted before each buffer to meet its alignment; serializers that disable it — such as
+the default IPC encoder — omit the padding.
+
+## FlatBuffer schema
+
+The array tree and buffer table are defined by `array.fbs`:
+
+:::{literalinclude} ../../vortex-flatbuffers/flatbuffers/vortex-array/array.fbs
+:start-at: /// An Array describes
+:::
+
+`Array` is the schema's `root_type`: `root` is the top `ArrayNode`, and `buffers` is the single,
+tree-global buffer table (see [The buffer table](#the-buffer-table)). `ArrayNode` is a FlatBuffer
+**table** — its fields are optional and the format can add new ones without breaking older readers.
+`Buffer` is a FlatBuffer **struct**: a fixed, inline, 8-byte record with no vtable, which by
+construction cannot gain or lose fields (see [The Buffer struct](#the-buffer-struct)).
+
+## Wire layout
+
+On the wire, a serialized array is:
+
+```
+[pad?] [buffer 0] [pad?] [buffer 1] ... [pad?] [Array flatbuffer] [u32 flatbuffer length, LE]
+```
+
+The data-buffer region comes first, the `Array` FlatBuffer is the suffix, and the final 4 bytes are
+the little-endian `u32` length of that FlatBuffer. A reader recovers the pieces from the end inward:
+read the trailing `u32` length `L`; the FlatBuffer occupies the `L` bytes ending 4 bytes before the
+end; everything before it is the data-buffer region (`vortex-array/src/serde.rs` `SerializedArray::try_from`).
+
+Each `[pad?]` is alignment padding inserted **only when zero-copy padding is enabled**: before each
+data buffer to meet its alignment, and once more before the FlatBuffer to 8-align it. When padding is
+disabled (the IPC encoder's default), the `[pad?]` slots are empty and every `Buffer.padding` is `0`.
+
+## The buffer table
+
+`Array.buffers` is a single vector of `Buffer` structs describing every data buffer in the whole node
+tree. It is a two-level indirection: a node names its buffers by index into this shared table, and
+each entry says where its bytes live in the data-buffer region.
+
+### The Buffer struct
+
+Because `Buffer` is a FlatBuffer *struct* rather than a table, each entry is a fixed **8-byte** inline
+record — the schema comment calls it a "packed 64-bit struct"
+(`vortex-flatbuffers/flatbuffers/vortex-array/array.fbs` `Buffer`) — stored back-to-back with no offsets or
+vtable. Its four fields, in declaration order, describe one data buffer:
+
+| Byte offset | Field | Type | Size | Meaning |
+|-------------|-------|------|------|---------|
+| 0 | `padding` | `uint16` | 2 | Number of pad bytes written **immediately before** this buffer. |
+| 2 | `alignment_exponent` | `uint8` | 1 | The buffer's minimum alignment, as an exponent of 2 (alignment = `2^alignment_exponent` bytes). |
+| 3 | `compression` | `Compression` (`uint8`) | 1 | Compression codec applied to the buffer (see [Buffer compression](#buffer-compression)). |
+| 4 | `length` | `uint32` | 4 | The buffer's *stored* length in bytes (the compressed length if compressed). |
+
+All scalars are little-endian. The record's total size (8 bytes) is fixed by the schema, so a reader
+indexes the table by multiplying the index by 8 rather than following an offset.
+
+### Locating buffer bytes
+
+A `Buffer` descriptor gives a buffer's length and its immediately-preceding padding, but **not** an
+absolute offset — a reader recovers the offset by walking the table in order. Within the data-buffer
+region (the prefix identified in [Wire layout](#wire-layout)), buffers are laid out
+`[pad_0][buffer_0][pad_1][buffer_1]…`, where each `pad_j` is exactly `buffers[j].padding` bytes. To
+locate buffer `k` (an absolute index into `Array.buffers`):
+
+```
+offset = 0
+for j in 0 .. k:
+ offset += buffers[j].padding # pad bytes written before buffer j
+ offset += buffers[j].length # buffer j's own bytes
+offset += buffers[k].padding # pad bytes written before buffer k
+bytes = region[offset .. offset + buffers[k].length]
+```
+
+Equivalently: accumulate `padding_j + length_j` over every `j < k`, then add `padding_k`. This is
+exactly the reference reader's walk, which advances a running `offset` by each descriptor's `padding`
+then `length` (`vortex-array/src/serde.rs` `SerializedArray::from_flatbuffer_and_segment_with_overrides`), mirroring the writer that emits `padding` zero
+bytes before each buffer and records that count in the descriptor
+(`vortex-array/src/serde.rs` `ArrayRef::serialize`). When padding is disabled every `padding` is `0`, so the buffers
+are simply concatenated.
+
+Each buffer is then aligned to `2^alignment_exponent`. On the padded (file) path the padding already
+places the buffer at an aligned offset, so its bytes can be used in place; otherwise a reader may have
+to copy to satisfy alignment (see [Zero-copy design](#zero-copy-design)).
+
+### Tree-global indices
+
+There is **one** buffer table for the entire node tree — `Array.buffers` — not one per node. Buffer
+indices are assigned **pre-order**: a node's own buffers come first, immediately followed by all of
+its first child's buffers (recursively), then the next child's, and so on. Concretely, a node
+occupying the index range `[b, b + m)` for its `m` own buffers hands its first child the starting
+index `b + m`, and each subsequent child starts after its predecessor's entire *recursive* buffer
+count (`vortex-array/src/serde.rs` `ArrayNodeFlatBuffer::try_write_flatbuffer`; the writer collects the actual buffer bytes by the same
+pre-order tree walk, `vortex-array/src/serde.rs` `ArrayRef::serialize`).
+
+Each `ArrayNode.buffers` list therefore holds that node's **absolute** indices into the shared table.
+This indirection is the bridge between the per-encoding pages and the bytes:
+
+:::{important}
+When an [Encoding Format](encoding-format.md) section says "**buffer `i`**", it means the *node-local*
+position `i` — that is, `ArrayNode.buffers[i]`. Resolve that to an absolute index into
+`Array.buffers`, then locate its bytes with the walk in [Locating buffer bytes](#locating-buffer-bytes).
+:::
+
+A node's own indices are contiguous and ascending, which lets a reader slice all of its buffers as one
+range (`vortex-array/src/serde.rs` `SerializedArray::collect_buffers`), but the format only requires that each
+`ArrayNode.buffers[i]` be a valid index into `Array.buffers`.
+
+### Buffer compression
+
+`Buffer.compression` selects a per-buffer compression codec from the `Compression` enum
+(`vortex-flatbuffers/flatbuffers/vortex-array/array.fbs` `Compression`):
+
+| Value | Name | Meaning |
+|-------|------|---------|
+| 0 | `None` | The buffer's bytes are stored verbatim. |
+| 1 | `LZ4` | The buffer's bytes are LZ4-compressed. |
+
+This is distinct from Vortex's array *encodings* (which are lossless logical re-representations of the
+values): it is an opaque byte-level codec applied to an already-encoded buffer's bytes. When
+`compression = LZ4`, `Buffer.length` is the **compressed** byte count — the value the offset walk in
+[Locating buffer bytes](#locating-buffer-bytes) uses — and a reader must LZ4-decompress those bytes
+into a freshly allocated, aligned buffer before using them. Such a buffer therefore **cannot be used
+zero-copy**: the in-place slice holds compressed bytes, so a decode-and-allocate step is unavoidable
+for it. `None` buffers keep the zero-copy property.
+
+:::{note}
+Reference-implementation status. Buffer-level `LZ4` is defined and reserved in the schema, but the
+reference writer currently emits `None` for every buffer (`vortex-array/src/serde.rs` `ArrayRef::serialize`) and the
+reference slice path does not decompress (`vortex-array/src/serde.rs` `SerializedArray::from_flatbuffer_and_segment_with_overrides`). The framing a decoder
+would need for `LZ4` — in particular where the *uncompressed* length comes from — is not yet pinned by
+the schema (the node-level `uncompressed_size_in_bytes` statistic is advisory, not a per-buffer
+field). Treat `LZ4` as forward-compatible reservation: a reader **must** recognise the enum value and
+**must** fail loudly rather than hand compressed bytes to a decoder expecting raw ones.
+:::
+
+## Node statistics
+
+Each `ArrayNode` may carry an `ArrayStats` table
+(`vortex-flatbuffers/flatbuffers/vortex-array/array.fbs` `ArrayStats`) summarising the node's values:
+
+- `min`, `max`, `sum` — each a **Protobuf-serialized `ScalarValue`** (a bare value, no dtype). `min`
+ and `max` decode against the node's `DType`; `sum` decodes against a **widened accumulator dtype
+ derived from** the node's `DType` (not the `DType` verbatim). The [Scalar Format](scalar-format.md)
+ specifies how to decode the `ScalarValue` bytes **once that dtype is known** — it does not itself
+ define the widening; because statistics are advisory, a reader that does not reproduce the `sum`
+ widening may skip `sum`. `min` and `max` each pair with a `Precision`
+ (`vortex-flatbuffers/flatbuffers/vortex-array/array.fbs` `Precision`): `Exact` means the bound is the
+ true extreme, while `Inexact` means it is only a valid lower/upper bound that may not be tight — a
+ reader pruning on it must treat it conservatively.
+- `is_sorted`, `is_strict_sorted`, `is_constant` — tri-state booleans whose default is `null`
+ (unknown).
+- `null_count`, `nan_count`, `uncompressed_size_in_bytes` — `uint64`s whose default is `null`
+ (unknown).
+
+Every field is optional; an absent field — or an absent `ArrayStats` altogether — means "unknown",
+never a computed value. **Statistics are advisory.** They drive pruning and pushdown decisions (for
+example skipping a node whose `max` is below a filter bound, or short-circuiting a `min`/`max`
+aggregate), but a reader may ignore them entirely and still decode correct data — the reference reader
+only *populates* a node's statistics set from them when they are present
+(`vortex-array/src/serde.rs` `SerializedArray::decode`). A writer must never store a statistic it cannot prove.
+
+## Decoding a node
+
+Decoding turns a parsed node into a typed array. The node carries no length or dtype, so both are
+inputs, supplied top-down from the schema. For one node:
+
+1. **Resolve the encoding.** `ArrayNode.encoding` is a `u16` index into the encoding registry carried
+ by the enclosing [file](file-format.md) or [IPC stream](ipc-format.md) — the interned list that
+ maps indices to registry encoding IDs. Look the index up to obtain the ID (for example
+ `vortex.primitive`). An index with no registry entry is an error, and an ID the reader does not
+ implement **must** fail loudly rather than be guessed — unless the reader explicitly opts into
+ opaque foreign passthrough (`session.allows_unknown()`), which preserves the node's metadata,
+ buffers, and children verbatim without interpreting them (`vortex-array/src/serde.rs` `SerializedArray::decode`).
+2. **Interpret the components per that encoding.** The [Encoding Format](encoding-format.md) page for
+ the resolved ID defines how to read the node's `metadata` bytes, its referenced buffers (via the
+ [tree-global index resolution](#tree-global-indices) above), and its children — using the supplied
+ `n` and `DType`. Children are decoded recursively by the same procedure; the parent encoding
+ derives each child's dtype and length.
+3. **Optionally apply statistics.** If `stats` is present a reader may load it (see
+ [Node statistics](#node-statistics)) or ignore it.
+
+This contract is language-neutral: the FlatBuffer supplies structure and byte locations, and the
+per-encoding pages supply meaning. No step depends on a particular runtime's types.
+
+## Why FlatBuffers
+
+Vortex uses FlatBuffers rather than Protocol Buffers or a custom binary format because FlatBuffers
+support O(1) random access into the serialized data without parsing the entire message. This matters
+for wide schemas where only a few columns are accessed per query — the reader can jump directly to
+the relevant node without deserializing the rest.
+
+All FlatBuffers in Vortex are aligned to 8 bytes. Schema definitions live in the `vortex-flatbuffers`
+crate and cover arrays, layouts, the file footer, and IPC messages.
+
+## Zero-copy design
+
+The alignment and padding system is designed so that serialized buffers can be used directly as
+in-memory arrays without copying — *when the padding is present and the buffer is uncompressed*. This
+is what the on-disk [File Format](file-format.md) is built around: it writes the padding each buffer's
+alignment requires, so when the I/O subsystem reads a segment into an aligned buffer, a
+[file reader](reading-a-file.md) can hand the fetched bytes straight to an array — no reallocation,
+no copy.
+
+The [IPC format](ipc-format.md) reuses the same array representation but its default encoder omits the
+inter-buffer padding, so a receiver may need to copy or realign some buffers rather than use them in
+place. Zero-copy is therefore a property of the *padded, uncompressed* layout (the file path), not of
+the array representation in every context.
diff --git a/docs/specification/dtype-format.md b/docs/specification/dtype-format.md
new file mode 100644
index 00000000000..1fc496e194b
--- /dev/null
+++ b/docs/specification/dtype-format.md
@@ -0,0 +1,132 @@
+# DType Format
+
+This page is the serialization reference for Vortex's logical type system. For what the types
+*mean* — the distinction between logical and physical types, the full list of built-in types, and
+how Vortex compares to Arrow — see the [DTypes concept page](../concepts/dtypes.md).
+
+## Model
+
+A `DType` is a single logical type drawn from a closed `union` of type variants. Three properties of
+the encoding are worth calling out, because they differ from many other columnar formats:
+
+- **Nullability is part of the type.** Each built-in value variant carries its own `nullable` flag,
+ rather than nullability living on a separate schema/field object. `Null` is its own variant. The
+ exception is `Extension`, which has no `nullable` field — its nullability is inherited from its
+ `storage_dtype`.
+- **There is no separate schema type.** Columnar data is just a `Struct_` whose `names` and
+ `dtypes` line up positionally. A file can equally hold a bare `Primitive` or `Utf8` at the root —
+ see the [File Format](file-format.md).
+- **Nested types embed their children.** `List`, `FixedSizeList`, and `Struct_` contain child
+ `DType`s directly, so a type is a self-contained tree. `Extension` wraps a `storage_dtype` with an
+ `id` and `metadata` bytes that together narrow its domain (for example, `vortex.date` over an
+ `I32`). The `metadata` bytes are opaque **at the DType layer**, but Vortex's built-in extensions
+ define concrete metadata formats and required storage dtypes — see
+ [Built-in extension types](#built-in-extension-types).
+
+## Serializations
+
+Vortex serializes DTypes two ways, for two different jobs:
+
+- **FlatBuffer** (`dtype.fbs`) is the canonical, zero-copy serialization. It is what the
+ [File Format](file-format.md) stores in its root `DType` segment (when the schema is embedded
+ rather than supplied externally) and what the [IPC format](ipc-format.md) sends in a
+ `DTypeMessage`. It is the schema-on-the-wire encoding.
+- **Protobuf** (`dtype.proto`) mirrors the same model for contexts where a DType is embedded as
+ *metadata* inside something else — notably compute expressions and the metadata of arrays and
+ extension types. The two schemas describe the same logical types.
+
+### FlatBuffer definition
+
+The `Type` union is the heart of the schema; `DType` is just a wrapper around it. New variants are
+appended to the union to preserve backward compatibility — note that `FixedSizeList` sits after
+`Extension` for exactly this reason.
+
+:::{literalinclude} ../../vortex-flatbuffers/flatbuffers/vortex-dtype/dtype.fbs
+:start-at: enum PType
+:::
+
+### Protobuf definition
+
+:::{literalinclude} ../../vortex-proto/proto/dtype.proto
+:language: protobuf
+:start-at: syntax =
+:::
+
+## Built-in extension types
+
+An `Extension`'s `metadata` bytes are opaque at the DType layer, but Vortex registers four
+**built-in** extensions by default (`vortex-array/src/dtype/session.rs` `DTypeSession::default`),
+each with a concrete metadata format and a required `storage_dtype`. A reader needs both to
+*interpret* such a column (as opposed to merely decoding the raw storage values):
+
+| `id` | Metadata bytes | Storage dtype | Meaning |
+|------|----------------|---------------|---------|
+| `vortex.date` | `[unit_tag: u8]` | `Days` → `I32`, `Milliseconds` → `I64` | days / ms since the Unix epoch; only these two `TimeUnit`s are valid (`vortex-array/src/extension/datetime/date.rs` `date_ptype`, `serialize_metadata`) |
+| `vortex.time` | `[unit_tag: u8]` | `Nanoseconds`/`Microseconds` → `I64`, `Milliseconds`/`Seconds` → `I32` | time-of-day (`vortex-array/src/extension/datetime/time.rs` `time_ptype`) |
+| `vortex.timestamp` | `[unit_tag: u8][tz_len: u16 LE][tz: UTF-8]` | `I64` | instant; full layout in the [DateTimeParts encoding](encoding-format/misc.md) (`vortex-array/src/extension/datetime/timestamp.rs` `serialize_metadata`) |
+| `vortex.uuid` | empty (any version) or `[version: u8]` | `FixedSizeList(non-nullable U8, 16)` | 16 raw bytes per value (`vortex-array/src/extension/uuid/vtable.rs` `serialize_metadata`) |
+
+The `unit_tag` byte is the `TimeUnit` enum — `0` nanoseconds, `1` microseconds, `2` milliseconds,
+`3` seconds, `4` days (`vortex-array/src/extension/datetime/unit.rs` `TimeUnit`).
+
+**Unknown extension ids.** An `Extension` whose `id` is not registered is preserved as an opaque
+**foreign** extension (`vortex-array/src/dtype/extension/foreign.rs`): a reader keeps the
+`storage_dtype`, `id`, and `metadata` bytes verbatim and can decode the raw storage values, but
+cannot interpret the extension's domain semantics — the same opaque-passthrough posture the array
+layer takes for an unrecognised encoding.
+
+## Field names
+
+A `Struct_` stores its field names as a list of FlatBuffer strings (`names: [string]` in the
+FlatBuffer definition above), and in memory a field name is just an `Arc` whose contents are
+unconstrained. A FlatBuffer string is *length-prefixed*: it records an explicit byte length and may
+therefore hold any UTF-8 bytes, including an interior NUL (`U+0000`) or other control characters.
+That is strictly more permissive than what a `DType` can portably carry across every serialization
+and interop boundary, so the format constrains field names as follows.
+
+:::{important}
+A struct field name **MUST NOT** contain a NUL byte (`U+0000`) — a hard, structural rule (the Arrow C
+Data Interface cannot represent it; see below). A writer **MUST** enforce this when a `DType` is
+constructed or serialized, and a reader or exporter that encounters a NUL-bearing name **MUST** surface
+a clean error — it **MUST NOT** abort the process. Other control characters **SHOULD** be avoided for
+portability, but are not structurally forbidden — the reference implementation round-trips them today.
+:::
+
+### The Arrow C Data Interface constraint
+
+The binding driving this rule is the [Arrow C Data Interface](https://arrow.apache.org/docs/format/CDataInterface.html),
+which every Vortex language binding uses to hand columnar data to Arrow-based consumers. In that
+interface a field or schema name is a **NUL-terminated `const char*`** — the string ends at the
+first NUL byte and has no separate length. A Vortex FlatBuffer string, by contrast, is
+length-prefixed. The length-prefixed form is the more expressive of the two: it can represent a
+name containing an interior NUL, which the NUL-terminated form cannot. A whole class of otherwise
+well-formed names is therefore *not round-trippable* through the C Data Interface, and Vortex adopts
+the more restrictive rule so that any `DType` a writer produces can be exported without loss.
+
+NUL is the hard case. When a name carrying an interior NUL reaches the C Data export path, the
+conversion into a NUL-terminated C string fails, and — because the failure happens inside a
+non-unwinding FFI callback (the Arrow C stream's schema callback is an `extern "C"` function) — the
+panic cannot unwind across the boundary and instead aborts the whole process with `SIGABRT`. A
+single malformed field name can therefore crash any embedding of a Vortex binding, which is why the
+constraint is enforced when the name is constructed. Other control characters are byte-representable in a
+`const char*` (a non-NUL control byte does not terminate the string), so they do **not** trigger this
+failure — the reference implementation round-trips them today. They remain a portability hazard across
+the polyglot read surface, and Vortex's own display path escapes them, so the format **recommends
+against** them (`SHOULD` avoid) without structurally forbidding them.
+
+### Enforcement and clean failure
+
+Enforcement belongs at the writer: an implementation **MUST** reject an illegal name when a `DType`
+is constructed or serialized, before it can be persisted or sent on the wire, so that no downstream
+reader is ever handed a name it cannot represent. Where a name nevertheless reaches an
+export/conversion boundary — for example converting a `DType` to an Arrow schema or a target
+engine's type — the exporter **MUST** validate the name and return a recoverable error naming the
+offending field, never panic or abort. The DuckDB struct-type exporter is the model here: it maps a
+NUL-bearing name to a `VortexError` rather than unwrapping the conversion failure.
+
+### Metadata-key constraints (out of scope)
+
+The analogous constraints on *metadata keys* (which are subject to the same C-Data and
+interoperability limits) are intentionally not specified here. They will be defined alongside the
+in-flight column-and-file metadata proposal (`vortex-data/rfcs#62`), which introduces the metadata
+mechanism they would govern.
diff --git a/docs/specification/encoding-format.md b/docs/specification/encoding-format.md
new file mode 100644
index 00000000000..8bda783328f
--- /dev/null
+++ b/docs/specification/encoding-format.md
@@ -0,0 +1,398 @@
+# Encoding Format
+
+Every Vortex array node names an **encoding** — the concrete physical layout of its buffers,
+metadata, and child slots. The [Array Format](array-format.md) page describes the container that
+carries a node (the FlatBuffer tree plus the buffer table); this page is the authoritative
+reference for what the bytes inside a node *mean*, encoding by encoding.
+
+This page is written for reader/writer implementers porting Vortex to a new language. Its acceptance
+bar is deliberately strict: each section must be complete and precise enough that an implementer
+working **from this text alone**, with no access to the Rust reference, can decode the encoding
+correctly for every case. The exceptions are the encodings that explicitly wrap an external codec —
+`vortex.pco`, `vortex.zstd`, and `vortex.parquet.variant` — whose component byte streams are defined
+by their named upstream specifications (at the versions Vortex pins) rather than restated here. Where
+a rule has a subtle exception, the exception is stated inline rather than left to be inferred.
+
+:::{note}
+Scope. This page covers the **stable** encodings — those available without the `unstable_encodings`
+build feature. Encodings gated behind `unstable_encodings` (e.g. `vortex.zstd_buffers` and
+`vortex.onpair` — the btrblocks cascade registers as `vortex.onpair`), and any third-party plugin
+encoding, are out of scope: a conformant reader that does not recognise an encoding ID must fail
+loudly with a clear error, never guess a layout or return silently-wrong data.
+:::
+
+```{contents}
+:local:
+:depth: 2
+```
+
+## Validity
+
+**Validity** answers one question for every logical position `i` in an array of length `n`: *is the
+value at `i` present, or is it null?* It is the single most error-prone part of a clean-room reader,
+because validity is not stored in one uniform place — each encoding *sources* it differently, and
+several encodings store it in a transformed or unsliced form that a naive reader will mis-apply.
+Getting this wrong produces silently-corrupted nulls rather than a loud failure, so this section
+specifies the full contract exactly.
+
+Throughout, "**valid**" means the value is present (non-null) and "**invalid**" (or "null") means it
+is absent. The convention is uniform: wherever validity is materialised as booleans, **`true` = valid,
+`false` = null**. A null at position `i` means the logical value there is null regardless of whatever
+bytes the encoding's data buffers happen to hold at that position — the data may be an arbitrary
+placeholder.
+
+### The logical result: four forms
+
+Decoding validity for a node yields one of four logical forms. They are representations of the same
+thing — a per-position valid/null decision — differing only in how compactly the all-valid and
+all-null cases are expressed:
+
+| Form | Meaning |
+|------|---------|
+| `NonNullable` | The type forbids nulls. Every position is valid, by construction. |
+| `AllValid` | The type permits nulls, but every position happens to be valid. |
+| `AllInvalid` | Every position is null. |
+| `Array` | Per-position validity given by a boolean array (see below). |
+
+`NonNullable` and `AllValid` are indistinguishable when *reading a value* (both mean "position `i` is
+present"); they differ only in the type's declared nullability. `AllValid` is exactly equivalent to
+an `Array` of all-`true`; `AllInvalid` is exactly equivalent to an `Array` of all-`false`. An
+implementation may collapse to the compact forms as an optimisation, but it is never *wrong* to
+decode a validity as a full boolean array of length `n` — the compact forms are semantic shorthand,
+not a distinct wire encoding.
+
+#### The `Array` form is itself an encoded array
+
+When validity is an `Array`, that array is a **non-nullable Boolean array of length exactly `n`**,
+where `true` = valid. Critically, it is a *normal Vortex array node* and may use **any encoding** —
+it is commonly `Dict`-, `RunEnd`-, or `Sparse`-encoded rather than a flat bitmap. A reader **must
+decode it recursively** through the same encoding dispatch used for data arrays; it must **not**
+assume a flat, packed bitmap.
+
+:::{important}
+The two invariants a validity `Array` must satisfy, and which a reader should enforce:
+
+1. Its length equals the parent array's length `n`.
+2. Its dtype is **non-nullable** `Bool`.
+
+(A validity array that were itself nullable would be a contradiction — validity has no validity.)
+:::
+
+### Rule 0 — the nullability gate comes first
+
+Before any encoding-specific logic runs, apply the **nullability gate**, which is universal across
+every encoding:
+
+> If the node's `DType` is **non-nullable**, its validity is `NonNullable`. Stop — do not read any
+> validity slot, child, or metadata. The per-encoding rules below apply **only** when the dtype is
+> nullable.
+
+This gate is not per-encoding; it wraps the encoding dispatch. A writer is free to omit a validity
+slot entirely on a non-nullable array, and a nullable array with no stored nulls is likewise valid.
+Consequently every per-encoding rule in this section may assume the dtype is nullable.
+
+### The three sourcing mechanisms
+
+Once past the gate (dtype is nullable), an encoding produces its validity by one of three mechanisms
+(or, for a few encodings, a constant-validity rule — see [Constant-validity encodings](#constant-validity-encodings)).
+Each stable encoding is assigned to exactly one; the [reference table](#validity-reference) below is authoritative.
+
+#### Mechanism 1 — stored slot
+
+The encoding carries validity **directly**, in a dedicated child slot (its "validity slot"). The
+slot is decoded to the logical form as follows — this is the **physical carriage rule**:
+
+| Physical state of the validity slot | Decoded validity |
+|-------------------------------------|------------------|
+| Slot absent (no child present) | `AllValid` |
+| Slot present, a `Constant` bool `true` | `AllValid` |
+| Slot present, a `Constant` bool `false` | `AllInvalid` |
+| Slot present, any other array | `Array` (decode that array recursively) |
+
+In all cases the decoded meaning is the same: *decode the slot to a boolean mask of length `n` and
+read position `i`; an absent slot means all-valid.* The `Constant`-collapse rows are an
+optimisation — a writer that stores an all-true validity as a `Constant(true)` and one that stores
+it as a flat all-`true` `Bool` array are semantically identical; the former merely decodes to
+`AllValid` directly. (The collapse fires **only** for the literal `Constant` encoding carrying a
+boolean scalar. An all-true array in some other encoding is carried as an `Array` and decoded
+normally; the resulting bits are the same.)
+
+##### The offset rule (most common mistake)
+
+The stored validity slot is **row-aligned to the array's logical rows**: slot position `i`
+corresponds to array row `i`, directly. When an array is sliced, its stored validity slot is sliced
+to match. Therefore, for a stored-slot encoding, a reader must **not** apply any array-level offset
+to the validity — doing so shifts every null and corrupts the result after any slice.
+
+Two encodings look like exceptions but are **not** — on the wire a reader applies **no** offset to
+their validity slot either:
+
+- **`vortex.pco` and `vortex.zstd`** hold an in-memory `slice_start .. slice_stop` range, but that
+ range is **runtime-only state for lazy in-place slicing and is not serialized**: it is absent from
+ the on-wire `PcoMetadata`/`ZstdMetadata`, and `deserialize` sets it to `0 .. len`. An on-wire
+ pco/zstd node is therefore **always unsliced** — its validity slot is a normal row-aligned stored
+ slot of length `n`, decoded like any other, with **no** offset applied. (The slice range matters
+ only for in-place slicing of an already-decoded array, never for a file reader.)
+- **`fastlanes.bitpacked`** has an `offset` field (`0 ≤ offset < 1024`). That offset applies to the
+ **packed data buffer only** — it is the start position within the first FastLanes block of 1024
+ values. It is **never** applied to the validity slot, which remains row-aligned like every other
+ stored slot.
+
+:::{warning}
+Never apply `fastlanes.bitpacked`'s `offset` to its stored validity slot: that offset is
+packed-buffer-only, and over-applying it to validity is the single most damaging validity bug — it
+corrupts every null after the slice point. On the wire **every** stored validity slot is row-aligned
+and takes no offset — pco and zstd included, since their in-memory slice range is never serialized.
+:::
+
+#### Mechanism 2 — delegate to a child
+
+The encoding stores no validity of its own; its validity **is** the validity of one specific child
+array. Decode that child's validity (recursively, through this same contract) and use it unchanged.
+
+The exact delegate child differs per encoding and must be matched precisely — delegating to the wrong
+child yields a validity of the wrong length or the wrong values:
+
+| Encoding | Delegates to child |
+|----------|--------------------|
+| `vortex.alp` | `encoded` |
+| `vortex.alprd` | `left_parts` |
+| `fastlanes.for` | `encoded` |
+| `vortex.zigzag` | `encoded` |
+| `vortex.datetimeparts` | `days` |
+| `vortex.decimal_byte_parts` | `msp` (most-significant-parts, child 0) |
+| `vortex.ext` (Extension) | `storage` |
+| `vortex.variant` | `core_storage` |
+| `fastlanes.delta` | `deltas` — **with transform**, see below |
+| `fastlanes.rle` (FastLanes RLE) | `indices` — **offset-aware**, see below |
+
+Most delegates are direct: the child is row-aligned to the parent, so the child's validity is the
+parent's validity verbatim. Two carry an extra transform that a reader must reproduce:
+
+- **`fastlanes.delta`** stores its `deltas` values — and therefore the `deltas` child's validity — in
+ **FastLanes bit-transposed order** (blocks of 1024, transposed for SIMD unpacking). To recover
+ row-order validity: take `deltas`' validity; if it is an `Array`, materialise it to a bit buffer,
+ **untranspose** the bit buffer (inverse of the FastLanes transpose), and wrap it back into a
+ boolean array; the compact forms (`AllValid` / `AllInvalid`) pass through untouched. Then slice
+ the result to `offset .. offset + n`, where `offset` is the Delta array's block offset. Skipping
+ either the untranspose or the offset slice yields scrambled or shifted nulls.
+- **`fastlanes.rle`** (the FastLanes run-length encoding — *distinct from* `vortex.runend`, which is a
+ combine encoding, see below) stores its `indices` child unsliced. Slice `indices` to
+ `offset .. offset + n` first, *then* take that slice's validity.
+
+:::{note}
+`vortex.variant`'s validity delegate is implemented as a direct call to `core_storage`'s validity
+rather than via the shared "from child" helper, but the observable rule is identical: Variant's
+validity is its `core_storage` child's validity.
+:::
+
+#### Mechanism 3 — combine
+
+The encoding synthesises validity by **combining** the validity of its children according to its
+semantics. There are four such encodings; each has an exact rule.
+
+##### `vortex.dict` (Dictionary)
+
+A dictionary has a `codes` child (one code per logical row) and a `values` child (the dictionary
+entries). Row `i` points at dictionary entry `codes[i]`. Row `i` is **valid iff `codes[i]` is itself
+valid AND the dictionary value it points to (`values[codes[i]]`) is valid.** Equivalently, row `i`
+is null iff its code is null, or the code resolves to a null dictionary value.
+
+Concretely, from `codes.validity()` and `values.validity()`:
+
+- both all-valid (`NonNullable`/`AllValid`) → `AllValid`.
+- either is `AllInvalid` → `AllInvalid`.
+- `codes` per-position `Array`, `values` all-valid → the codes' validity array (a null code is a
+ null row; every value is valid).
+- `codes` all-valid, `values` per-position `Array` → gather the values' validity through the codes:
+ a `Dict` array of (`codes`, `values_validity`). Row `i`'s validity is the validity of the value
+ its code selects.
+- both per-position `Array` → gather the values' validity through the codes as above, then set to
+ `false` every position whose code is itself null (`fill_null(false)`), because a null code is a
+ null row irrespective of which value it would have named.
+
+##### `vortex.runend` (Run-End)
+
+A run-end array has an `ends` child (exclusive run boundaries) and a `values` child (one value per
+run), plus an `offset` and length `n`. Its validity is the **run-expansion of the run values'
+validity**: run value `r` covers a contiguous span of rows, and every row in that span shares run
+`r`'s validity.
+
+From `values.validity()`:
+
+- all-valid (`NonNullable`/`AllValid`) → `AllValid`.
+- `AllInvalid` → `AllInvalid`.
+- per-position `Array` → a `RunEnd` array built from the *same* `ends`, the run values' validity as
+ its values, and the *same* `offset` and `n`. The run-expansion is **offset-aware**: `offset` and
+ `n` select the sliced logical window, exactly as for the data. A reader must carry `offset` and
+ `n` through, or validity will not line up with a sliced run-end array.
+
+##### `vortex.sparse` (Sparse)
+
+A sparse array stores a `fill_value` scalar (the value at every non-patched position) and a set of
+patches (`indices` + patch `values`). Its validity combines:
+
+- **default** (all non-patched positions): the fill value's own validity — `fill_value.is_valid()`.
+ If the fill scalar is null, unpatched positions are null; otherwise they are valid.
+- **patched positions**: the patch **values'** validity, at their indices.
+
+Result: a `Sparse` validity array whose fill is `fill_value.is_valid()` and whose patches are the
+patch values' validity placed at the same `indices`. The patch offset metadata (`offset`,
+`offset_within_chunk`, `chunk_offsets`) is preserved unchanged, so the validity is sliced-consistent
+with the data. Row `i` is valid iff: `i` is a patch index and that patch value is valid; or `i` is
+not a patch index and the fill value is valid.
+
+##### `vortex.chunked` (Chunked)
+
+A chunked array concatenates child chunks in order. Its validity is the **ordered concatenation of
+each chunk's validity**, each chunk contributing its own length. Row `i` falls in exactly one chunk;
+its validity is that chunk's validity at the chunk-local position. If every chunk shares the same
+compact form (all `AllValid`, or all `AllInvalid`, or all `NonNullable`), the result collapses to
+that form; otherwise it is an `Array` formed by concatenating the chunks' validities. A chunked array
+with no chunks (`n = 0`) is `AllValid` (its declared nullable dtype with no rows).
+
+### Constant-validity encodings
+
+Three encodings determine validity from their type or a scalar alone, with no child or slot to read:
+
+| Encoding | Validity |
+|----------|----------|
+| `vortex.constant` | `AllInvalid` if the constant scalar is null, else `AllValid`. |
+| `vortex.null` | Always `AllInvalid`. (The `Null` type is null at every position by definition; dtype is `Null`.) |
+| `vortex.sequence` | Always `AllValid`. (A sequence `A[i] = base + i·step` has a value at every position.) |
+
+(For `vortex.constant`, recall Rule 0 has already excluded the non-nullable case; a non-nullable
+constant is `NonNullable`.)
+
+### Nested containers: top-level validity composes with field validity
+
+A container (`vortex.struct`, `vortex.list`, `vortex.listview`, `vortex.fixed_size_list`) carries its
+**own top-level validity** (via a stored slot — Mechanism 1) that is **independent of, and composes
+with, the validity of its fields/elements**:
+
+- The container's `validity()` returns only the **top-level, row-level** validity — whether the whole
+ container row (the struct row, or the list at row `i`) is present.
+- Each field/element child array carries its **own** validity, decoded independently.
+- The two compose by masking: **a null container row makes the entire row null regardless of any
+ field's own validity.** When reading field `f` at row `i`, the logical result is null if the
+ container row `i` is null, *even if* field `f`'s child array reports that position as valid.
+ Field-level validity only distinguishes present-vs-null *within* rows whose container is valid.
+
+A reader must therefore combine the two levels (container-row null OR field null ⇒ null); it must not
+treat a struct's field validity as the row's validity, nor vice versa.
+
+(The row-alignment and length relationships between a container and its field/element children — for
+example that struct fields are row-aligned to the struct while a list's element child is a separate
+flattened buffer indexed by offsets — are specified in the canonical/container layout section.)
+
+### Validity reference
+
+The authoritative per-encoding assignment. "Mechanism" is one of the three above, or a
+constant-validity rule. All rules assume Rule 0 has passed (dtype is nullable); a non-nullable dtype
+is `NonNullable` for every encoding.
+
+| Encoding ID | Mechanism | Source / detail |
+|-------------|-----------|-----------------|
+| `vortex.primitive` | Stored slot | Validity slot, row-aligned. |
+| `vortex.bool` | Stored slot | Validity slot, row-aligned. |
+| `vortex.varbin` | Stored slot | Validity slot, row-aligned. |
+| `vortex.varbinview` | Stored slot | Validity slot, row-aligned. |
+| `vortex.decimal` | Stored slot | Validity slot (child 0), row-aligned. |
+| `vortex.struct` | Stored slot | Top-level validity slot; composes with field validity. |
+| `vortex.list` | Stored slot | Top-level validity slot; composes with element validity. |
+| `vortex.listview` | Stored slot | Top-level validity slot; composes with element validity. |
+| `vortex.fixed_size_list` | Stored slot | Top-level validity slot; composes with element validity. |
+| `vortex.masked` | Stored slot | Validity slot, row-aligned. |
+| `vortex.bytebool` | Stored slot | Validity slot, row-aligned. |
+| `vortex.fsst` | Stored slot | Validity slot, row-aligned. |
+| `vortex.parquet.variant` | Stored slot | Validity slot (child 0), row-aligned. |
+| `fastlanes.bitpacked` | Stored slot | Validity slot, row-aligned. `offset` applies to the packed buffer **only**, never to validity. |
+| `vortex.pco` | Stored slot | Validity slot, row-aligned; on-wire always unsliced, **no** offset (the `slice_start..slice_stop` range is runtime-only, not serialized). |
+| `vortex.zstd` | Stored slot | Validity slot, row-aligned; on-wire always unsliced, **no** offset (the `slice_start..slice_stop` range is runtime-only, not serialized). |
+| `vortex.alp` | Delegate | child `encoded`. |
+| `vortex.alprd` | Delegate | child `left_parts`. |
+| `fastlanes.for` | Delegate | child `encoded`. |
+| `vortex.zigzag` | Delegate | child `encoded`. |
+| `vortex.datetimeparts` | Delegate | child `days`. |
+| `vortex.decimal_byte_parts` | Delegate | child `msp` (child 0). |
+| `vortex.ext` | Delegate | child `storage`. |
+| `vortex.variant` | Delegate | child `core_storage`. |
+| `fastlanes.delta` | Delegate | child `deltas`, then **untranspose** (FastLanes) and slice `offset..offset+n`. |
+| `fastlanes.rle` | Delegate | child `indices`, sliced `offset..offset+n` **before** taking its validity. |
+| `vortex.dict` | Combine | `codes` valid AND `values[codes]` valid. |
+| `vortex.runend` | Combine | run-expand `values`' validity (offset- and length-aware). |
+| `vortex.sparse` | Combine | `fill_value.is_valid()` default, patched positions take patch values' validity. |
+| `vortex.chunked` | Combine | ordered concatenation of each chunk's validity. |
+| `vortex.constant` | Constant rule | `AllInvalid` if scalar null, else `AllValid`. |
+| `vortex.null` | Constant rule | Always `AllInvalid`. |
+| `vortex.sequence` | Constant rule | Always `AllValid`. |
+
+### Decoding procedure
+
+The full validity-decode of a node, expressed as a recursive procedure returning a boolean decision
+for each of the `n` positions (or one of the compact forms):
+
+```text
+decode_validity(node) -> validity of length n:
+ # Rule 0 — nullability gate (universal, first).
+ if not node.dtype.is_nullable():
+ return NonNullable # stop; read nothing further
+
+ match node.encoding: # arms key on the encoding ID from the Validity reference table
+
+ # --- Mechanism 1: stored slot ---
+ stored-slot encoding:
+ slot = node.validity_slot # a child array, or absent
+ if slot is absent:
+ v = AllValid
+ else:
+ v = decode_array(slot) # recursive; non-nullable Bool, length n
+ # No offset is ever applied to a stored validity slot. (bitpacked's `offset` is
+ # packed-buffer-only; pco/zstd track an in-memory slice range that is NOT serialized,
+ # so an on-wire node is always unsliced and its validity slot is already length n.)
+ return v
+
+ # --- Mechanism 2: delegate to child ---
+ delegate encoding:
+ child = node.child(delegate_name) # per reference table
+ v = decode_validity(child)
+ if node.encoding == delta:
+ v = untranspose(v); v = v.slice(node.offset .. node.offset + n)
+ if node.encoding == rle:
+ v = decode_validity(child.slice(node.offset .. node.offset + n))
+ return v
+
+ # --- Mechanism 3: combine ---
+ dict: return combine_dict(decode_validity(codes), decode_validity(values), codes)
+ runend: return runexpand(decode_validity(values), ends, node.offset, n)
+ sparse: return sparse_validity(fill_value.is_valid(), decode_validity(patch_values),
+ indices, node.patch_offsets) # offset metadata preserved
+ chunked: return concat(decode_validity(chunk) for chunk in chunks)
+
+ # --- constant-validity encodings ---
+ constant: return AllInvalid if node.scalar.is_null() else AllValid
+ null: return AllInvalid
+ sequence: return AllValid
+
+ # --- unrecognised (experimental / third-party) ---
+ _: error("unknown encoding ") # never guess; fail loudly
+```
+
+Where `decode_array` decodes an arbitrary Vortex array node to values (here, a non-nullable boolean
+mask), through the same encoding dispatch — validity `Array`s are ordinary encoded arrays.
+
+## Per-encoding byte layouts
+
+The per-encoding byte-layout reference — buffer table, metadata fields, and child slots for each
+stable encoding — is split into family pages, one per encoding family:
+
+```{toctree}
+:maxdepth: 1
+
+encoding-format/canonical
+encoding-format/containers
+encoding-format/fastlanes
+encoding-format/alp
+encoding-format/dict-runend-sparse
+encoding-format/misc
+```
diff --git a/docs/specification/encoding-format/alp.md b/docs/specification/encoding-format/alp.md
new file mode 100644
index 00000000000..e1ac90d39a4
--- /dev/null
+++ b/docs/specification/encoding-format/alp.md
@@ -0,0 +1,198 @@
+# ALP Encodings — Byte Layout
+
+The **ALP** family is the floating-point-compression group: Adaptive Lossless floating-Point (ALP)
+and its real-double variant (ALPRD). This page is the per-encoding byte-layout reference for that
+family, one section per encoding.
+
+It is part of the [Encoding Format](../encoding-format.md) specification; null handling for every
+encoding here follows the cross-cutting [Validity](../encoding-format.md#validity) contract on that
+page and is not restated per section.
+
+Encodings covered on this page: `ALP`, `ALPRD`.
+
+(encoding-layout-ALP)=
+## `vortex.alp` — Byte layout
+
+ALP (Adaptive Lossless floating-Point) stores an `f32`/`f64` column as scaled integers. Each value
+`x` is represented by an integer `i` chosen so that `x = i × 10^(f − e)`, for two small per-array
+exponents `e` and `f`. Values that do not round-trip through that scaling — the **exceptions** — are
+kept verbatim as floating-point *patches*. The scaled integers live in the `encoded` child; the
+exceptions live in patch children.
+
+The logical dtype is `f32` (with `i32` scaled integers) or `f64` (with `i64` scaled integers); the
+integer width is fixed by the float width.
+
+**Wire ID:** `vortex.alp`.
+
+**Buffers:** none. `vortex.alp` owns no buffers (`nbuffers = 0`); everything is carried in metadata
+and children (`array.rs` (`ALP::nbuffers`)).
+
+### Metadata (`ALPMetadata`, Protobuf)
+
+| Field | Protobuf type / tag | Meaning |
+|-------|---------------------|---------|
+| `exp_e` | `uint32`, tag 1 | The exponent `e`. Bounded to `0 ≤ e ≤ 10` for `f32` and `0 ≤ e ≤ 18` for `f64`; always fits a `u8`. |
+| `exp_f` | `uint32`, tag 2 | The factor `f`, same bounds as `e`. |
+| `patches` | `PatchesMetadata`, tag 3, optional | Present iff the array has exceptions. Describes the patch children — see [Patch metadata and children](#alp-patch-metadata). |
+
+The bounds and their per-type values come from `ALPFloat::MAX_EXPONENT` (`10` for `f32`, `18` for
+`f64`) checked in `array.rs` (`ALPData::validate_components`); the fields are stored as `u32` and narrowed to `u8` in
+`array.rs` (`ALP::deserialize`).
+
+### Child slots
+
+Children are positional (`array.rs` (`ALP::deserialize`)):
+
+| Idx | Name | Present when | Logical dtype |
+|-----|------|--------------|---------------|
+| 0 | `encoded` | always | `i32` (for an `f32` array) / `i64` (for an `f64` array), **same nullability** as the ALP array. |
+| 1 | `patch_indices` | `patches` set | non-nullable unsigned int (ptype from `PatchesMetadata`). |
+| 2 | `patch_values` | `patches` set | the ALP array's float dtype (`f32`/`f64`). |
+| 3 | `patch_chunk_offsets` | `patches` set **and** metadata `chunk_offsets_ptype` present | non-nullable unsigned int (acceleration index). |
+
+**Child lengths.** `encoded` has length `n` (row-aligned to the ALP array); the patch children
+`patch_indices` and `patch_values` have length `P` (the patch count), and `patch_chunk_offsets` has
+length `chunk_offsets_len` — all per the shared [Patch metadata](#alp-patch-metadata). (The ALP-RD
+children follow the same convention: `left_parts` / `right_parts` are length `n`, patch children
+length `P`.)
+
+`encoded` is decoded recursively like any node (it is typically itself `fastlanes`-packed). At an
+exception position the integer stored in `encoded` is a **meaningless placeholder** — the "fill
+value", the first non-exception encoded integer, substituted for compressibility
+(`mod.rs` (`encode_chunk_unchecked`)). The true value is supplied by the patch, so a reader **must** apply patches and
+must never trust `encoded` at a patched position.
+
+### Decode
+
+Given `encoded` decoded to an `i32`/`i64` slice, the exponents `e`, `f`, and the optional patches:
+
+1. **Integer → float.** For each position `k`, compute in the target float type as a **two-step
+ multiply by the tabulated constants** — this exact sequence is normative:
+ ```text
+ value[k] = (float) encoded[k] × F10[f] × IF10[e]
+ ```
+ where `F10[f] = 10^f` and `IF10[e] = 10^(−e)` are the per-float-type exact table entries; the
+ multiply order matches `decode_single` in `mod.rs` (`ALPFloat::decode_single`). Example: `e = 2`, `f = 0`,
+ `encoded[k] = 1234` → `1234 × 1 × 10^(−2) = 12.34`.
+
+ :::{warning}
+ Do **not** collapse this to a single `powi(10, f − e)`. ALP is lossless: the encoder only omits
+ (rather than patches) values that round-trip through this *exact two-step* sequence in the target
+ float type. A single combined power double-rounds and can differ by 1 ULP on non-exception
+ values — a silent lossy divergence. Use the two tabulated constants, in this order.
+ :::
+2. **Apply patches** (if `patches` present). For each patch `j` in `0 .. patches.len`, overwrite
+ ```text
+ value[ indices[j] − offset ] = patch_values[j]
+ ```
+ Patch values are already floats and replace the decoded placeholder outright (`decompress.rs` (`decompress_unchunked_core`);
+ the `index − offset` write is `patch.rs` (`PrimitiveArray::patch_typed`)).
+3. **Validity** is delegated (below).
+
+(alp-patch-metadata)=
+### Patch metadata and children
+
+`PatchesMetadata` (Protobuf, `patches.rs` (`PatchesMetadata`)) describes the patch children. This layout is shared
+verbatim by `vortex.alprd`.
+
+| Field | Protobuf type / tag | Meaning |
+|-------|---------------------|---------|
+| `len` | `uint64`, tag 1 | Number of patches `P` (= length of both the `patch_indices` and `patch_values` children). |
+| `offset` | `uint64`, tag 2 | Absolute offset subtracted from every stored index; `0` for an unsliced array. Array-local position = `index − offset`. |
+| `indices_ptype` | `PType` enum, tag 3 | Unsigned-int ptype of the `patch_indices` child. |
+| `chunk_offsets_len` | `uint64`, tag 4, optional | Length of the `patch_chunk_offsets` child, when present. |
+| `chunk_offsets_ptype` | `PType` enum, tag 5, optional | Unsigned-int ptype of `patch_chunk_offsets`; its presence signals that child 3 exists. |
+| `offset_within_chunk` | `uint64`, tag 6, optional | Rows sliced off the front of the first chunk (slice bookkeeping). |
+
+`patch_indices` is a strictly-ascending non-nullable unsigned-int array of length `P`; each entry is
+the absolute logical position of an exception. `patch_values` (length `P`, non-null) holds the value
+for that position. The pair `(indices[j], values[j])` reads as: *logical row `indices[j] − offset`
+takes value `values[j]`.* `patch_chunk_offsets` is an **optional O(1) lookup index** — one entry per
+1024-row chunk (`PATCH_CHUNK_SIZE = 1024`, `patches.rs` (`PATCH_CHUNK_SIZE`)) pointing into the patch arrays; a decoder
+that scans all `P` patches linearly does not need it and may ignore it.
+
+### Validity
+
+`vortex.alp` sources validity by **delegation to its `encoded` child**: decode `encoded`'s validity
+recursively and use it unchanged (`array.rs` (`ALP::ValidityVTable`), `array.rs` (`ALP::validity_child`)). The full contract — including
+the universal nullability gate — is specified once in [Validity](../encoding-format.md#validity); it
+is not restated here.
+
+(encoding-layout-ALPRD)=
+## `vortex.alprd` — Byte layout
+
+ALP-RD ("real doubles") targets floats that do not scale to integers cleanly. It splits each value's
+IEEE-754 bit pattern into a low part of `right_bit_width` bits (`right_parts`) and a high part of at
+most 16 bits (the "left" bits). Because the high parts repeat heavily across a vector, they are
+**dictionary-encoded**: the `left_parts` child stores a small dictionary *code* per row, and the
+metadata `dict` maps each code back to its 16-bit pattern. High parts not in the dictionary are held
+as **exceptions** in left-parts patches. Recombining a left pattern with its right part reproduces
+the original IEEE bits exactly (lossless).
+
+The logical dtype is `f32` or `f64`.
+
+**Wire ID:** `vortex.alprd`.
+
+**Buffers:** none (`nbuffers = 0`, `array.rs` (`ALPRD::nbuffers`)); all data is in metadata and children.
+
+### Metadata (`ALPRDMetadata`, Protobuf)
+
+| Field | Protobuf type / tag | Meaning |
+|-------|---------------------|---------|
+| `right_bit_width` | `uint32`, tag 1 | Number of low bits `R` per value; fits a `u8` (`array.rs` (`ALPRD::deserialize`)). The left part occupies the remaining `BITS − R` bits (`≤ 16`; `BITS` is 32 for `f32`, 64 for `f64`). |
+| `dict_len` | `uint32`, tag 2 | Number of valid dictionary entries `D` (`D ≤ 8`, `MAX_DICT_SIZE`, `mod.rs` (`MAX_DICT_SIZE`)). |
+| `dict` | repeated `uint32`, tag 3 | The dictionary. Entry `dict[c]` is the 16-bit left bit-pattern for code `c`. Only the first `dict_len` entries are used; each fits a `u16` (`array.rs` (`ALPRD::deserialize`)). |
+| `left_parts_ptype` | `PType` enum, tag 4 | Unsigned-int ptype of the `left_parts` child (`u16` in the reference encoder). |
+| `patches` | `PatchesMetadata`, tag 5, optional | Left-parts exceptions. Layout is exactly the [Patch metadata](#alp-patch-metadata) above, with the type note below. |
+
+### Child slots
+
+Children are positional (`array.rs` (`SLOT_NAMES`)):
+
+| Idx | Name | Present when | Logical dtype |
+|-----|------|--------------|---------------|
+| 0 | `left_parts` | always | unsigned int (ptype = `left_parts_ptype`), **same nullability** as the ALP-RD array. One dictionary code per row, `0 ≤ code < dict_len`. |
+| 1 | `right_parts` | always | `u32` (for `f32`) / `u64` (for `f64`), **non-nullable**. The low `R` bits of each value. |
+| 2 | `patch_indices` | `patches` set | non-nullable unsigned int; see [Patch metadata](#alp-patch-metadata). |
+| 3 | `patch_values` | `patches` set | unsigned int (`left_parts_ptype`, non-nullable). The **true 16-bit left patterns** of the exceptions — *not* codes (`array.rs` (`ALPRD::deserialize`)). |
+
+`left_parts` and `right_parts` are decoded recursively (both are typically `fastlanes`-bit-packed).
+At an exception position, `left_parts` stores code `0` as a placeholder; the real left pattern comes
+from the patch. ALP-RD patches never carry chunk offsets, and a **freshly encoded** array's patch
+`offset` is `0` (`mod.rs` (`RDEncoder::encode_generic`)). Slicing, however, updates the patch
+`offset` and re-serializes it (`array.rs` (`ALPRD::serialize`)), so a **sliced** ALP-RD array
+persists a non-zero `offset`; a reader must always apply the `indices[j] − offset` rule below and
+never assume `offset == 0`.
+
+### Decode
+
+Given `left_parts` codes, `right_parts`, the `dict`, `R = right_bit_width`, and optional patches, for
+each row `k`:
+
+1. **Dictionary-decode** the left bits: `L[k] = dict[ left_parts[k] ]` (`mod.rs` (`alp_rd_decode`)).
+2. **Apply patches** (if present). For each patch `j`, overwrite the left bits with the exception's
+ true pattern:
+ ```text
+ L[ indices[j] − offset ] = patch_values[j]
+ ```
+ This runs *after* the dictionary decode, so it replaces the placeholder `dict[0]` at exception
+ positions (`mod.rs` (`alp_rd_decode`), `mod.rs` (`alp_rd_apply_patches`)).
+3. **Recombine** into the value's unsigned integer type (`u32` for `f32`, `u64` for `f64`):
+ ```text
+ bits[k] = ( (UINT) L[k] << R ) | right_parts[k]
+ ```
+ (`mod.rs` (`alp_rd_decode`); equivalently `ops.rs` (`ALPRD::scalar_at`)).
+4. **Reinterpret** `bits[k]` as the IEEE-754 float via a bit-cast (`u32 → f32`, `u64 → f64`;
+ `mod.rs` (`alp_rd_combine_inplace`)).
+5. **Validity** is delegated (below).
+
+Worked example (`f64`, `R = 52`, so the left part is the top 12 bits): if `dict = [0x3FF]` and row `k`
+has `left_parts[k] = 0` (selecting `dict[0] = 0x3FF`) and `right_parts[k] = 0x0000000000000000`, then
+`L[k] = 0x3FF`, `bits[k] = 0x3FF << 52 = 0x3FF0000000000000`, which bit-casts to `1.0`. (`R = 52` keeps
+the left part `BITS − R = 12` bits, within the ≤16-bit dictionary-code width.)
+
+### Validity
+
+`vortex.alprd` sources validity by **delegation to its `left_parts` child** (`array.rs` (`ALPRD::ValidityVTable`),
+`array.rs` (`ALPRD::validity_child`)): decode `left_parts`' validity recursively and use it unchanged. `right_parts`
+is non-nullable and contributes no validity. See [Validity](../encoding-format.md#validity).
diff --git a/docs/specification/encoding-format/canonical.md b/docs/specification/encoding-format/canonical.md
new file mode 100644
index 00000000000..32d904e9fee
--- /dev/null
+++ b/docs/specification/encoding-format/canonical.md
@@ -0,0 +1,359 @@
+# Canonical Encodings — Byte Layout
+
+The **canonical** encodings are the base, largely uncompressed forms — the Arrow-style flat layouts
+that logical values canonicalise to, plus the trivial constant and generative encodings. This page
+is the per-encoding byte-layout reference for that family, one section per encoding.
+
+It is part of the [Encoding Format](../encoding-format.md) specification; null handling for every
+encoding here follows the cross-cutting [Validity](../encoding-format.md#validity) contract on that
+page and is not restated per section.
+
+Encodings covered on this page: `Primitive`, `Bool`, `VarBin`, `VarBinView`, `Decimal`, `Null`,
+`Constant`, `Sequence`.
+
+```{contents}
+:local:
+:depth: 1
+```
+
+## Conventions used on this page
+
+Each section below decodes one encoding from its **serialized components**: the array-specific
+**metadata** bytes, the **data buffers** it references (by position in the node's buffer table), and
+its **child** array nodes. The [Array Format](../array-format.md) page describes the container that
+carries these; this page defines what each encoding's components *mean*. Terms used uniformly:
+
+- **`n`** is the node's logical length. Vortex arrays do **not** store their own length or dtype —
+ both are supplied by the parent (ultimately the file/IPC schema), so `n` and the `DType` are
+ inputs to every decode here, never parsed from the node. See [Array Format](../array-format.md)
+ and [DType Format](../dtype-format.md).
+- **Metadata** is a Protobuf message (the reference implementation uses `prost`), decoded by field
+ tag. An encoding whose metadata is *empty* serializes zero metadata bytes; its section says so.
+ Beyond that, a metadata blob may be **zero bytes even for an encoding that defines metadata
+ fields** — whenever every field holds its proto3 default, `prost` omits it, so nothing is written.
+ For example a `Bool` with `offset = 0`, or a `Decimal` with `values_type = I8` (the `= 0` enum
+ value), serialises no metadata bytes. A reader must therefore decode absent or empty metadata to
+ the proto3 defaults for every field and must **not** error on "expected metadata".
+ Integer/scalar values inside buffers and metadata are **little-endian** throughout.
+- **Buffers** are referenced positionally: "buffer 0", "buffer 1", … index into this node's slice of
+ the file's buffer table, in the order each section lists.
+- **Child slots.** Each encoding has a fixed, named, ordered list of *slots*; a slot may be
+ mandatory or optional. On the wire the children are a **dense list of the present slots, in slot
+ order** — an absent optional slot is simply omitted, not written as a placeholder. A reader
+ therefore reconstructs slot identity positionally from how many children are present (each section
+ states the exact mapping). The most common optional slot is `validity`, present when validity is
+ materialised as a **stored child** — an `Array`, or the `Constant(false)` child a writer emits for
+ `AllInvalid` — and absent only for `NonNullable`/`AllValid` (see below).
+- **Validity** (null handling) is **not** restated per encoding: it follows the cross-cutting
+ [Validity](../encoding-format.md#validity) contract, and each section names which mechanism the
+ encoding uses and links the [Validity reference](../encoding-format.md#validity-reference) table
+ row. Recall Rule 0: if the node's `DType` is non-nullable, validity is `NonNullable` and no
+ validity slot is read at all.
+
+(encoding-layout-Primitive)=
+
+## Primitive — `vortex.primitive`
+
+A flat array of fixed-width numeric values, one per logical row. Mirrors the Arrow primitive layout.
+
+**Wire encoding ID:** `vortex.primitive` (`vortex-array/src/arrays/primitive/vtable/mod.rs` `Primitive::id`).
+
+**Buffers** (`primitive/vtable/mod.rs` `Primitive::nbuffers`, `Primitive::buffer_name`; buffer length checked in `Primitive::deserialize`):
+
+| # | Name | Contents | Length |
+|---|------|----------|--------|
+| 0 | `values` | `n` values laid out contiguously, each `w` bytes, little-endian (two's-complement for integers, IEEE-754 for floats). | exactly `w · n` bytes |
+
+The element width `w` and interpretation come from the node's `DType::Primitive(ptype, …)`; the
+`ptype` is one of the 11 `PType`s (`U8`/`U16`/`U32`/`U64`/`I8`/`I16`/`I32`/`I64`/`F16`/`F32`/`F64`),
+whose byte widths are 1/2/4/8 as per the width in the name (`F16` = 2 bytes)
+(`vortex-array/src/dtype/ptype.rs` `PType`, `PType::byte_width`). See the `PType` enum in
+[DType Format](../dtype-format.md#flatbuffer-definition). The buffer is aligned to `w`
+(`primitive/vtable/mod.rs` `Primitive::deserialize`). `F16` holds
+the raw IEEE-754 half-precision bits.
+
+**Metadata:** none (zero bytes). The `ptype` is taken from the `DType`, not the metadata
+(`primitive/vtable/mod.rs` `Primitive::deserialize`).
+
+**Child slots** (`primitive/vtable/mod.rs` `Primitive::deserialize`; slot names `primitive/array/mod.rs` `SLOT_NAMES`):
+
+| slot | name | role | presence |
+|------|------|------|----------|
+| 0 | `validity` | validity mask | optional — present when validity is a stored child (an `Array`, or `Constant(false)` for `AllInvalid`) |
+
+So the child list is empty (validity sourced from the dtype's nullability) or holds exactly the
+validity array as child 0.
+
+**Decode:** for each `i` in `0..n`, the value occupies `values[i·w .. (i+1)·w]`, decoded as the
+`ptype`. A row that validity marks null carries an arbitrary placeholder in these bytes — read the
+value only after consulting validity.
+
+**Validity:** stored slot (Mechanism 1), row-aligned; no offset is applied. See the
+[Validity reference](../encoding-format.md#validity-reference) row for `vortex.primitive`.
+
+(encoding-layout-Bool)=
+
+## Bool — `vortex.bool`
+
+A bit-packed boolean array — one bit per logical row. Mirrors the Arrow boolean layout, and is also
+the flat form a `validity` `Array` takes.
+
+**Wire encoding ID:** `vortex.bool` (`vortex-array/src/arrays/bool/vtable/mod.rs` `Bool::id`).
+
+**Buffers** (`bool/vtable/mod.rs` `Bool::nbuffers`, `Bool::buffer_name`; length bound checked in `Bool::validate`):
+
+| # | Name | Contents | Length |
+|---|------|----------|--------|
+| 0 | `bits` | packed bits, **LSB-first within each byte** (`0` = false, `1` = true) | at least `⌈(offset + n) / 8⌉` bytes |
+
+**Metadata:** Protobuf `BoolMetadata` (`bool/vtable/mod.rs` `BoolMetadata`; the `0 ≤ offset < 8` bound is asserted in `Bool::serialize`):
+
+| tag | field | type | meaning |
+|-----|-------|------|---------|
+| 1 | `offset` | `uint32` | starting bit offset within the first byte of `bits`; always `0 ≤ offset < 8` |
+
+**Child slots:** identical shape to Primitive — slot 0 `validity`, optional (present when validity
+is a stored child: an `Array`, or `Constant(false)` for `AllInvalid`) (`bool/vtable/mod.rs`
+`Bool::deserialize`; slot names `bool/array.rs` `SLOT_NAMES`).
+
+**Decode:** the value at logical row `i` is the bit at absolute bit index `offset + i`: byte
+`(offset + i) / 8`, bit `(offset + i) % 8` counting the least-significant bit as bit 0. That is,
+`value(i) = (bits[(offset+i) >> 3] >> ((offset+i) & 7)) & 1 == 1`. The `offset` (a sub-byte slice
+start, `< 8`; whole leading bytes are dropped at write time) applies to `bits` **only**
+(`bool/array.rs` `BoolData::to_bit_buffer`). Bits at
+null positions are undefined.
+
+**Validity:** stored slot (Mechanism 1), row-aligned. See the
+[Validity reference](../encoding-format.md#validity-reference) row for `vortex.bool`.
+
+(encoding-layout-VarBin)=
+
+## VarBin — `vortex.varbin`
+
+Variable-length binary/UTF-8 values stored as one concatenated byte buffer plus an offsets array.
+Mirrors the Arrow variable-binary layout (offsets + data), except the offsets are a **child array**,
+not a buffer.
+
+**Wire encoding ID:** `vortex.varbin` (`vortex-array/src/arrays/varbin/vtable/mod.rs` `VarBin::id`).
+
+**Buffers** (`varbin/vtable/mod.rs` `VarBin::nbuffers`, `VarBin::buffer_name`; `offsets[n] ≤ len(bytes)` checked in `varbin/array.rs` `VarBinData::validate`):
+
+| # | Name | Contents | Length |
+|---|------|----------|--------|
+| 0 | `bytes` | all element bytes concatenated in row order | `≥ offsets[n]` (leading bytes before `offsets[0]` and trailing bytes after `offsets[n]` both permitted) |
+
+**Metadata:** Protobuf `VarBinMetadata` (`varbin/vtable/mod.rs` `VarBinMetadata`):
+
+| tag | field | type | meaning |
+|-----|-------|------|---------|
+| 1 | `offsets_ptype` | `PType` enum | integer `ptype` of the `offsets` child (e.g. `I32`, `I64`) |
+
+The `PType` enum values are `U8=0, U16=1, U32=2, U64=3, I8=4, I16=5, I32=6, I64=7, F16=8, F32=9,
+F64=10` (`vortex-array/src/dtype/ptype.rs` `PType`; see [DType Format](../dtype-format.md#flatbuffer-definition)).
+
+**Child slots** (`varbin/array.rs` `OFFSETS_SLOT`, `VALIDITY_SLOT`):
+
+| slot | name | role | presence |
+|------|------|------|----------|
+| 0 | `offsets` | element boundary offsets | **mandatory** |
+| 1 | `validity` | validity mask | optional — present when validity is a stored child (an `Array`, or `Constant(false)` for `AllInvalid`) |
+
+`offsets` is itself a Vortex array node of dtype `Primitive(offsets_ptype, NonNullable)` and length
+`n + 1`; decode it recursively (it may use any encoding). The child list is therefore `[offsets]`
+(no stored validity) or `[offsets, validity]` (`varbin/vtable/mod.rs` `VarBin::deserialize`).
+
+**Decode:** decode `offsets` to `n + 1` integers `O[0..=n]`. They are monotonically non-decreasing
+and satisfy `O[n] ≤ len(bytes)` (`varbin/array.rs` `VarBinData::validate`). `O[0]` is **not necessarily 0**: slicing a VarBin does not rebase
+the offsets and does not trim the `bytes` buffer (`varbin/compute/slice.rs` `VarBin::_slice`), so after a slice `O[0]` may be `> 0` and `bytes`
+may carry bytes both before `O[0]` and after `O[n]`. Element `i` is the byte range
+`bytes[O[i] .. O[i+1]]` (`varbin/array.rs` `VarBinArrayExt::bytes_at`) — decode strictly this way; do not assume element 0 starts at byte 0, and do
+not normalise or trim the buffer. If the node's `DType` is `Utf8`, those bytes are a UTF-8 string; if
+`Binary`, raw bytes. Null rows still occupy an offsets slot (typically an empty range).
+
+**Validity:** stored slot (Mechanism 1, slot 1), row-aligned. See the
+[Validity reference](../encoding-format.md#validity-reference) row for `vortex.varbin`.
+
+(encoding-layout-VarBinView)=
+
+## VarBinView — `vortex.varbinview`
+
+Variable-length binary/UTF-8 stored as fixed-width 16-byte **views** that either inline short values
+or reference a data buffer. Mirrors the Arrow `StringView`/`BinaryView` layout.
+
+**Wire encoding ID:** `vortex.varbinview` (`vortex-array/src/arrays/varbinview/vtable/mod.rs` `VarBinView::id`).
+
+**Buffers:** a variable count, `k + 1` total, where the **last** buffer is the views and the
+preceding `k ≥ 0` buffers hold spilled long-value bytes (`varbinview/vtable/mod.rs`
+`VarBinView::nbuffers`, `VarBinView::buffer_name`):
+
+| # | Name | Contents | Length |
+|---|------|----------|--------|
+| `0 .. k-1` | `buffer_0` … `buffer_{k-1}` | data blocks holding the bytes of values longer than 12 bytes | arbitrary |
+| `k` (last) | `views` | `n` fixed 16-byte view entries | exactly `16 · n` bytes |
+
+A decoder splits the buffer list as *(all-but-last = data buffers, last = views)*
+(`varbinview/vtable/mod.rs` `VarBinView::deserialize`; the views buffer is validated to be exactly
+`16 · n` bytes, and each view entry is 16 bytes, `varbinview/view.rs` `BinaryView`).
+
+**Metadata:** none (zero bytes) (`varbinview/vtable/mod.rs` `VarBinView::serialize`).
+
+**Child slots:** slot 0 `validity`, optional (present when validity is a stored child: an `Array`,
+or `Constant(false)` for `AllInvalid`) (`varbinview/vtable/mod.rs` `VarBinView::deserialize`; slot
+names `varbinview/array.rs` `SLOT_NAMES`).
+
+**Decode:** the `views` buffer is `n` entries of 16 bytes each (`BinaryView`). For entry `i`, read
+the first 4 bytes as `size: u32` (little-endian), the total value length. Then:
+
+- **Inlined** (`size ≤ 12`): the value is bytes `4 .. 4+size` of the entry; bytes `4+size .. 16` are
+ zero padding (`varbinview/view.rs` `Inlined`, `BinaryView::MAX_INLINED_SIZE`).
+- **Reference** (`size > 12`): the remaining 12 bytes are three little-endian `u32`s —
+ `prefix` occupies bytes `4..8` (the first 4 bytes of the value, a comparison shortcut),
+ `buffer_index` bytes `8..12`, and `offset` bytes `12..16`. The value is
+ `data_buffer[buffer_index][offset .. offset + size]` (`varbinview/view.rs` `Ref`).
+
+| entry bytes | inlined (`size ≤ 12`) | reference (`size > 12`) |
+|-------------|-----------------------|--------------------------|
+| `0..4` | `size` (`u32` LE) | `size` (`u32` LE) |
+| `4..8` | value bytes `0..4` | `prefix` = value bytes `0..4` |
+| `8..12` | value bytes `4..8` | `buffer_index` (`u32` LE) |
+| `12..16` | value bytes `8..12` | `offset` (`u32` LE) |
+
+`Utf8` dtype → the assembled bytes are a UTF-8 string; `Binary` → raw bytes.
+
+**Validity:** stored slot (Mechanism 1), row-aligned. See the
+[Validity reference](../encoding-format.md#validity-reference) row for `vortex.varbinview`.
+
+(encoding-layout-Decimal)=
+
+## Decimal — `vortex.decimal`
+
+Fixed-precision decimals stored as scaled integers of a chosen backing width.
+
+**Wire encoding ID:** `vortex.decimal` (`vortex-array/src/arrays/decimal/vtable/mod.rs` `Decimal::id`).
+
+**Buffers** (`decimal/vtable/mod.rs` `Decimal::nbuffers`, `Decimal::buffer_name`):
+
+| # | Name | Contents | Length |
+|---|------|----------|--------|
+| 0 | `values` | `n` unscaled integers, each `w` bytes, little-endian two's-complement | exactly `w · n` bytes |
+
+**Metadata:** Protobuf `DecimalMetadata` (`decimal/vtable/mod.rs` `DecimalMetadata`):
+
+| tag | field | type | meaning |
+|-----|-------|------|---------|
+| 1 | `values_type` | `DecimalType` enum | physical integer width of `values` |
+
+`DecimalType` values and their widths `w`: `I8 = 0` (1 byte), `I16 = 1` (2), `I32 = 2` (4),
+`I64 = 3` (8), `I128 = 4` (16), `I256 = 5` (32) (`vortex-array/src/dtype/decimal/types.rs`
+`DecimalType`, `DecimalType::byte_width`). `I256` is a 32-byte little-endian two's-complement
+integer. The buffer is aligned to `w` for `I8`–`I128` (i.e. 1/2/4/8/16), but **`I256` is aligned to
+16, not 32**: Vortex's `i256` is `#[repr(transparent)]` over `arrow_buffer::i256 { low: u128, high:
+i128 }` (`vortex-array/src/dtype/bigint/mod.rs` `i256`), whose alignment is 16, and the reader
+enforces exactly that (`Alignment::of::()` = 16) (`decimal/vtable/mod.rs` `Decimal::deserialize`). A
+reader that demands 32-byte alignment for `I256` would reject valid buffers.
+
+The logical **precision** (`u8`) and **scale** (`i8`) come from the node's `DType::Decimal`, *not*
+from the metadata (`decimal/array.rs` `DecimalArrayExt::decimal_dtype`). `values_type` is only the storage width and need not be the smallest that fits
+the precision — a writer may store precision-38 values in an `I8` buffer if they all fit
+(`decimal/array.rs` `DecimalData`).
+
+**Child slots:** slot 0 `validity`, optional (present when validity is a stored child: an `Array`,
+or `Constant(false)` for `AllInvalid`) (`decimal/vtable/mod.rs` `Decimal::deserialize`; slot names
+`decimal/array.rs` `SLOT_NAMES`).
+
+**Decode:** reinterpret `values` as `n` integers of width `w`. Row `i`'s logical value is
+`unscaled[i] / 10^scale` (i.e. the integer with the decimal point moved left `scale` places; a
+negative `scale` moves it right) (`decimal/array.rs` `DecimalData`). Null rows carry an arbitrary placeholder integer.
+
+**Validity:** stored slot (Mechanism 1, child 0), row-aligned. See the
+[Validity reference](../encoding-format.md#validity-reference) row for `vortex.decimal`.
+
+(encoding-layout-Null)=
+
+## Null — `vortex.null`
+
+An all-null column of the `Null` type. Carries no data — only its length.
+
+**Wire encoding ID:** `vortex.null` (`vortex-array/src/arrays/null/mod.rs` `Null::id`).
+
+**Buffers:** none (`null/mod.rs` `Null::nbuffers`).
+
+**Metadata:** none (zero bytes) (`null/mod.rs` `Null::serialize`).
+
+**Child slots:** none.
+
+**Decode:** the node's `DType` **must** be `Null` (`null/mod.rs` `Null::validate`). The array is `n` null values; there is nothing
+else to read. Every logical position is null.
+
+**Validity:** constant-validity rule — always `AllInvalid` (`null/mod.rs` `Null::validity`). See
+[Constant-validity encodings](../encoding-format.md#constant-validity-encodings).
+
+(encoding-layout-Constant)=
+
+## Constant — `vortex.constant`
+
+`n` repetitions of a single scalar value.
+
+**Wire encoding ID:** `vortex.constant` (`vortex-array/src/arrays/constant/vtable/mod.rs` `Constant::id`).
+
+**Buffers** (`constant/vtable/mod.rs` `Constant::nbuffers`, `Constant::buffer_name`):
+
+| # | Name | Contents | Length |
+|---|------|----------|--------|
+| 0 | `scalar` | the Protobuf encoding of one `ScalarValue` (the value only; no dtype) | variable |
+
+The buffer holds a serialized `ScalarValue` — the same value-only Protobuf message described in
+[Scalar Format](../scalar-format.md) (a `null_value`, `bool_value`, primitive, decimal, string,
+list, etc.) (`vortex-proto/proto/scalar.proto` `ScalarValue`). It is deliberately carried in a
+buffer rather than metadata (`constant/vtable/mod.rs` `Constant::buffer`).
+
+**Metadata:** none (zero bytes) (`constant/vtable/mod.rs` `Constant::serialize`).
+
+**Child slots:** none.
+
+**Decode:** decode the `ScalarValue` from buffer 0, interpreting it with the node's `DType` (supplied
+externally) (`constant/vtable/mod.rs` `Constant::deserialize`). The array is that one scalar repeated for all `n` rows. If the decoded scalar is null,
+every row is null; otherwise every row holds the scalar's value.
+
+:::{warning}
+A `vortex.constant` node's `DType` **MUST** be one the `ScalarValue` protobuf can represent. In
+particular, `Struct` and `FixedSizeList` scalars do **not** round-trip through `ScalarValue` (see
+[Scalar Format](../scalar-format.md)), so a writer **MUST NOT** emit a Constant with a `Struct` or
+`FixedSizeList` dtype — the reference writer would serialize a `ListValue` that no reader (including
+itself) can decode back. Constants of primitive, `Bool`, `Utf8`/`Binary`, `Decimal`, `List`, `Null`,
+and `Extension` (over a representable storage) dtypes are fine.
+:::
+
+**Validity:** constant-validity rule — `AllInvalid` if the scalar is null, else `AllValid` (the
+non-nullable case is already `NonNullable` via Rule 0) (`constant/vtable/validity.rs` `Constant::validity`). See
+[Constant-validity encodings](../encoding-format.md#constant-validity-encodings).
+
+(encoding-layout-Sequence)=
+
+## Sequence — `vortex.sequence`
+
+A generated arithmetic sequence: `A[i] = base + i · step`, materialised lazily. Values are integers.
+
+**Wire encoding ID:** `vortex.sequence` (`encodings/sequence/src/array.rs` `Sequence::id`).
+
+**Buffers:** none (`encodings/sequence/src/array.rs` `Sequence::nbuffers`).
+
+**Metadata:** Protobuf `SequenceMetadata` (`sequence/src/array.rs` `SequenceMetadata`):
+
+| tag | field | type | meaning |
+|-----|-------|------|---------|
+| 1 | `base` | `ScalarValue` | value at index 0 (`A[0]`) |
+| 2 | `multiplier` | `ScalarValue` | the per-step increment (the `step`) |
+
+Both are `ScalarValue` messages (see [Scalar Format](../scalar-format.md)) and are decoded as
+non-nullable primitives of the node's `ptype`. Both fields are required (`sequence/src/array.rs` `Sequence::deserialize`).
+
+**Child slots:** none.
+
+**Decode:** the node's `DType` must be `Primitive(ptype, …)` with an **integer** `ptype`
+(`sequence/src/array.rs` `SequenceData::validate`). Decode
+`base` and `multiplier` to that `ptype`. Row `i` (for `i` in `0..n`) is `base + i · multiplier`,
+evaluated in the `ptype`'s integer arithmetic (`sequence/src/array.rs` `SequenceData::index_value`). `n ≥ 1` (`sequence/src/array.rs` `SequenceData::validate`).
+
+**Validity:** constant-validity rule — always `AllValid` (every index has a value) (`sequence/src/array.rs` `Sequence::validity`). See
+[Constant-validity encodings](../encoding-format.md#constant-validity-encodings).
diff --git a/docs/specification/encoding-format/containers.md b/docs/specification/encoding-format/containers.md
new file mode 100644
index 00000000000..47cdad3c4fc
--- /dev/null
+++ b/docs/specification/encoding-format/containers.md
@@ -0,0 +1,352 @@
+# Container Encodings — Byte Layout
+
+The **container** encodings are the nested and structural forms — structs, the list variants, and
+the wrapper encodings that compose, concatenate, extend, or mask other arrays. This page is the
+per-encoding byte-layout reference for that family, one section per encoding.
+
+It is part of the [Encoding Format](../encoding-format.md) specification; null handling for every
+encoding here follows the cross-cutting [Validity](../encoding-format.md#validity) contract on that
+page and is not restated per section.
+
+Encodings covered on this page: `Struct`, `List`, `ListView`, `FixedSizeList`, `Chunked`,
+`Extension`, `Variant`, `Masked`.
+
+```{contents}
+:local:
+:depth: 1
+```
+
+## Common structure for this family
+
+Every encoding on this page shares three structural conventions. They are stated once here and
+assumed by each section below.
+
+**No self-describing dtype or length.** A serialized array node does **not** store its own `DType`
+or logical length. The parent supplies both, top-down, when it decodes a child: the child is fetched
+by *(child index, expected dtype, expected length)*. Consequently the child dtypes of a container
+are **derived by the parent**, never read from the child node — struct field dtypes come from the
+struct's dtype, a list's element dtype from the list's dtype, the offsets/sizes integer types from
+the parent's metadata, and a validity child is always a non-nullable `Bool` of length `n`.
+
+**Optional child slots are omitted, not null-padded.** The child list carried by a node is the
+sequence of *present* child slots, packed with no gaps. When an optional slot (a stored validity
+child, or Variant's `shredded` child) is absent, it is simply not written, and the following
+children shift down. A reader therefore distinguishes the layout by the **child count**: each
+section below tabulates the exact count cases. (A stored validity child is present only when it
+carries information — an `Array` mask, or an all-null `Constant(false)`; an all-valid or
+non-nullable validity produces no child. See [Validity](../encoding-format.md#validity).)
+
+**No data buffers.** Every encoding on this page reports **zero buffers**: all their state lives in
+child arrays and metadata. Bytes are held by the (leaf) children these containers wrap.
+
+Metadata, where non-empty, is a Protocol Buffers message (proto3 field defaults apply to absent
+fields). An encoding whose metadata is specified as *empty* must be rejected by a reader if the
+metadata segment is non-empty.
+
+(encoding-layout-Struct)=
+## `vortex.struct` — Struct
+
+A struct array stores a fixed set of named fields as parallel columns; every field child has the
+same length `n` as the struct, and field `f`'s value for row `i` is child-column `f` at position
+`i`. It carries an independent row-level validity that composes with each field's own validity.
+
+- **Wire encoding ID:** `vortex.struct` (`vortex-array/src/arrays/struct_/vtable/mod.rs` `Struct::id`).
+- **Buffers:** none (`nbuffers` = 0, `struct_/vtable/mod.rs` `Struct::nbuffers`).
+- **Metadata:** empty. A reader must reject non-empty metadata
+ (`struct_/vtable/mod.rs` `Struct::deserialize`). Field names, field dtypes, and the field count all come from the
+ node's `DType::Struct(fields, nullability)`, supplied top-down — not from metadata
+ (`struct_/vtable/mod.rs` `Struct::deserialize`).
+
+**Child slots** — let `N` = number of struct fields (`fields.nfields()`):
+
+| Child count | Child 0 | Children `1..=N` (or `0..N`) | Meaning |
+|-------------|---------|------------------------------|---------|
+| `N` | field `0` | fields `1..N` | no stored validity child → validity from nullability |
+| `N + 1` | validity | fields `0..N` (at child indices `1..=N`) | child 0 is the stored validity |
+
+Every field child `f` has dtype `fields[f]` and length `n` (`struct_/vtable/mod.rs` `Struct::validate`,
+`Struct::deserialize`). Fields are **row-aligned**: struct row `i` is the tuple `(field_0[i], …, field_{N-1}[i])`.
+Field order is positional and matches the dtype's field order; names are not stored per node. Field
+names need not be unique — accessors resolve a name to the first matching field
+(`struct_/array.rs` `StructDataParts`).
+
+**Decode.** Read the field count `N` from the dtype. If the node has `N` children, they are the
+fields and validity is all-valid/non-nullable; if `N + 1`, child 0 is validity and children
+`1..=N` are the fields. Decode each field child at its dtype and length `n`. Struct row `i` reads
+position `i` from every field child.
+
+**Validity.** Stored slot (Mechanism 1), row-aligned. The struct carries a *top-level* row validity
+that **composes** with field validity: a null struct row makes the whole row null regardless of any
+field's own value. See [Validity → Nested containers](../encoding-format.md#validity) — do not treat
+field validity as the row validity or vice versa.
+
+(encoding-layout-List)=
+## `vortex.list` — List
+
+A list array stores variable-length lists via a single **offsets** child that indexes into a flat
+**elements** child. It mirrors the Apache Arrow `List` layout.
+
+- **Wire encoding ID:** `vortex.list` (`vortex-array/src/arrays/list/vtable/mod.rs` `List::id`).
+- **Buffers:** none (`nbuffers` = 0, `list/vtable/mod.rs` `List::nbuffers`).
+- **Metadata** (`ListMetadata`, `list/vtable/mod.rs` `ListMetadata`):
+ - `elements_len: u64` (tag 1) — logical length of the `elements` child.
+ - `offset_ptype: PType` (tag 2) — integer type of the `offsets` child (see the `PType`
+ enumeration in [Canonical encodings](canonical.md) or [DType Format](../dtype-format.md)).
+
+**Child slots** (`list/array.rs` `ELEMENTS_SLOT`):
+
+| Index | Name | Present | DType | Length | Role |
+|-------|------|---------|-------|--------|------|
+| 0 | `elements` | always | list element dtype (from `DType::List`) | `elements_len` | flat concatenation of all list contents |
+| 1 | `offsets` | always | `Primitive(offset_ptype, NonNullable)` | `n + 1` | list boundaries |
+| 2 | `validity` | optional | non-nullable `Bool` | `n` | row-level nulls |
+
+Child count is **2** (no stored validity) or **3** (child 2 is validity)
+(`list/vtable/mod.rs` `List::deserialize`). `n` is the outer length; `offsets.len() == n + 1`
+(`list/vtable/mod.rs` `List::validate`).
+
+**Offset semantics.** `offsets` is a **non-nullable integer** array of length `n + 1`, monotonically
+non-decreasing, with `offsets[0] >= 0` and `offsets[n] <= elements_len`. List `i` occupies the
+half-open range `elements[offsets[i] .. offsets[i+1]]`; its length is `offsets[i+1] - offsets[i]`
+(zero-length lists allowed) (`list/array.rs` `ListData`, `ListData::validate`, `ListArrayExt::list_elements_at`).
+
+**Decode.** Read `elements_len` and `offset_ptype` from metadata. Decode `elements` (child 0) at the
+element dtype and length `elements_len`; decode `offsets` (child 1) as `Primitive(offset_ptype,
+NonNullable)` of length `n + 1`. For row `i`, slice `elements[offsets[i] .. offsets[i+1]]`. If a
+third child is present it is the validity mask of length `n`.
+
+**Validity.** Stored slot (Mechanism 1), top-level and row-aligned; composes with element validity
+exactly as for Struct. See [Validity → Nested containers](../encoding-format.md#validity).
+
+(encoding-layout-ListView)=
+## `vortex.listview` — ListView
+
+A list-view array is the canonical `DType::List` encoding. Unlike `List`, each row carries **both**
+an offset and a size, so the views may be out of order, may overlap, and need not tile the elements
+child. It mirrors Apache Arrow's column-major `ListView`.
+
+- **Wire encoding ID:** `vortex.listview` (`vortex-array/src/arrays/listview/vtable/mod.rs` `ListView::id`).
+- **Buffers:** none (`nbuffers` = 0, `listview/vtable/mod.rs` `ListView::nbuffers`).
+- **Metadata** (`ListViewMetadata`, `listview/vtable/mod.rs` `ListViewMetadata`):
+ - `elements_len: u64` (tag 1) — logical length of the `elements` child.
+ - `offset_ptype: PType` (tag 2) — integer type of the `offsets` child (see the `PType`
+ enumeration in [Canonical encodings](canonical.md) or [DType Format](../dtype-format.md)).
+ - `size_ptype: PType` (tag 3) — integer type of the `sizes` child (see the `PType` enumeration in
+ [Canonical encodings](canonical.md) or [DType Format](../dtype-format.md)).
+
+**Child slots** (`listview/array.rs` `ELEMENTS_SLOT`):
+
+| Index | Name | Present | DType | Length | Role |
+|-------|------|---------|-------|--------|------|
+| 0 | `elements` | always | list element dtype (from `DType::List`) | `elements_len` | flat pool of list contents |
+| 1 | `offsets` | always | `Primitive(offset_ptype, NonNullable)` | `n` | per-row start into `elements` |
+| 2 | `sizes` | always | `Primitive(size_ptype, NonNullable)` | `n` | per-row length |
+| 3 | `validity` | optional | non-nullable `Bool` | `n` | row-level nulls |
+
+Child count is **3** (no stored validity) or **4** (child 3 is validity)
+(`listview/vtable/mod.rs` `ListView::deserialize`). Both `offsets` and `sizes` have length exactly `n` — **not**
+`n + 1` as in `List` (`listview/vtable/mod.rs` `ListView::validate`, `ListView::deserialize`).
+
+**Offset/size semantics.** `offsets` and `sizes` are **non-nullable integer** arrays of equal length
+`n`. Row `i` is the range `elements[offsets[i] .. offsets[i] + sizes[i]]`
+(`listview/array.rs` `ListViewArrayExt::list_elements_at`). Constraints (`listview/array.rs` `ListViewData::new_unchecked`, `validate_offsets_and_sizes`): every
+`offsets[i] + sizes[i] <= elements_len` (and no overflow); if `offsets[i] == elements_len` then
+`sizes[i]` must be 0. (`offset_ptype` and `size_ptype` may be any integer widths independently — the
+decode reads each child at its own type; there is no width relationship between them.) Offsets are **not** required
+to be sorted, gaps and overlaps are permitted, and these constraints hold even for rows the validity
+marks null.
+
+**Decode.** Read the three metadata fields. Decode `elements` (child 0) at `elements_len`; decode
+`offsets` (child 1) and `sizes` (child 2) as non-nullable primitives of their metadata ptypes, each
+length `n`. For row `i`, slice `elements[offsets[i] .. offsets[i] + sizes[i]]`. A fourth child, if
+present, is the validity mask of length `n`.
+
+:::{note}
+The in-memory `is_zero_copy_to_list` optimization flag (`listview/array.rs` `ListViewData`) is **not**
+serialized. On decode it is reset to `false` (`ListViewData::try_new`, `listview/vtable/mod.rs` `ListView::deserialize`);
+it is a runtime hint, never a wire fact.
+:::
+
+**Validity.** Stored slot (Mechanism 1), top-level and row-aligned; composes with element validity
+as for Struct/List. See [Validity → Nested containers](../encoding-format.md#validity).
+
+(encoding-layout-FixedSizeList)=
+## `vortex.fixed_size_list` — FixedSizeList
+
+A fixed-size-list array stores lists that all have the same length `list_size`, so no offsets are
+needed: the elements are laid out contiguously and row `i` is a fixed-width slice.
+
+- **Wire encoding ID:** `vortex.fixed_size_list`
+ (`vortex-array/src/arrays/fixed_size_list/vtable/mod.rs` `FixedSizeList::id`).
+- **Buffers:** none (`nbuffers` = 0, `fixed_size_list/vtable/mod.rs` `FixedSizeList::nbuffers`).
+- **Metadata:** empty; a reader must reject non-empty metadata
+ (`fixed_size_list/vtable/mod.rs` `FixedSizeList::deserialize`). `list_size` (a `u32`) and the element dtype come from the
+ node's `DType::FixedSizeList(element_dtype, list_size, nullability)`, supplied top-down
+ (`fixed_size_list/vtable/mod.rs` `FixedSizeList::deserialize`).
+
+**Child slots** (`fixed_size_list/array.rs` `ELEMENTS_SLOT`):
+
+| Index | Name | Present | DType | Length | Role |
+|-------|------|---------|-------|--------|------|
+| 0 | `elements` | always | element dtype (from dtype) | `n * list_size` | flat contiguous elements |
+| 1 | `validity` | optional | non-nullable `Bool` | `n` | row-level nulls |
+
+Child count is **1** (no stored validity) or **2** (child 1 is validity)
+(`fixed_size_list/vtable/mod.rs` `FixedSizeList::deserialize`). `elements.len() == n * list_size`
+(`fixed_size_list/array.rs` `FixedSizeListData::validate`).
+
+**Element semantics.** Row `i` is the slice `elements[i * list_size .. (i + 1) * list_size]`
+(`fixed_size_list/array.rs` `FixedSizeListArrayExt::fixed_size_list_elements_at`). Degenerate case `list_size == 0`: `elements` is empty and the
+row count `n` cannot be recovered from the children — it is supplied by the parent as the node length
+(`fixed_size_list/array.rs` `FixedSizeListData.degenerate_len`, `FixedSizeListData::validate`).
+
+**Decode.** Read `list_size` and element dtype from the dtype. Decode `elements` (child 0) at the
+element dtype and length `n * list_size`. For row `i`, slice `elements[i*list_size .. (i+1)*list_size]`.
+A second child, if present, is the validity mask of length `n`.
+
+**Validity.** Stored slot (Mechanism 1), top-level and row-aligned; composes with element validity as
+for the other list containers. See [Validity → Nested containers](../encoding-format.md#validity).
+
+(encoding-layout-Chunked)=
+## `vortex.chunked` — Chunked
+
+A chunked array is the logical concatenation, in order, of a sequence of same-dtype child chunks. A
+leading child holds the chunk boundary offsets.
+
+- **Wire encoding ID:** `vortex.chunked` (`vortex-array/src/arrays/chunked/vtable/mod.rs` `Chunked::id`).
+- **Buffers:** none (`nbuffers` = 0, `chunked/vtable/mod.rs` `Chunked::nbuffers`).
+- **Metadata:** empty; a reader must reject non-empty metadata (`chunked/vtable/mod.rs` `Chunked::deserialize`). The
+ number of chunks is `child_count - 1`.
+
+**Child slots** (`chunked/array.rs` `CHUNK_OFFSETS_SLOT`) — let `K` = number of chunks:
+
+| Index | Name | DType | Length | Role |
+|-------|------|-------|--------|------|
+| 0 | `chunk_offsets` | `Primitive(U64, NonNullable)` | `K + 1` | exclusive prefix-sum of chunk lengths |
+| `1 + j` | `chunks[j]` | the node's own dtype | `chunk_offsets[j+1] - chunk_offsets[j]` | the `j`-th chunk |
+
+Every chunk child has the **same dtype as the chunked node itself** (`chunked/vtable/mod.rs` `Chunked::validate`).
+There is always at least the `chunk_offsets` child (`chunked/vtable/mod.rs` `Chunked::deserialize`).
+
+**Offset semantics.** `chunk_offsets` is a non-nullable `u64` array of length `K + 1`, built by the
+writer as an exclusive prefix sum (`chunked/array.rs` `ChunkedData::compute_chunk_offsets`) — so
+`chunk_offsets[0] == 0` **by construction**, it is non-decreasing, and `chunk_offsets[j+1] -
+chunk_offsets[j]` is chunk `j`'s length. The invariant enforced on read is `chunk_offsets[K] == n`
+(the total length) (`chunked/vtable/mod.rs` `Chunked::validate`, `Chunked::deserialize`). Row `i` lives in the unique
+chunk `j` with `chunk_offsets[j] <= i < chunk_offsets[j+1]`, at chunk-local position
+`i - chunk_offsets[j]` (`chunked/array.rs` `ChunkedArrayExt::find_chunk_idx`).
+
+**Decode.** Let `K = child_count - 1`. Decode child 0 as `Primitive(U64, NonNullable)` of length
+`K + 1` to get `chunk_offsets`. For `j` in `0..K`, decode child `1 + j` at the node's dtype and
+length `chunk_offsets[j+1] - chunk_offsets[j]`. The logical array is the chunks concatenated in
+order; map a global row `i` to `(chunk j, i - chunk_offsets[j])` by binary search over
+`chunk_offsets`.
+
+**Validity.** Chunked does **not** carry its own validity slot. It uses the **combine** mechanism:
+its validity is the ordered concatenation of each chunk's validity. See
+[Validity → `vortex.chunked`](../encoding-format.md#validity).
+
+(encoding-layout-Extension)=
+## `vortex.ext` — Extension
+
+An extension array attaches a semantic extension type to a single storage child without changing the
+storage bytes. All values and validity are those of the storage child, reinterpreted through the
+extension type.
+
+- **Wire encoding ID:** `vortex.ext` (`vortex-array/src/arrays/extension/vtable/mod.rs` `Extension::id`).
+- **Buffers:** none (`nbuffers` = 0, `extension/vtable/mod.rs` `Extension::nbuffers`).
+- **Metadata:** empty; a reader must reject non-empty metadata (`extension/vtable/mod.rs` `Extension::deserialize`). The
+ extension identity, the storage dtype, and any extension-specific metadata are carried by the
+ node's `DType::Extension(ext_dtype)` (supplied top-down), **not** in the array metadata segment
+ (`extension/vtable/mod.rs` `Extension::deserialize`).
+
+**Child slots** (`extension/array.rs` `STORAGE_SLOT`):
+
+| Index | Name | Present | DType | Length | Role |
+|-------|------|---------|-------|--------|------|
+| 0 | `storage` | always (exactly 1 child) | `ext_dtype.storage_dtype()` | `n` | the underlying values |
+
+The node has exactly one child (`extension/vtable/mod.rs` `Extension::deserialize`); its dtype is the extension type's
+declared storage dtype and its length equals `n` (`extension/vtable/mod.rs` `Extension::validate`, `Extension::deserialize`).
+
+**Decode.** Take the storage dtype from the extension dtype. Decode child 0 at that dtype and length
+`n`. Row `i` of the extension array is storage row `i`, interpreted according to the extension type.
+
+**Validity.** Delegate (Mechanism 2) to the `storage` child — the extension array's validity **is**
+its storage child's validity, unchanged (`ValidityVTableFromChild`, `extension/vtable/mod.rs` `Extension::ValidityVTable`).
+See [Validity → delegate](../encoding-format.md#validity).
+
+(encoding-layout-Variant)=
+## `vortex.variant` — Variant
+
+A variant array stores semi-structured (`DType::Variant`) values. Every row's full value lives in the
+`core_storage` child; an optional row-aligned `shredded` child holds typed values for selected paths
+to accelerate typed access.
+
+- **Wire encoding ID:** `vortex.variant` (`vortex-array/src/arrays/variant/vtable/mod.rs` `Variant::id`).
+- **Buffers:** none (`nbuffers` = 0, `variant/vtable/mod.rs` `Variant::nbuffers`).
+- **Metadata** (`VariantMetadataProto`, `variant/vtable/mod.rs` `VariantMetadataProto`):
+ - `shredded_dtype: Option` (tag 1) — the dtype of the `shredded` child, as an encoded
+ `vortex-proto` `DType`. Present **iff** a `shredded` child exists.
+
+**Child slots** (`variant/mod.rs` `CORE_STORAGE_SLOT`):
+
+| Index | Name | Present | DType | Length | Role |
+|-------|------|---------|-------|--------|------|
+| 0 | `core_storage` | always | the node's own `DType::Variant` | `n` | full variant value per row |
+| 1 | `shredded` | optional | `shredded_dtype` (from metadata) | `n` | typed values for selected paths |
+
+Child count is **1** when `shredded_dtype` is absent, **2** when present
+(`variant/vtable/mod.rs` `Variant::deserialize`). `core_storage` has the **same** `DType::Variant` as the node and
+length `n`; it is a logical variant array and may itself be any variant-typed encoding (chunked,
+constant, …) — do not assume a physical layout for it (`variant/mod.rs` `VariantArrayExt`, `Array::try_new`). When present,
+`shredded` is row-aligned (length `n`) and may have any dtype (`variant/vtable/mod.rs` `Variant::validate`, `Variant::deserialize`).
+
+**Decode.** Decode metadata; if `shredded_dtype` is set, expect 2 children, else 1. Decode child 0
+(`core_storage`) at the node's variant dtype and length `n`. If present, decode child 1
+(`shredded`) at `shredded_dtype` and length `n`. Row `i`'s logical value is `core_storage[i]`;
+`shredded[i]`, when present and non-null at a path, supplies the typed value for that path (a
+reader that ignores shredding can decode the full value from `core_storage` alone).
+
+:::{note}
+`core_storage` always retains the complete value for every row, so it is authoritative for
+structural decoding. The exact rules for **merging** a `shredded` typed tree back over `core_storage`
+(field-by-field precedence, null fallback) are a compute-layer concern
+(`variant/vtable/mod.rs` `merge_typed_scalar_as_variant`) beyond this byte-layout section.
+:::
+
+**Validity.** Delegate (Mechanism 2) to the `core_storage` child — Variant's validity is
+`core_storage`'s validity. See [Validity → delegate](../encoding-format.md#validity).
+
+(encoding-layout-Masked)=
+## `vortex.masked` — Masked
+
+A masked array superimposes a nullability mask onto a single non-null child, producing a nullable
+view without rewriting the child. It is always nullable, and its child must contain no **top-level**
+nulls (a nested child — e.g. a `List` — may still carry inner-element nulls; only top-level row nulls
+are forbidden).
+
+- **Wire encoding ID:** `vortex.masked` (`vortex-array/src/arrays/masked/vtable/mod.rs` `Masked::id`).
+- **Buffers:** none (`nbuffers` = 0, `masked/vtable/mod.rs` `Masked::nbuffers`; a reader must reject any buffer,
+ `masked/vtable/mod.rs` `Masked::deserialize`).
+- **Metadata:** empty; a reader must reject non-empty metadata (`masked/vtable/mod.rs` `Masked::deserialize`).
+
+**Child slots** (`masked/array.rs` `MaskedSlots`):
+
+| Index | Name | Present | DType | Length | Role |
+|-------|------|---------|-------|--------|------|
+| 0 | `child` | always | the node's dtype made **non-nullable** | `n` | the underlying values (no nulls) |
+| 1 | `validity` | optional | non-nullable `Bool` | `n` | which rows are null |
+
+Child count is **1** (no stored validity child) or **2** (child 1 is validity)
+(`masked/vtable/mod.rs` `Masked::deserialize`). The `child` dtype is the node dtype with nullability removed
+(`masked/vtable/mod.rs` `Masked::validate`, `Masked::deserialize`); the node dtype is always nullable
+(`masked/array.rs` `MaskedData::try_new`, `Array::try_new`). Row-aligned: `child.len() == n` (`masked/vtable/mod.rs` `Masked::validate`).
+
+**Decode.** Decode child 0 at `dtype.as_nonnullable()` and length `n`. If a second child is present,
+it is the validity mask (length `n`); otherwise validity comes from the node's nullability. Row `i`
+is null iff validity says so, and equals `child[i]` otherwise. (Because the child is guaranteed
+top-level null-free, the mask is the sole source of top-level nulls.)
+
+**Validity.** Stored slot (Mechanism 1), row-aligned — the `validity` child directly gives per-row
+validity of length `n` (`masked/array.rs` `MaskedArrayExt::masked_validity`). See [Validity](../encoding-format.md#validity).
diff --git a/docs/specification/encoding-format/dict-runend-sparse.md b/docs/specification/encoding-format/dict-runend-sparse.md
new file mode 100644
index 00000000000..5c680d89db6
--- /dev/null
+++ b/docs/specification/encoding-format/dict-runend-sparse.md
@@ -0,0 +1,228 @@
+# Dictionary, Run-End, and Sparse Encodings — Byte Layout
+
+The **dictionary**, **run-end**, and **sparse** encodings derive their logical values — and their
+validity — by combining child arrays (codes and values, runs and ends, or fill value and patches).
+This page is the per-encoding byte-layout reference for that family, one section per encoding.
+
+It is part of the [Encoding Format](../encoding-format.md) specification; null handling for every
+encoding here follows the cross-cutting [Validity](../encoding-format.md#validity) contract on that
+page and is not restated per section.
+
+Encodings covered on this page: `Dict`, `RunEnd`, `Sparse`.
+
+Throughout, `n` is the node's logical length (the row count carried by the array container, after any
+slicing). Each encoding's validity is sourced by the **combine** mechanism; the exact combine rule is
+given once in [Validity](../encoding-format.md#validity) and is **not** restated here. These sections
+specify the **data** decode — how row `i` resolves to a value — only.
+
+(encoding-layout-Dict)=
+## `vortex.dict` (Dictionary) — byte layout
+
+A dictionary stores each logical row as an integer **code** that indexes a shared **values** table.
+The decode identity is exact:
+
+> **row `i` = `values[codes[i]]`.**
+
+**Wire ID:** `vortex.dict` (`vortex-array/src/arrays/dict/vtable/mod.rs` `Dict::id`). **Buffers:** none
+(0) (`dict/vtable/mod.rs` `Dict::nbuffers`). **Children:** exactly 2 — `codes` (slot 0) and
+`values` (slot 1) (`dict/array.rs` `DictSlots`); a reader must reject any node that does not present
+exactly these two children (`dict/vtable/mod.rs` `Dict::deserialize`).
+
+### Metadata
+
+Protobuf message `DictMetadata` (`dict/array.rs` `DictMetadata`):
+
+| Proto tag | Field | Type | Meaning |
+|-----------|-------|------|---------|
+| 1 | `values_len` | `uint32` | Length (row count) of the `values` child. |
+| 2 | `codes_ptype` | `PType` enum | Primitive type of the `codes` child (an integer type). |
+| 3 | `is_nullable_codes` | optional `bool` | Whether the `codes` child's dtype is nullable. Absent (added after stabilisation) ⇒ fall back to the node's own nullability (`dict/vtable/mod.rs` `Dict::deserialize`). |
+| 4 | `all_values_referenced` | optional `bool` | Optimisation hint (`true` ⇒ every value is referenced by some code). **Not needed to decode**; never rely on it for correctness. |
+
+### Child slots
+
+| Slot | Name | dtype | Length |
+|------|------|-------|--------|
+| 0 | `codes` | `Primitive(codes_ptype, nullable?)` — integer; `nullable?` from `is_nullable_codes` (else the node's nullability) | `n` |
+| 1 | `values` | the node's own logical dtype | `values_len` |
+
+`codes[i]` is a non-negative integer used directly as a 0-based index into `values`; every code
+satisfies `0 <= codes[i] < values_len`. Both children are ordinary Vortex nodes and are decoded
+recursively.
+
+### Decode
+
+1. Decode the `codes` child to an integer array of length `n`, and the `values` child to an array of
+ length `values_len` (`dict/vtable/mod.rs` `Dict::deserialize`).
+2. For each row `i` in `0..n`: read `c = codes[i]`, then emit `values[c]`
+ (`dict/vtable/mod.rs` `Dict::execute`).
+
+Nullability composes per the combine rule (a null code, or a code pointing at a null value, yields a
+null row); see [Validity](../encoding-format.md#validity).
+
+**Example.** `codes = [0, 2, 1, 0]`, `values = ["a", "b", "c"]` ⇒ `["a", "c", "b", "a"]`.
+
+(encoding-layout-RunEnd)=
+## `vortex.runend` (Run-End) — byte layout
+
+A run-end array stores one **value** per run together with the **exclusive end position** of each
+run. Row values are recovered by expanding each run across the positions it covers. The encoding is
+**offset-aware**: a slice is expressed by an `offset` into the run coordinate space plus the logical
+length `n`, without rewriting `ends` or `values`.
+
+**Wire ID:** `vortex.runend` (`encodings/runend/src/array.rs` `RunEnd::id`). **Buffers:** none (0)
+(`encodings/runend/src/array.rs` `RunEnd::nbuffers`). **Children:** exactly 2 — `ends` (slot 0) and
+`values` (slot 1) (`encodings/runend/src/array.rs` `NUM_SLOTS`, `SLOT_NAMES`).
+
+### Metadata
+
+Protobuf message `RunEndMetadata` (`encodings/runend/src/array.rs` `RunEndMetadata`):
+
+| Proto tag | Field | Type | Meaning |
+|-----------|-------|------|---------|
+| 1 | `ends_ptype` | `PType` enum | Primitive type of the `ends` child (an **unsigned** integer type). |
+| 2 | `num_runs` | `uint64` | Number of runs = length of **both** the `ends` and `values` children (`encodings/runend/src/array.rs` `RunEnd::serialize`, `RunEndData::validate_parts`). |
+| 3 | `offset` | `uint64` | Logical slice offset into the unsliced run coordinate space (see below). Unsliced arrays have `offset = 0`. |
+
+### Child slots
+
+| Slot | Name | dtype | Length |
+|------|------|-------|--------|
+| 0 | `ends` | `Primitive(ends_ptype, NonNullable)` — unsigned integer (`encodings/runend/src/array.rs` `RunEnd::deserialize`, `RunEndData::validate_parts`) | `num_runs` |
+| 1 | `values` | the node's own logical dtype (`encodings/runend/src/array.rs` `RunEnd::deserialize`, `RunEnd::validate`) | `num_runs` |
+
+`ends` is **strictly increasing** (`encodings/runend/src/compress.rs` `runend_encode`). `ends[j]` is
+the **exclusive** end of run `j` in the *unsliced*
+coordinate space: run `j` covers unsliced positions `[ends[j-1], ends[j])`, with `ends[-1] = 0`
+(`encodings/runend/src/compress.rs` `runend_encode`). The
+total unsliced extent is `ends[num_runs-1]` (`encodings/runend/src/array.rs`
+`RunEndData::logical_len_from_ends`). Validity of a well-formed node satisfies
+`ends[num_runs-1] >= offset + n` (and, when `offset != 0`, `ends[0] >= offset`)
+(`encodings/runend/src/array.rs` `RunEndData::validate_parts`).
+
+:::{note}
+"Children: exactly 2" and "`ends` strictly increasing" are **writer requirements** (the format
+contract). The reference *reader* does not hard-enforce them — `RunEnd::deserialize` fetches children
+by index without rejecting a spurious third, and strict-increase is only a debug-assertion — so a
+conformant reader SHOULD validate both itself rather than assume them. (`vortex.dict` and
+`vortex.sparse`, by contrast, do reject a wrong child count on read.)
+:::
+
+### Decode (offset-aware run expansion)
+
+A sliced run-end array selects the logical window `offset .. offset + n` of the unsliced coordinate
+space. Logical row `i` (`0 <= i < n`) corresponds to unsliced position `i + offset`.
+
+**Random access (single row `i`):**
+
+1. Let `p = i + offset`.
+2. Find the run `j` = the smallest index with `ends[j] > p` (equivalently a right-side binary search
+ for `p` over `ends`) (`encodings/runend/src/array.rs` `RunEndArrayExt::find_physical_index`).
+ Because `ends[num_runs-1] > p` for every in-range `i`, this always lands
+ inside `0..num_runs`.
+3. Emit `values[j]` (`encodings/runend/src/ops.rs` `RunEnd::scalar_at`).
+
+**Bulk expansion (all `n` rows):** first map every raw end into the sliced window by clamping —
+`trimmed[j] = min(ends[j] - offset, n)` (`encodings/runend/src/iter.rs` `trimmed_ends_iter`) — then
+fill left to right (`encodings/runend/src/compress.rs` `runend_decode_slice`):
+
+```text
+cursor = 0
+for j in 0 .. num_runs:
+ e = min(ends[j] - offset, n) # trimmed end; ends[j] >= offset is guaranteed
+ for row in cursor .. e: # empty span if e <= cursor
+ out[row] = values[j]
+ cursor = e
+ if e == n: break # window filled
+```
+
+Runs whose end equals `offset` (`ends[j] == offset`; the strict `ends[j] < offset` case is precluded
+by the `ends[0] >= offset` invariant (`encodings/runend/src/array.rs` `RunEndData::validate_parts`))
+trim to `0` and emit nothing; runs whose end lies beyond
+`offset + n` clamp to `n` and terminate expansion (`encodings/runend/src/iter.rs` `trimmed_ends_iter`).
+A reader that ignores
+`offset`/`n` and expands `ends` verbatim produces a shifted, wrong-length result on any sliced array.
+
+**Example.** `ends = [2, 5, 10]`, `values = [1, 2, 3]`.
+- `offset = 0, n = 10` ⇒ `[1, 1, 2, 2, 2, 3, 3, 3, 3, 3]`.
+- `offset = 2, n = 5` (the `2..7` window) ⇒ `trimmed = [0, 3, 5]` ⇒ `[2, 2, 2, 3, 3]`.
+
+(encoding-layout-Sparse)=
+## `vortex.sparse` (Sparse) — byte layout
+
+A sparse array stores a single **fill value** for the overwhelming majority of positions plus a
+sorted set of **patches** — `(index, value)` pairs — for the exceptional positions. Row `i` is the
+patch value if `i` is a patch index, else the fill value.
+
+**Wire ID:** `vortex.sparse` (`encodings/sparse/src/lib.rs` `Sparse::id`). **Buffers:** 1 — buffer 0,
+named `fill_value` (`encodings/sparse/src/lib.rs` `Sparse::nbuffers`, `Sparse::buffer_name`).
+**Children:** exactly
+2 — `patch_indices` (slot 0) and `patch_values` (slot 1) (`encodings/sparse/src/lib.rs` `SparseSlots`,
+`Sparse::deserialize`).
+
+### Buffer 0 — `fill_value`
+
+The fill scalar's *value* serialized as `ScalarValue` protobuf bytes (`encodings/sparse/src/lib.rs`
+`Sparse::buffer`). It is **not** in the metadata (`encodings/sparse/src/lib.rs` `Sparse::serialize`).
+Decode it against the node's own `DType` (the fill scalar's dtype equals the node dtype)
+(`encodings/sparse/src/lib.rs` `Sparse::deserialize`). If the fill
+scalar is null, every non-patched position is null.
+
+### Metadata
+
+Protobuf message `SparseMetadata` (`encodings/sparse/src/lib.rs` `SparseMetadata`) wraps a single
+required `PatchesMetadata` (tag 1) (`vortex-array/src/patches.rs` `PatchesMetadata`):
+
+| Proto tag | Field | Type | Meaning |
+|-----------|-------|------|---------|
+| 1 | `len` | `uint64` | Number of patches = length of **both** the `patch_indices` and `patch_values` children. |
+| 2 | `offset` | `uint64` | Absolute slice offset subtracted from each stored index (see below). Unsliced arrays have `offset = 0`. |
+| 3 | `indices_ptype` | `PType` enum | Primitive type of `patch_indices` (an **unsigned** integer type). |
+| 4 | `chunk_offsets_len` | optional `uint64` | In-memory lookup accelerator only — see note. |
+| 5 | `chunk_offsets_ptype` | optional `PType` enum | In-memory lookup accelerator only — see note. |
+| 6 | `offset_within_chunk` | optional `uint64` | In-memory lookup accelerator only — see note. |
+
+:::{note}
+The `chunk_offsets` accelerator — the `patch_chunk_offsets` child (a potential third patches child)
+and `PatchesMetadata` tags 4–6 — is an in-memory `O(1)`-lookup index. It is **never attached to a
+serialized Sparse node** by any production writer path: `Sparse::try_new`
+(`encodings/sparse/src/lib.rs` `Sparse::try_new`), `encode` (`encodings/sparse/src/lib.rs`
+`SparseData::encode`), and `slice` (`encodings/sparse/src/slice.rs` `Sparse::slice`) all
+pass or preserve `None`. A conformant reader therefore resolves every row purely by `offset` + a
+binary search over `patch_indices`, treating tags 4–6 and a third child as absent — and rejecting a
+node that carries them (the deserializer requires *exactly* two children, `patch_indices` and
+`patch_values`) (`encodings/sparse/src/lib.rs` `Sparse::deserialize`).
+:::
+
+### Child slots
+
+| Slot | Name | dtype | Length |
+|------|------|-------|--------|
+| 0 | `patch_indices` | `Primitive(indices_ptype, NonNullable)` — unsigned integer (`vortex-array/src/patches.rs` `PatchesMetadata::indices_dtype`), **strictly sorted** | `len` |
+| 1 | `patch_values` | the node's own logical dtype (`encodings/sparse/src/lib.rs` `Sparse::deserialize`) | `len` |
+
+`patch_indices` are stored in the **absolute (pre-slice) coordinate space**: `offset` is how many
+leading positions were sliced off the front (`vortex-array/src/patches.rs` `Patches::slice`). Each
+stored index satisfies
+`stored_index - offset < n` (`vortex-array/src/patches.rs` `Patches::new`).
+
+### Decode (patch resolution)
+
+Logical row `i` (`0 <= i < n`) maps to absolute position `a = i + offset`
+(`vortex-array/src/patches.rs` `Patches::search_index`).
+
+1. Binary-search `patch_indices` for the value `a` (`vortex-array/src/patches.rs`
+ `Patches::search_index_binary_search`).
+2. If found at patch position `p`, emit `patch_values[p]` (`vortex-array/src/patches.rs`
+ `Patches::get_patched`).
+3. Otherwise emit the `fill_value` scalar (`encodings/sparse/src/ops.rs` `Sparse::scalar_at`).
+
+Nullability composes per the combine rule (patched positions take the patch value's validity;
+non-patched positions take `fill_value`'s validity); see
+[Validity](../encoding-format.md#validity).
+
+**Example.** `patch_indices = [2, 5, 8]`, `patch_values = [100, 200, 300]`, `fill_value = null`,
+`n = 10`.
+- `offset = 0` ⇒ `[null, null, 100, null, null, 200, null, null, 300, null]`.
+- `offset = 2, n = 5` (the `2..7` window) ⇒ absolute positions `2..7` ⇒
+ `[100, null, null, 200, null]`.
diff --git a/docs/specification/encoding-format/fastlanes.md b/docs/specification/encoding-format/fastlanes.md
new file mode 100644
index 00000000000..4310d34c487
--- /dev/null
+++ b/docs/specification/encoding-format/fastlanes.md
@@ -0,0 +1,363 @@
+# FastLanes Encodings — Byte Layout
+
+The **FastLanes** family is the SIMD-oriented integer-compression group: bit-packing,
+frame-of-reference, delta, and run-length encodings that share the FastLanes block structure. This
+page is the per-encoding byte-layout reference for that family, one section per encoding.
+
+It is part of the [Encoding Format](../encoding-format.md) specification; null handling for every
+encoding here follows the cross-cutting [Validity](../encoding-format.md#validity) contract on that
+page and is not restated per section.
+
+Encodings covered on this page: `BitPacked`, `FoR`, `Delta`, `RLE`.
+
+Throughout, `n` is the node's logical length (the row count carried by the array container, after any
+slicing).
+
+## FastLanes fundamentals
+
+All four encodings on this page share the FastLanes block machinery. This section defines it once; the
+per-encoding sections reference it. It is precise enough to reimplement.
+
+### Blocks of 1024
+
+Data is processed in fixed **blocks of 1024 values** (`FL_CHUNK_SIZE = 1024`, `lib.rs` `FL_CHUNK_SIZE`). A writer
+splits the input into consecutive 1024-value blocks and **zero-pads the final block** up to 1024. All
+stored child/buffer lengths are therefore multiples of 1024 (or of a per-block count derived from it),
+and the array's logical length `n` — plus, when sliced, an `offset` — selects the live window out of
+that padded extent.
+
+### Element width and lanes
+
+Let `T` be the **unsigned** integer type whose width matches the logical primitive type (`u8` for
+`i8`/`u8`, `u16` for `i16`/`u16`, … `u64` for `i64`/`u64`). Write `T_BITS` for its bit width
+(8/16/32/64) and define
+
+```text
+LANES = 1024 / T_BITS # u8→128, u16→64, u32→32, u64→16
+```
+
+(`fastlanes` 0.5.1 `lib.rs` `FastLanes`). A block is viewed as `LANES` interleaved **lanes**, each holding
+`T_BITS` values.
+
+### The order permutation `FL_ORDER`
+
+```text
+FL_ORDER = [0, 4, 2, 6, 1, 5, 3, 7] # an involution: FL_ORDER[FL_ORDER[k]] == k
+```
+
+(`fastlanes` 0.5.1 `lib.rs` `FL_ORDER`).
+
+(fastlanes-packing-order)=
+### Bit-packing layout (`W` bits per value)
+
+Bit-packing stores one block of 1024 values at a uniform width `W` bits each (`0 ≤ W ≤ T_BITS`),
+occupying exactly `1024·W` bits = `128·W` bytes. View that region as `1024·W / T_BITS` words of type
+`T`, laid out little-endian in memory. Index the words as `word[w·LANES + lane]` for `w` in `0..W`,
+`lane` in `0..LANES`.
+
+For a fixed `lane`, concatenate its `W` words from `w = 0` (least-significant) to `w = W−1`
+(most-significant), each contributing `T_BITS` bits low-bit-first, to form that lane's `W·T_BITS`-bit
+stream. The value for **row** `r` (`r` in `0..T_BITS`) is the `W`-bit field at bit offset `r·W` in the
+lane stream. That value's position **within the block's logical (row) order** is
+
+```text
+pack_index(r, lane) = FL_ORDER[r / 8] * 16 + (r % 8) * 128 + lane
+```
+
+(`fastlanes` 0.5.1 `macros.rs` `index`, the `unpack!` kernel `macros.rs` `index`). `pack_index` is a
+bijection of `{0..T_BITS} × {0..LANES}` onto `0..1024`. To unpack a block directly into logical order:
+
+```text
+for lane in 0 .. LANES:
+ for r in 0 .. T_BITS:
+ v = W-bit field at bit offset r*W of lane `lane`'s stream
+ block[pack_index(r, lane)] = v
+```
+
+Special cases: `W = 0` ⇒ the region is empty and every value is `0`; `W = T_BITS` ⇒ word
+`word[r·LANES + lane]` holds `block[pack_index(r, lane)]` verbatim. Unpacking yields **row order** —
+the internal transposition is invisible in the output of plain bit-packing. (Values are unpacked as
+unsigned `T`; reinterpreting the block as a signed logical type is a no-op on the bit pattern, valid
+because packed values are non-negative and any out-of-range value is carried as a patch.)
+
+(fastlanes-transpose)=
+### The 1024-element transpose
+
+Delta stores its values in a **transposed** block order that a reader must invert. The permutation is
+
+```text
+transpose(i) = (i % 16) * 64 + FL_ORDER[(i / 16) % 8] * 8 + (i / 128) # i in 0..1024
+```
+
+(`fastlanes` 0.5.1 `transpose.rs` `transpose`). The forward transform is `output[i] = input[transpose(i)]`;
+the inverse (**untranspose**) is
+
+```text
+untranspose(input)[transpose(i)] = input[i] for i in 0..1024
+```
+
+(`transpose.rs` `Transpose`). The same permutation applied at bit granularity is used to untranspose Delta's
+validity bitmap; that mechanism is specified in [Validity](../encoding-format.md#validity) and is not
+restated here.
+
+(encoding-layout-BitPacked)=
+## `fastlanes.bitpacked` (BitPacked) — byte layout
+
+BitPacked stores an integer column at a reduced, uniform bit width `W`, using the FastLanes
+[bit-packing layout](#fastlanes-packing-order). Values too wide to fit in `W` bits (and, for signed
+columns, any that would not round-trip) are carried verbatim as **patches**.
+
+**Wire ID:** `fastlanes.bitpacked` (`bitpacking/vtable/mod.rs` `BitPacked::id`). **Logical dtype:** any integer
+primitive (`i8`…`i64`, `u8`…`u64`).
+
+**Buffers:** exactly 1 — buffer 0, named `packed` (`bitpacking/vtable/mod.rs` `BitPacked::buffer_name`). Its length is
+`ceil((n + offset) / 1024) * 128 * W` bytes (`bitpacking/array/mod.rs` `BitPackedData::validate`) — one `128·W`-byte
+[block](#fastlanes-packing-order) per 1024 padded values. `W = 0` ⇒ empty buffer.
+
+### Metadata (`BitPackedMetadata`, Protobuf)
+
+| Field | Protobuf type / tag | Meaning |
+|-------|---------------------|---------|
+| `bit_width` | `uint32`, tag 1 | The pack width `W` (`0 ≤ W ≤ 64`; always `< T_BITS` for a genuinely packed column). Stored as `u32`, narrowed to `u8` (`bitpacking/vtable/mod.rs` `BitPacked::deserialize`). |
+| `offset` | `uint32`, tag 2 | Physical start position within the **first** block, `0 ≤ offset < 1024` (`bitpacking/array/mod.rs` `BitPackedData.offset`, `BitPackedData::try_new`). Applies to the `packed` buffer **only** — see below. |
+| `patches` | `PatchesMetadata`, tag 3, optional | Present iff the column has exceptions. Describes the patch children ([shared patch layout](#fastlanes-patch-metadata)). |
+
+### Child slots
+
+Slots are positional; the validity slot's index depends on whether patches (and patch chunk-offsets)
+are present (`bitpacking/vtable/mod.rs` `BitPacked::deserialize`, `bitpacking/array/mod.rs` `BitPackedSlots`):
+
+| Idx | Name | Present when | Logical dtype |
+|-----|------|--------------|---------------|
+| 0 | `patch_indices` | `patches` set | non-nullable unsigned int (ptype from `PatchesMetadata`) |
+| 1 | `patch_values` | `patches` set | the node's own integer dtype (the true exception values; **all-valid**) |
+| 2 | `patch_chunk_offsets` | `patches` set **and** `chunk_offsets_ptype` present | non-nullable unsigned int (O(1) lookup accelerator) |
+| *k* | `validity_child` | **only when the array stores per-position validity** — omitted for `NonNullable`/`AllValid` per the [Validity](../encoding-format.md#validity) contract | non-nullable `Bool`; when present it is the **last** slot, at index `k = 0` (no patches), `2` (patches, no chunk offsets), or `3` (patches + chunk offsets) |
+
+`patch_values` is decoded recursively like any node; it holds the real values and is never bit-packed
+against `W`.
+
+### The `offset` (most common mistake)
+
+`offset` is a physical cursor into the first packed block, produced by slicing: a slice of `[a..b]`
+sets `offset = (old_offset + a) % 1024` and keeps only the covering blocks (`bitpacking/compute/slice.rs` `slice_bitpacked`).
+It is applied to the **packed buffer only** and is **never** applied to the validity slot, which stays
+row-aligned. Over-applying it to validity corrupts every null after the slice — see the warning in
+[Validity](../encoding-format.md#validity).
+
+### Decode
+
+Given `packed`, `W`, `offset`, `n`, and optional patches:
+
+1. **Unpack blocks.** `num_blocks = ceil((offset + n) / 1024)`. Unpack each block into 1024 logical
+ (row-order) values per [the bit-packing layout](#fastlanes-packing-order), concatenating into a
+ padded buffer of `num_blocks · 1024` values (`bitpacking/array/unpack_iter.rs`,
+ `bitpacking/array/bitpack_decompress.rs` `unpack_into_primitive_builder`).
+2. **Take the window.** Keep logical positions `offset .. offset + n`.
+3. **Reinterpret** the unsigned block values as the node's integer ptype (identity on the bits).
+4. **Apply patches** (if present). For each patch `j` in `0 .. len`, overwrite
+ ```text
+ value[ patch_indices[j] − patch_offset ] = patch_values[j]
+ ```
+ where `patch_offset` is the patches' **own** `offset` field ([shared patch layout](#fastlanes-patch-metadata)) —
+ distinct from the block `offset` above (`bitpacking/array/bitpack_decompress.rs` `apply_patches_to_uninit_range`). A patch
+ position holds a meaningless unpacked placeholder until this step overwrites it.
+5. **Validity** is a stored, row-aligned slot; see [Validity](../encoding-format.md#validity).
+
+**Example (no patches, no slice).** `W = 3`, `offset = 0`, `n = 4`, a single block whose row-order
+values are `[5, 0, 1, 7, 0, …]` decodes to `[5, 0, 1, 7]`.
+
+(encoding-layout-FoR)=
+## `fastlanes.for` (FoR) — byte layout
+
+Frame-of-Reference stores each value as its offset from a single **reference** integer (typically the
+column minimum): `value[i] = encoded[i] + reference`. Subtracting a common base shrinks the magnitudes
+so the `encoded` child bit-packs to a narrow width.
+
+**Wire ID:** `fastlanes.for` (`for/vtable/mod.rs` `FoR::id`). **Logical dtype:** any integer primitive.
+
+**Buffers:** none (`nbuffers = 0`, `for/vtable/mod.rs` `FoR::nbuffers`).
+
+### Metadata (raw `ScalarValue`, Protobuf)
+
+FoR's metadata is **not** a wrapper message. It is the reference scalar's **value** alone, serialized
+as `ScalarValue` protobuf bytes **without** its dtype (`for/vtable/mod.rs` `FoR::serialize`). On read, decode
+the bytes as a `ScalarValue` against the node's own `DType` to recover the reference
+(`for/vtable/mod.rs` `FoR::deserialize`). The reference is non-null and shares the node's integer type
+(`for/array/mod.rs` `FoRData::try_new`).
+
+### Child slots
+
+| Idx | Name | Present when | Logical dtype | Length |
+|-----|------|--------------|---------------|--------|
+| 0 | `encoded` | always | the node's own integer dtype | `n` |
+
+`encoded` is decoded recursively; it is commonly (but not necessarily) `fastlanes.bitpacked`
+(`for/vtable/mod.rs` `FoR::deserialize`).
+
+### Decode
+
+1. Decode `encoded` to an integer array of length `n`, and decode the `reference` scalar from metadata.
+2. For each position `i`: `value[i] = encoded[i] wrapping_add reference` — modular (wrapping) integer
+ addition (`for/array/for_decompress.rs` `decompress`, `decompress_primitive`). If `reference == 0`, `encoded` is the
+ result unchanged.
+3. **Validity** is delegated to the `encoded` child; see [Validity](../encoding-format.md#validity).
+
+Wrapping addition means the reference may be added correctly even when the true values wrap the type;
+the reference/encoded split is purely arithmetic.
+
+**Example.** `reference = 100`, `encoded = [0, 5, 3]` ⇒ `[100, 105, 103]`.
+
+(encoding-layout-Delta)=
+## `fastlanes.delta` (Delta) — byte layout
+
+Delta stores per-lane running differences over a **transposed** block, so each lane holds a contiguous
+run of the original sequence and the deltas within a lane are small. Reconstructing a block is a
+per-lane prefix sum followed by an [untranspose](#fastlanes-transpose).
+
+**Wire ID:** `fastlanes.delta` (`delta/vtable/mod.rs` `Delta::id`). **Logical dtype:** any integer primitive.
+
+**Buffers:** none (`nbuffers = 0`, `delta/vtable/mod.rs` `Delta::nbuffers`).
+
+### Metadata (`DeltaMetadata`, Protobuf)
+
+| Field | Protobuf type / tag | Meaning |
+|-------|---------------------|---------|
+| `deltas_len` | `uint64`, tag 1 | Length of the `deltas` child; always a multiple of 1024 (`delta/vtable/mod.rs` `validate_parts`). |
+| `offset` | `uint32`, tag 2 | Physical start position within the first block, `0 ≤ offset < 1024` (`delta/array/mod.rs` `DeltaData::try_new`). |
+
+### Child slots
+
+Both children share the node's integer type; the node's nullability comes from `deltas`
+(`delta/vtable/mod.rs` `Delta::try_new`). Let `num_blocks = deltas_len / 1024`.
+
+| Idx | Name | Logical dtype | Length |
+|-----|------|---------------|--------|
+| 0 | `bases` | the node's integer dtype (node nullability; always all-valid — validity lives in `deltas`) | `num_blocks · LANES` (`delta/vtable/mod.rs` `Delta::deserialize`) |
+| 1 | `deltas` | the node's integer dtype (carries validity) | `deltas_len` |
+
+`deltas` holds the per-lane differences in [bit-packing (`pack_index`) order](#fastlanes-packing-order),
+padded to whole 1024-blocks; `bases[c·LANES + lane]` is the seed value for lane `lane` of block `c`
+(the first transposed element of that lane; the corresponding stored delta is `0`). Both are decoded
+recursively (`deltas` is commonly `fastlanes.bitpacked`).
+
+### Decode
+
+Reinterpret `bases` and `deltas` as the **unsigned** type `T` for the arithmetic, then per block
+`c` in `0 .. num_blocks` (`delta/array/delta_decompress.rs` `decompress_primitive`; `fastlanes` 0.5.1
+`delta.rs` `Delta::undelta`):
+
+```text
+# 1. per-lane prefix sum, in pack_index order (produces a transposed block)
+for lane in 0 .. LANES:
+ acc = bases[c*LANES + lane]
+ for r in 0 .. T_BITS:
+ idx = pack_index(r, lane) # see FastLanes fundamentals
+ acc = acc wrapping_add deltas[c*1024 + idx]
+ trans[idx] = acc
+
+# 2. untranspose the block back into row order
+for i in 0 .. 1024:
+ out[c*1024 + transpose(i)] = trans[i] # see FastLanes fundamentals
+```
+
+Then reinterpret `out` back to the node's (possibly signed) ptype and **slice**
+`out[offset .. offset + n]` (`delta/array/delta_decompress.rs` `delta_decompress`). Skipping either the
+untranspose or the offset slice yields scrambled or shifted values.
+
+**Validity** is delegated to `deltas`, then untransposed and sliced `offset .. offset + n`; see
+[Validity](../encoding-format.md#validity).
+
+:::{note}
+A worked numeric example spans a full 1024-value block, so it is impractical to tabulate here. The two
+permutation functions (`pack_index`, `transpose`) plus the per-lane `wrapping_add` recurrence above are
+a complete, self-contained reconstruction. As a spot check of `pack_index` for `T = u16`
+(`T_BITS = 16`, `LANES = 64`): `pack_index(0, lane) = lane`, and `pack_index(1, 0) = 128`.
+:::
+
+(encoding-layout-RLE)=
+## `fastlanes.rle` (RLE) — byte layout
+
+FastLanes run-length encoding is **per-block**: within each 1024-value block, it stores that block's
+run values and, for each of the 1024 rows, a **chunk-local index** into those run values. (This is
+distinct from `vortex.runend`, a combine encoding on the [Dict/RunEnd/Sparse page](dict-runend-sparse.md).)
+
+**Wire ID:** `fastlanes.rle` (`rle/vtable/mod.rs` `RLE::id`). **Logical dtype:** integer primitive; the ptype
+comes from `values`, the nullability from `indices` (`rle/vtable/mod.rs` `RLE::try_new`).
+
+**Buffers:** none (`nbuffers = 0`, `rle/vtable/mod.rs` `RLE::nbuffers`).
+
+### Metadata (`RLEMetadata`, Protobuf)
+
+| Field | Protobuf type / tag | Meaning |
+|-------|---------------------|---------|
+| `values_len` | `uint64`, tag 1 | Length of the `values` child (total run values across all blocks). |
+| `indices_len` | `uint64`, tag 2 | Length of the `indices` child; a multiple of 1024 (`rle/vtable/mod.rs` `validate_parts`). |
+| `indices_ptype` | `PType` enum, tag 3 | Ptype of `indices` (`u8` or `u16`). |
+| `values_idx_offsets_len` | `uint64`, tag 4 | Length of the `values_idx_offsets` child = `indices_len / 1024` (one per block). |
+| `values_idx_offsets_ptype` | `PType` enum, tag 5 | Ptype of `values_idx_offsets` (unsigned int). |
+| `offset` | `uint64`, tag 6, default 0 | Physical start position within the first block, `0 ≤ offset < 1024` (`rle/array/mod.rs` `RLEData::try_new`). |
+
+### Child slots
+
+| Idx | Name | Logical dtype | Length |
+|-----|------|---------------|--------|
+| 0 | `values` | `Primitive(ptype, NonNullable)` — the run values | `values_len` |
+| 1 | `indices` | `Primitive(u8 or u16, node nullability)` — per-row chunk-local run index (**carries validity**) | `indices_len` |
+| 2 | `values_idx_offsets` | non-nullable unsigned int — start index into `values` for each block | `indices_len / 1024` |
+
+The `values_idx_offsets` entries are **absolute** offsets into `values`; a reader rebases them by
+subtracting `values_idx_offsets[0]`, because slicing may drop leading blocks and re-slice `values`
+without rewriting the offsets (`rle/array/mod.rs` `RLEArrayExt::values_idx_offset`, `rle/kernel.rs` `RLE::slice`).
+
+### Decode
+
+Decode `values` → `V`, `indices` → `I` (u8/u16), `values_idx_offsets` → `O`. If `n == 0` the array
+is empty (`O`, `I`, and `V` are all empty) — return the empty array **without** reading `O[0]`.
+Otherwise let `num_blocks = ceil((offset + n) / 1024)` and `base = O[0]`
+(`rle/array/rle_decompress.rs` `rle_decode_typed`):
+
+```text
+for b in 0 .. num_blocks:
+ start = O[b] - base
+ end = (b + 1 < num_blocks) ? (O[b+1] - base) : V.len() # exclusive; run values for block b
+ for row in 0 .. 1024:
+ out[b*1024 + row] = V[start + I[b*1024 + row]]
+# then take the live window:
+result = out[offset .. offset + n]
+```
+
+Each block's run values are `V[start .. end]`; row `row` of block `b` picks run value
+`V[start + I[b*1024 + row]]`.
+
+**Robustness at null positions.** Where `indices` is null, its stored index value is a don't-care and
+may be out of range (a byproduct of further-compressing the indices). Because RLE's validity is
+sourced from `indices`, such rows are null regardless of the value read; a defensive reader clamps the
+index into `0 .. (end − start)` before the lookup to avoid an out-of-bounds access
+(`rle/array/rle_decompress.rs` `rle_decode_typed`).
+
+**Validity** is delegated to `indices` sliced `offset .. offset + n` *before* taking its validity; see
+[Validity](../encoding-format.md#validity).
+
+**Example (single block).** `V = [10, 20, 30]`, `O = [0]`, `offset = 0`, `n = 5`, and
+`I[0..5] = [0, 0, 1, 2, 2]` ⇒ `[10, 10, 20, 30, 30]`.
+
+(fastlanes-patch-metadata)=
+## Shared patch layout
+
+`fastlanes.bitpacked` reuses the workspace-wide `PatchesMetadata` message (`patches.rs` `PatchesMetadata`) and its
+three patch children.
+
+| Field | Protobuf type / tag | Meaning |
+|-------|---------------------|---------|
+| `len` | `uint64`, tag 1 | Number of patches `P` = length of both `patch_indices` and `patch_values`. |
+| `offset` | `uint64`, tag 2 | Absolute offset subtracted from every stored index; `0` for an unsliced array. Array-local position = `patch_indices[j] − offset`. |
+| `indices_ptype` | `PType` enum, tag 3 | Unsigned-int ptype of `patch_indices`. |
+| `chunk_offsets_len` | `uint64`, tag 4, optional | Length of `patch_chunk_offsets`, when present. |
+| `chunk_offsets_ptype` | `PType` enum, tag 5, optional | Ptype of `patch_chunk_offsets`; its presence signals child slot 2 exists. |
+| `offset_within_chunk` | `uint64`, tag 6, optional | Rows sliced off the front of the first chunk (slice bookkeeping). |
+
+`patch_indices` is a strictly-ascending non-nullable unsigned-int array of length `P`; `patch_values`
+(length `P`, non-null) holds the true value for each. `patch_chunk_offsets` is an **optional O(1)
+lookup index** (one entry per 1024-row chunk, `PATCH_CHUNK_SIZE = 1024`, `patches.rs` `PATCH_CHUNK_SIZE`); a reader
+that scans all `P` patches linearly may ignore it.
diff --git a/docs/specification/encoding-format/misc.md b/docs/specification/encoding-format/misc.md
new file mode 100644
index 00000000000..994ef4254ad
--- /dev/null
+++ b/docs/specification/encoding-format/misc.md
@@ -0,0 +1,426 @@
+# Miscellaneous Encodings — Byte Layout
+
+The remaining stable encodings that do not fall into the other families: string compression
+(`FSST`), byte-wide booleans, split-representation encodings for temporal and decimal values, integer
+transforms, and the external-codec encodings. This page is the per-encoding byte-layout reference for
+that group, one section per encoding.
+
+It is part of the [Encoding Format](../encoding-format.md) specification; null handling for every
+encoding here follows the cross-cutting [Validity](../encoding-format.md#validity) contract on that
+page and is not restated per section.
+
+Encodings covered on this page: `FSST`, `ByteBool`, `DateTimeParts`, `DecimalByteParts`, `ZigZag`,
+`Pco`, `Zstd`, `ParquetVariant`.
+
+```{contents}
+:local:
+:depth: 1
+```
+
+Throughout, `n` is the array's logical length, a **child** is a nested Vortex array node decoded
+recursively through the same encoding dispatch, and a **buffer** is a raw byte buffer. Children are
+carried in named **slots**; on the wire an optional slot (notably the validity slot) may be omitted,
+so the number of serialised children is smaller than the slot count. Each section states both the
+slot layout and the exact on-wire child ordering a reader must map back onto it. All multi-byte
+integers are little-endian. Metadata is a Protocol Buffers message; the `tag` column gives the
+protobuf field number.
+
+(encoding-layout-FSST)=
+## `vortex.fsst` — Byte layout
+
+FSST (Fast Static Symbol Table) string compression, per the FSST scheme (Boncz, Neumann, Leis,
+*VLDB 2020*). Each row's bytes are compressed to a sequence of 8-bit **codes**; a code either names
+an entry in a shared **symbol table** (expanding to 1–8 bytes) or is the **escape code**, which emits
+the following raw byte verbatim.
+
+- **Wire ID:** `vortex.fsst` (`array.rs` `FSST::id`).
+- **DType:** `Utf8` or `Binary` (either nullability, enforced in `array.rs` `FSSTData::validate_parts`).
+
+This section specifies the **current** 3-buffer format. A legacy 2-buffer form also exists and is
+covered under *Compatibility* below.
+
+**Buffers** (3, `array.rs` `FSST::nbuffers`; names from `array.rs` `FSST::buffer_name`):
+
+| Index | Name | Contents |
+|-------|------|----------|
+| 0 | `symbols` | The symbol table: 8 bytes per symbol, `n_sym` symbols (`n_sym ≤ 255`), so `8 · n_sym` bytes. Symbol `s` occupies bytes `[8·s, 8·s+8)`; its content is the first `lengths[s]` of those bytes, in order. (`n_sym ≤ 255` checked in `array.rs` `FSSTData::validate_parts`.) |
+| 1 | `symbol_lengths` | One `u8` per symbol (`n_sym` bytes). `lengths[s]` ∈ `1..=8` is the number of content bytes of symbol `s` (`array.rs` `FSSTData::validate_symbol_lengths`). |
+| 2 | `compressed_codes` | The concatenated code bytes for all rows. Row `i` occupies `compressed_codes[codes_offsets[i] .. codes_offsets[i+1]]`. |
+
+**Metadata** (`FSSTMetadata`, `array.rs` `FSSTMetadata`):
+
+| Field | Tag | Type | Meaning |
+|-------|-----|------|---------|
+| `uncompressed_lengths_ptype` | 1 | `PType` enum | Integer type of the `uncompressed_lengths` child. |
+| `codes_offsets_ptype` | 2 | `PType` enum | Integer type of the `codes_offsets` child. |
+
+**Child slots** (2–3; wire child order equals slot order; `array.rs` `FSST::deserialize`, slot names `array.rs` `SLOT_NAMES`):
+
+| Slot | Name | Role | DType / length |
+|------|------|------|----------------|
+| 0 | `uncompressed_lengths` | Decoded byte length of each row (used to size/split the decoded heap). | Non-nullable integer (`uncompressed_lengths_ptype`), length `n`. |
+| 1 | `codes_offsets` | VarBin-style offsets into `compressed_codes`. | Non-nullable integer (`codes_offsets_ptype`), length `n + 1` (`array.rs` `FSSTData::validate_parts`). |
+| 2 | `codes_validity` | Validity slot. | Non-nullable `Bool`, length `n`. Omitted when the array carries no per-position nulls. |
+
+**Decode** (row `i`, when valid):
+
+1. Take the compressed byte range `R = compressed_codes[codes_offsets[i] .. codes_offsets[i+1]]`.
+2. Walk `R` with a cursor `p` starting at 0, emitting to an output buffer:
+ - Let `c = R[p]`. If `c == 255` (the escape code, `fsst-rs` 0.5.11 `lib.rs` `ESCAPE_CODE`), emit `R[p+1]` verbatim and advance `p += 2`.
+ - Otherwise emit the first `symbol_lengths[c]` bytes of symbol `c` (bytes `symbols[8·c .. 8·c + symbol_lengths[c]]`, in order) and advance `p += 1`.
+ - Stop when `p` reaches the end of `R`.
+3. The emitted byte count equals `uncompressed_lengths[i]`. The emitted bytes are row `i`'s value (interpreted as UTF-8 for `Utf8`, raw for `Binary`) (`canonical.rs` `fsst_decode_views`).
+
+**Compatibility (legacy 2-buffer form).** Older files store buffers `[symbols, symbol_lengths]` and
+children `[codes, uncompressed_lengths]`, where `codes` is a full `VarBin` array (its bytes are the
+`compressed_codes` heap, its offsets are `codes_offsets`, and its validity is the codes validity).
+A reader recognises this form by the buffer count being 2 rather than 3; decode is identical after
+lifting the bytes/offsets/validity out of the `codes` child (`array.rs` `FSST::deserialize`, `FSST::deserialize_legacy`).
+
+**Validity.** Stored slot (`codes_validity`), row-aligned (`array.rs` `FSST::validity`). See
+[Validity](../encoding-format.md#validity).
+
+(encoding-layout-ByteBool)=
+## `vortex.bytebool` — Byte layout
+
+A boolean array stored as **one byte per value** (rather than a packed bitmap). Trades space for
+branch-free, byte-addressable access.
+
+- **Wire ID:** `vortex.bytebool` (`array.rs` `ByteBool::id`).
+- **DType:** `Bool` (either nullability).
+
+**Buffers** (1, `array.rs` `ByteBool::nbuffers`; name from `array.rs` `ByteBool::buffer_name`):
+
+| Index | Name | Contents |
+|-------|------|----------|
+| 0 | `values` | Exactly `n` bytes, one per row (`array.rs` `ByteBoolData::validate`). Writers emit `0x00` for `false` and `0x01` for `true` (`array.rs` `ByteBool::from_vec`). |
+
+**Metadata:** empty (a reader must reject non-empty metadata, `array.rs` `ByteBool::deserialize`).
+
+**Child slots** (0–1, `array.rs` `ByteBool::deserialize`; slot name `array.rs` `SLOT_NAMES`):
+
+| Slot | Name | Role | DType / length |
+|------|------|------|----------------|
+| 0 | `validity` | Validity slot. | Non-nullable `Bool`, length `n`. Omitted when the array carries no per-position nulls. |
+
+**Decode:** row `i` (when valid) is `values[i] != 0` — the canonical decoder treats byte `0` as
+`false` and any non-zero byte as `true` (`array.rs` `ByteBoolData::truthy_bytes`).
+
+**Validity.** Stored slot, row-aligned (`array.rs` `ByteBool::validity`). See [Validity](../encoding-format.md#validity).
+
+(encoding-layout-DateTimeParts)=
+## `vortex.datetimeparts` — Byte layout
+
+Splits a timestamp column into three integer children — whole **days**, whole **seconds within the
+day**, and a **sub-second remainder** — which compress far better independently than the combined
+64-bit instant. Decoding recombines them into a single instant in the timestamp's time unit.
+
+- **Wire ID:** `vortex.datetimeparts` (`array.rs` `DateTimeParts::id`).
+- **DType:** an `Extension` type whose `id` is `vortex.timestamp` and whose metadata is the
+ **Timestamp extension metadata**, laid out as `[unit_tag: u8][tz_len: u16 LE][tz: UTF-8]`
+ (`vortex-array/src/extension/datetime/timestamp.rs` `serialize_metadata`): a one-byte `unit_tag`,
+ then a little-endian `u16` timezone-name length `tz_len`, then `tz_len` UTF-8 bytes of timezone
+ name (`tz_len = 0` means no timezone). The `unit_tag` selects the time unit — `0` nanoseconds,
+ `1` microseconds, `2` milliseconds, `3` seconds, `4` days (`vortex-array/src/extension/datetime/unit.rs`
+ `TimeUnit`). DateTimeParts uses one of ns/µs/ms/s; unit `Days` is not decodable here (a
+ day-resolution timestamp does not split into sub-day parts). The Timestamp extension's **storage
+ dtype is `Primitive(I64, )`** (`vortex-array/src/extension/datetime/timestamp.rs`
+ `validate_dtype`) — the decoded instant is an `i64` in the selected unit.
+
+**Buffers:** none (`array.rs` `DateTimeParts::nbuffers`).
+
+**Metadata** (`DateTimePartsMetadata`, `array.rs` `DateTimePartsMetadata`):
+
+| Field | Tag | Type | Meaning |
+|-------|-----|------|---------|
+| `days_ptype` | 1 | `PType` enum | Integer type of the `days` child. |
+| `seconds_ptype` | 2 | `PType` enum | Integer type of the `seconds` child. |
+| `subseconds_ptype` | 3 | `PType` enum | Integer type of the `subseconds` child. |
+
+**Child slots** (3; wire child order equals slot order; `array.rs` `DateTimeParts::deserialize`, `DateTimePartsData::validate`):
+
+| Slot | Name | Role | DType / length |
+|------|------|------|----------------|
+| 0 | `days` | Whole days since the epoch. | Integer (`days_ptype`), nullability equal to the array's; length `n`. |
+| 1 | `seconds` | Whole seconds within the day. | Non-nullable integer (`seconds_ptype`), length `n`. |
+| 2 | `subseconds` | Sub-second remainder, already expressed in the timestamp's time unit. | Non-nullable integer (`subseconds_ptype`), length `n`. |
+
+**Decode** (row `i`): let `divisor` be the number of time-unit ticks per second —
+`1_000_000_000` (ns), `1_000_000` (µs), `1_000` (ms), or `1` (s) (`canonical.rs`
+`decode_to_temporal`). The instant, in the timestamp's time unit, is
+
+```text
+value[i] = days[i] · 86_400 · divisor + seconds[i] · divisor + subseconds[i]
+```
+
+(`seconds` is scaled by `divisor`; `subseconds` is added unscaled because it is already in the
+target unit). Wrap `value` as a `Timestamp` with the extension's `unit` and `tz`. The `seconds` or
+`subseconds` child may be a `Constant` array (a common optimisation, e.g. day-granularity data has a
+constant `0`); decode it as any other array.
+
+**Validity.** Delegates to the `days` child (`array.rs` `DateTimeParts::validity_child`). See [Validity](../encoding-format.md#validity).
+
+(encoding-layout-DecimalByteParts)=
+## `vortex.decimal_byte_parts` — Byte layout
+
+Stores a decimal column as a small number of signed-integer "part" columns, most-significant part
+first, so each part compresses independently. The current format emits a **single part** (`msp`, the
+most-significant part) that holds the whole unscaled integer; the metadata reserves room for lower
+parts that are not yet produced.
+
+- **Wire ID:** `vortex.decimal_byte_parts` (`decimal_byte_parts/mod.rs` `DecimalByteParts::id`).
+- **DType:** `Decimal(precision, scale)` (either nullability).
+
+**Buffers:** none (`decimal_byte_parts/mod.rs` `DecimalByteParts::nbuffers`).
+
+**Metadata** (`DecimalBytesPartsMetadata`, `decimal_byte_parts/mod.rs` `DecimalBytesPartsMetadata`):
+
+| Field | Tag | Type | Meaning |
+|-------|-----|------|---------|
+| `zeroth_child_ptype` | 1 | `PType` enum | Signed-integer type of the `msp` child. |
+| `lower_part_count` | 2 | `uint32` | Number of additional lower-part children. **Currently always 0**; a reader must reject any other value (`decimal_byte_parts/mod.rs` `DecimalByteParts::deserialize`). |
+
+**Child slots** (1, `decimal_byte_parts/mod.rs` `DecimalByteParts::deserialize`; slot name `decimal_byte_parts/mod.rs` `SLOT_NAMES`):
+
+| Slot | Name | Role | DType / length |
+|------|------|------|----------------|
+| 0 | `msp` | Most-significant part; holds the entire unscaled decimal integer. | Signed integer (`zeroth_child_ptype`, i.e. `i8`/`i16`/`i32`/`i64`), nullability equal to the array's; length `n` (`decimal_byte_parts/mod.rs` `DecimalBytePartsData::validate`). |
+
+**Decode** (row `i`, when valid): the unscaled integer is `msp[i]` (sign-preserved). The logical
+decimal value is `msp[i] · 10^(−scale)`, with `precision`/`scale` taken from the `Decimal` dtype.
+Materialised canonically, this is a `Decimal` array whose storage integer buffer is `msp` and whose
+decimal type is the array's dtype (`decimal_byte_parts/mod.rs` `to_canonical_decimal`).
+
+*(Reserved multi-part design, for reference: with `lower_part_count > 0`, `msp` would hold the
+high-order bits and the lower-part children the successively less-significant bits, concatenated to
+form the full unscaled integer — e.g. an `i128` split as `msp = bits[127:64]`, `lower[0] =
+bits[63:0]`. No writer emits this today.)*
+
+**Validity.** Delegates to the `msp` child (child 0) (`decimal_byte_parts/mod.rs` `DecimalByteParts::validity_child`). See
+[Validity](../encoding-format.md#validity).
+
+(encoding-layout-ZigZag)=
+## `vortex.zigzag` — Byte layout
+
+ZigZag maps signed integers to unsigned so that small-magnitude values (of either sign) become
+small unsigned values — friendlier to downstream unsigned-friendly codecs. The array holds a single
+unsigned child; decoding applies the ZigZag inverse.
+
+- **Wire ID:** `vortex.zigzag` (`array.rs` `ZigZag::id`).
+- **DType:** a signed-integer `Primitive` (`I8`/`I16`/`I32`/`I64`), either nullability. The child is
+ the unsigned integer of the **same bit width** and the same nullability (`array.rs`
+ `ZigZag::deserialize`, `ZigZagData::dtype_from_encoded_dtype`).
+
+**Buffers:** none (`array.rs` `ZigZag::nbuffers`).
+
+**Metadata:** empty (a reader must reject non-empty metadata, `array.rs` `ZigZag::deserialize`).
+
+**Child slots** (1, `array.rs` `ZigZag::deserialize`; slot name `array.rs` `SLOT_NAMES`):
+
+| Slot | Name | Role | DType / length |
+|------|------|------|----------------|
+| 0 | `encoded` | ZigZag-encoded values. | Unsigned integer of the same bit width as the array's signed type, same nullability; length `n`. |
+
+**Decode** (row `i`, when valid): with `u = encoded[i]` a `W`-bit unsigned value, the signed value is
+
+```text
+signed = (u >> 1) ^ (0 - (u & 1))
+```
+
+— a **logical** right shift of `u`, XOR-ed with the all-ones pattern when `u` is odd (i.e. two's
+complement negation of the low bit). Equivalently: if `u` is even, `signed = u / 2`; if `u` is odd,
+`signed = −(u + 1) / 2`. The result type is the signed integer of the same width
+(`compress.rs` `zigzag_decode_primitive`; `zigzag` 0.1.0 `lib.rs` `ZigZag::decode`).
+
+**Validity.** Delegates to the `encoded` child (`array.rs` `ZigZag::validity_child`). See [Validity](../encoding-format.md#validity).
+
+(encoding-layout-Pco)=
+## `vortex.pco` — Byte layout
+
+Wraps the [pcodec](https://github.com/mwlon/pcodec) (pco) numeric codec. Values are compressed with
+pco's *wrapped* format into one or more **chunks**, each split into **pages**; Vortex stores the pco
+components as buffers and the framing (page value-counts) in metadata. Crucially, **only the valid
+(non-null) values are compressed** — nulls are dropped before compression and re-inserted by
+scattering on decode.
+
+The byte layout of the pco components themselves — the wrapped-format `header`, each chunk's
+`ChunkMeta`, and the page bytes — is defined **only** by the pcodec library and is not restated here;
+it is the one encoding on these pages not fully decodable from the material referenced elsewhere in
+this spec. Vortex pins `pco = "1.0.1"`; decode these components per the
+[pcodec wrapped-format documentation](https://github.com/mwlon/pcodec/blob/main/docs/format.md) at
+that version — just as `vortex.zstd` defers to [RFC 8878](https://www.rfc-editor.org/rfc/rfc8878) and
+`vortex.parquet.variant` defers to the versioned Parquet Variant spec.
+
+- **Wire ID:** `vortex.pco` (`array.rs` `Pco::id`).
+- **DType:** a `Primitive` pco supports: `F16`/`F32`/`F64`, `I16`/`I32`/`I64`, `U16`/`U32`/`U64`
+ (either nullability; `array.rs` `number_type_from_ptype`). Note the pco stream is keyed by the
+ array's ptype, not stored in metadata.
+
+**Buffers** (dynamic): the `n_chunks` **chunk-meta** buffers first (one per chunk, in order),
+followed by all **page** buffers in order (chunk 0's pages, then chunk 1's pages, …). Total buffer
+count = `n_chunks + Σ pages_per_chunk` (`array.rs` `Pco::nbuffers`, `Pco::buffer_name`, `Pco::deserialize`).
+
+| Range | Name | Contents |
+|-------|------|----------|
+| `0 .. n_chunks` | `chunk_meta_{k}` | pco `ChunkMeta` bytes for chunk `k`. |
+| `n_chunks ..` | `page_{j}` | pco page bytes, flattened across chunks in order. |
+
+**Metadata** (`PcoMetadata`, `lib.rs` `PcoMetadata`):
+
+| Field | Tag | Type | Meaning |
+|-------|-----|------|---------|
+| `header` | 1 | `bytes` | The pco wrapped-format file header (needed to construct the decompressor). |
+| `chunks` | 2 | repeated `PcoChunkInfo` | One entry per chunk. Each `PcoChunkInfo.pages` (tag 1) is a repeated `PcoPageInfo`, and each `PcoPageInfo.n_values` (tag 1, `uint32`) is the count of values encoded in that page. (`lib.rs` `PcoChunkInfo`, `PcoPageInfo`.) |
+
+`n_chunks = len(chunks)`, and chunk `k` has `len(chunks[k].pages)` page buffers
+(`array.rs` `Pco::deserialize`).
+
+**Child slots** (0–1, `array.rs` `Pco::deserialize`; slot name `array.rs` `SLOT_NAMES`):
+
+| Slot | Name | Role | DType / length |
+|------|------|------|----------------|
+| 0 | `validity` | Validity slot, stored **unsliced**. | Non-nullable `Bool`, length `n`. Omitted when the array carries no per-position nulls. |
+
+**Decode:**
+
+1. Decode validity (length `n`) per the [Validity](../encoding-format.md#validity) contract; let `V`
+ be the number of valid positions.
+2. Construct the pco decompressor from `header`. For each chunk `k` in order, initialise a chunk
+ decompressor from `chunk_meta_{k}`; for each of its pages `j`, decode exactly
+ `chunks[k].pages[j].n_values` numbers from the corresponding page buffer. Concatenating across all
+ chunks/pages yields exactly `V` decoded numbers (the valid values, in row order).
+3. **Scatter** into an output of length `n`: the `m`-th decoded number goes to the `m`-th valid
+ position (per the validity mask); null positions receive a placeholder (e.g. zero)
+ (`array.rs` `PcoData::decompress`, `PcoData::decompress_values_typed`; only valid values are
+ compressed per `array.rs` `collect_valid`).
+
+**Slice metadata & validity.** `Pco` (with `Zstd`) is one of the two encodings whose validity slot is
+stored **unsliced** and paired with a `slice_start .. slice_stop` range, handled by the
+[Validity](../encoding-format.md#validity) offset rule. That slice range is **in-memory state used
+for lazy slicing and is not part of the serialised metadata**: a deserialised array is always
+unsliced (`slice_start = 0`, `slice_stop = n`), so for a file reader the validity slot has length `n`
+and the slice is the identity (`array.rs` `Pco::validity`, `PcoData::new`).
+
+(encoding-layout-Zstd)=
+## `vortex.zstd` — Byte layout
+
+Wraps the Zstandard codec ([RFC 8878](https://www.rfc-editor.org/rfc/rfc8878)). Values are
+compressed into one or more independently decompressible **frames**, optionally sharing a trained
+**dictionary**; the frame lengths live in metadata. Like `Pco`, **only valid values are
+compressed**, and nulls are re-inserted by scattering on decode. This mirrors the `Pco` layout.
+
+- **Wire ID:** `vortex.zstd` (`array.rs` `Zstd::id`).
+- **DType:** `Primitive`, `Binary`, or `Utf8` (either nullability; enforced in `array.rs`
+ `ZstdData::validate`).
+
+:::{note}
+Do not confuse this with `vortex.zstd_buffers`, a separate `unstable_encodings`-gated encoding that
+is out of scope for the stable spec.
+:::
+
+**Buffers** (dynamic): if a dictionary is present (`dictionary_size > 0`), buffer 0 is the
+`dictionary`; the remaining buffers are the `frame_{f}` buffers in order. If no dictionary, all
+buffers are frames (`array.rs` `Zstd::nbuffers`, `Zstd::buffer_name`, `Zstd::deserialize`).
+
+| Position | Name | Contents |
+|----------|------|----------|
+| 0 (only if `dictionary_size > 0`) | `dictionary` | The Zstd dictionary bytes. |
+| rest | `frame_{f}` | Zstd-compressed frame `f`, in order. |
+
+**Metadata** (`ZstdMetadata`, `lib.rs` `ZstdMetadata`):
+
+| Field | Tag | Type | Meaning |
+|-------|-----|------|---------|
+| `dictionary_size` | 1 | `uint32` | Byte length of the dictionary buffer, or `0` if none. |
+| `frames` | 2 | repeated `ZstdFrameMetadata` | One entry per frame. `ZstdFrameMetadata.uncompressed_size` (tag 1, `uint64`) is the frame's decompressed byte length; `ZstdFrameMetadata.n_values` (tag 2, `uint64`) is the number of values it encodes. (`lib.rs` `ZstdFrameMetadata`.) |
+
+`n_values` must be present in the current format. (The reader keeps a legacy fallback — `n_values ==
+0` ⇒ derive the count from `uncompressed_size / byte_width` — but current writers always set it;
+`array.rs` `ZstdData::decompress`.)
+
+**Child slots** (0–1, `array.rs` `Zstd::deserialize`; slot name `array.rs` `SLOT_NAMES`):
+
+| Slot | Name | Role | DType / length |
+|------|------|------|----------------|
+| 0 | `validity` | Validity slot, stored **unsliced**. | Non-nullable `Bool`, length `n`. Omitted when the array carries no per-position nulls. |
+
+**Decode:**
+
+1. Decode validity (length `n`); let `V` be the number of valid positions.
+2. Decompress each frame with Zstd (initialising the decompressor with the `dictionary` buffer when
+ present). Concatenating the frames' output in order yields the uncompressed value bytes for all
+ `V` valid values, in row order.
+3. Interpret the uncompressed bytes by dtype, then **scatter** into an output of length `n` (the
+ `m`-th decoded value to the `m`-th valid position; null positions get a placeholder):
+ - **`Primitive`:** the bytes are the raw little-endian values, `V · byte_width` bytes total
+ (`array.rs` `ZstdData::decompress`).
+ - **`Binary`/`Utf8`:** the bytes are length-prefixed — for each of the `V` valid values, a `u32`
+ little-endian byte length followed by exactly that many data bytes, concatenated in order (the
+ same length-prefixed layout Parquet uses) (`array.rs` `collect_valid_vbv`, `reconstruct_views`).
+
+**Slice metadata & validity.** As for `Pco`: the validity slot is stored **unsliced** with a
+`slice_start .. slice_stop` range applied per the [Validity](../encoding-format.md#validity) offset
+rule, and that range is in-memory-only — a deserialised array is unsliced (`slice_start = 0`,
+`slice_stop = n`), so a file reader sees a length-`n` validity slot and an identity slice
+(`array.rs` `Zstd::validity`, `ZstdData::new`).
+
+(encoding-layout-ParquetVariant)=
+## `vortex.parquet.variant` — Byte layout
+
+A lossless carrier for Arrow's canonical `arrow.parquet.variant` extension storage: semi-structured
+[Parquet Variant](https://github.com/apache/parquet-format/blob/master/VariantEncoding.md) values,
+with optional [shredding](https://github.com/apache/parquet-format/blob/master/VariantShredding.md).
+Vortex stores the canonical `metadata` / `value` / `typed_value` components as child arrays and
+carries their bytes verbatim; the binary interpretation of the `metadata` and `value` blobs is
+defined by the Parquet Variant spec, not by Vortex.
+
+- **Wire ID:** `vortex.parquet.variant` (`vtable.rs` `ParquetVariant::id`).
+- **DType:** `Variant(nullability)`.
+
+**Buffers:** none (`vtable.rs` `ParquetVariant::nbuffers`; a reader must reject any buffer,
+`vtable.rs` `ParquetVariant::deserialize`).
+
+**Metadata** (`ParquetVariantMetadataProto`, `vtable.rs` `ParquetVariantMetadataProto`):
+
+| Field | Tag | Type | Meaning |
+|-------|-----|------|---------|
+| `has_value` | 1 | `bool` | Whether the `value` child is present. |
+| `typed_value_dtype` | 2 | optional `DType` (proto) | The dtype of the `typed_value` child; present iff `typed_value` is present. |
+| `value_nullable` | 3 | `bool` | Whether the `value` child's dtype is nullable. |
+
+**Child slots** (4 slots; 1–4 serialised children; slot names `array.rs` `SLOT_NAMES`, dtype/length
+constraints `vtable.rs` `ParquetVariant::validate`):
+
+| Slot | Name | Role | DType / length |
+|------|------|------|----------------|
+| 0 | `validity` | Top-level row validity slot. | Non-nullable `Bool`, length `n`. Omitted when there are no per-position nulls. |
+| 1 | `metadata` | Parquet Variant metadata (field-name dictionary) per row. **Always present.** | Non-nullable `Binary`, length `n`. |
+| 2 | `value` | Unshredded Variant value bytes. Present iff `has_value`. | `Binary` (nullable = `value_nullable`), length `n`. |
+| 3 | `typed_value` | Shredded representation. Present iff `typed_value_dtype` is set. | Any type (`typed_value_dtype`); may be primitive, `List`, or `Struct` with recursively shredded children; length `n`. |
+
+**On-wire child ordering.** Children are packed in slot order, skipping absent optional slots:
+`[validity?] , metadata , [value?] , [typed_value?]`. A reader determines the layout as follows. Let
+`expected = 1 + has_value + (typed_value_dtype present)`. If the serialised child count equals
+`expected`, there is **no** validity child (validity derives from the dtype's nullability) and the
+first child is `metadata`. If the count equals `expected + 1`, the **first** child is the validity
+array and `metadata` follows. `value` (if `has_value`) and then `typed_value` (if present) follow
+`metadata` (`vtable.rs` `ParquetVariant::deserialize`).
+
+**Row semantics.** Each row is a Parquet Variant value, interpreted from the present components by
+the (`value`, `typed_value`) presence combination:
+
+| `value` | `typed_value` | Meaning |
+|---------|---------------|---------|
+| null | null | Missing value (valid only for a shredded object field). |
+| non-null | null | Unshredded — decode from `metadata` + `value`. |
+| null | non-null | Perfectly shredded — decode from `typed_value`. |
+| non-null | non-null | Partially shredded object — merge shredded fields (`typed_value`) with raw-only fields (`value`); duplicate field names are invalid writer output. |
+
+At least one of `value` / `typed_value` is always present (`vtable.rs` `ParquetVariant::validate`,
+`ParquetVariant::deserialize`).
+
+**Validity.** Stored slot (`validity`, child 0), row-aligned — the array's top-level row validity is
+read directly from this slot (mechanism 1, `validity.rs` `ParquetVariant::validity`), with **no**
+cross-component combine step. The component
+children (`metadata` / `value` / `typed_value`) carry their own validity for their own values,
+independently of the top-level row validity. See [Validity](../encoding-format.md#validity).
diff --git a/docs/specification/file-format.md b/docs/specification/file-format.md
new file mode 100644
index 00000000000..ea2e8d5b43b
--- /dev/null
+++ b/docs/specification/file-format.md
@@ -0,0 +1,247 @@
+# File Format
+
+:::{important}
+The Vortex File Format has been considered stable since the release of version 0.36.0. That means that you can expect all
+future versions of the Vortex library to be able to read files written by version 0.36.0 or later (up to and including
+the version doing the reading).
+:::
+
+:::{seealso}
+The majority of the complexity of the Vortex file format is encapsulated in [Vortex Layouts](/concepts/layouts).
+Unless you are interested in the specific byte layout of the file, you are probably looking for that documentation!
+:::
+
+Recall that [Vortex Layouts](/concepts/layouts) provide a mechanism to efficiently query large serialized Vortex
+arrays. The _Vortex File Format_ is designed to provide a container for these serialized arrays, as well as footer
+definition that allows efficiently querying the layout.
+
+Other considerations for the Vortex file format include:
+
+* Backwards compatibility, and (coming soon) forwards compatibility.
+* Fine-grained encryption.
+* Efficient access for both local disk and cloud storage.
+* Minimal overhead reading few columns or rows from wide or long arrays.
+
+## File Specification
+
+The Vortex file format has a very small definition, with much of the complexity encapsulated
+in [Vortex Layouts](/concepts/layouts).
+
+```
+<4 bytes> magic number 'VTXF'
+... segments of binary data, optionally with inter-segment padding
+... postscript data
+<2 bytes> u16 version tag
+<2 bytes> u16 postscript length
+<4 bytes> magic number 'VTXF'
+```
+
+The file format begins and ends with the 4-byte magic number `VTXF`.
+Immediately prior to the trailing magic number are two little-endian 16-bit integers: the version tag and the length of the postscript.
+
+:::{important}
+**Format version.** The current Vortex file format version is `1`. The version tag records the
+format version a file was written against, and a conformant reader MUST reject any file whose
+version tag is not *exactly equal* to the version the reader implements. The check is an exact
+match, not a `>=` lower bound: a reader that implements version `1` rejects both older and newer
+version tags. The reference reader reads the tag from the end-of-file marker and bails with an
+"unsupported version" error on any mismatch.
+:::
+
+The exact-match rule coexists with the [Backward Compatibility](#backward-compatibility) guarantee
+below because the **format version tag is deliberately stable**: additive FlatBuffer evolution (new
+optional fields, new union variants appended at the end) does **not** bump it, so every library
+release since 0.36.0 both writes and reads format version `1`. The tag changes only on a breaking
+wire change a reader genuinely cannot interpret; until then "a newer library reads an older file"
+holds *precisely because* both files are format version `1`. (A library version such as `0.36.0` is
+distinct from the format version tag `1`.)
+
+:::{note}
+**Leading versus trailing magic.** Both the leading and trailing 4-byte markers are `VTXF`. The
+reference reader validates only the *trailing* magic — the copy inside the 8-byte end-of-file
+marker, which it reads first — and does not currently read or validate the leading marker. The
+leading marker is written to identify the file at its start for external tooling; a reader that
+chooses to validate it MUST likewise require it to equal `VTXF`.
+:::
+
+A minimal Vortex file thus consists of just these byte ranges, plus the alignment, encryption, and compression
+configurations for the other pieces of metadata.
+
+
+
+## Postscript
+
+The postscript contains the locations of:
+
+1. a `dtype` segment representing the top-level logical data type (i.e., schema) — *optional*; when absent, the reader must obtain the root `DType` from an external source before binding the layout
+2. a `layout` segment containing the root `Layout` — *required*
+3. a `statistics` segment containing file-level per-field statistics (e.g., minima and maxima of each field/column, for whole-file pruning) — *optional*
+4. a `footer` segment containing a dictionary-encoded _segment map_, and other shared configuration such as compression and encryption schemes — *required*
+
+:::{literalinclude} ../../vortex-flatbuffers/flatbuffers/vortex-file/footer.fbs
+:start-after: [postscript]
+:end-before: [postscript]
+:::
+
+## Data Type
+
+Both viewed arrays and viewed layouts require an external `DType` to instantiate them. This helps us to avoid
+redundancy in the serialized format since it is very common for a child array or layout to inherit or infer its data
+type from the parent type.
+
+The root `DType` segment is a flat buffer serialized `DType` object. See [DType Format](/specification/dtype-format) for more
+information.
+
+:::{note}
+Unlike many columnar formats, the `DType` of a Vortex file is not required to be a `StructDType`. It is perfectly
+valid to store a `Float64` array, a `Boolean` array, or any other root data type.
+:::
+
+## Footer
+
+The footer is a flat buffer serialized `Footer` object. This object contains all the information required to
+load the root `Layout` object into a usable `LayoutReader`.
+For example, it contains the locations, compression schemes, encryption schemes, and required alignment of all segments in the file.
+
+:::{literalinclude} ../../vortex-flatbuffers/flatbuffers/vortex-file/footer.fbs
+:start-after: [footer]
+:end-before: [footer]
+:::
+
+:::{note}
+**Alignment is base-2-exponent encoded.** The `alignment_exponent` field carried by each
+`SegmentSpec` (in the footer's segment map) and each `PostscriptSegment` (in the postscript) stores
+the *log2* of the segment's required byte alignment, not the alignment itself. A reader recovers the
+required alignment as `1 << alignment_exponent` — for example, an `alignment_exponent` of `10`
+denotes 1024-byte alignment.
+:::
+
+The footer is separated from the Data Type such that large schemas can be omitted from the file if they can be
+shared or fetched from an external source.
+
+## Reified File Example
+
+Since Vortex files are largely self-describing, many mainstays of other columnar file formats (e.g., whether or not to
+have row groups) are decided by the **writer**, rather than being a rigid part of the specification. To build intuition,
+consider an example Vortex file with two non-nullable columns, "A" of type i32, and "B" of type UTF-8. Using the defaults
+as of June 2025, it might look as follows.
+
+
+
+## Write Completeness and File Integrity
+
+The Vortex file format is aggressively self-describing: the footer's segment map, the root
+`DType`, and the [Layout](/concepts/layouts) tree together *declare* what the file contains, and
+a reader takes those declarations at face value. That design keeps reads cheap — a reader never has
+to scan the body to discover its shape — but it moves a burden onto the **writer**: the declarations
+in the footer are a promise, and every byte they promise must actually be in the file. A file whose
+footer is syntactically well-formed but whose declarations out-run the bytes that were written is
+*silently corrupt* — it opens cleanly and reads back fewer rows (or empty columns) than were
+intended, with no error raised.
+
+This failure mode is real: a background write task that panics and closes its output channel without
+propagating the failure can leave behind a structurally-valid file that is empty or missing chunks
+rather than raising a loud write error.
+The section below states the invariant a conformant writer must uphold, and the validation a
+conformant reader must perform so that such a file is rejected loudly instead of read as if it were
+complete.
+
+### The write-completeness invariant
+
+A conformant writer MUST NOT finalize a file — write the trailing postscript, version tag, and
+closing magic number — unless all of the following hold. A file that satisfies them is *complete*;
+a file that does not is corrupt even if it parses.
+
+1. **Every declared segment is backed by real bytes.** For each `SegmentSpec` in the footer's
+ segment map, the byte range `[offset, offset + length)` must lie within the file and contain the
+ segment's complete payload. The writer records a segment's location *and* emits its bytes; if the
+ bytes are ever dropped (for example because a background task failed after the location was
+ recorded), the map entry becomes a promise the file cannot keep, and the write must fail rather
+ than complete.
+2. **Segment offsets are non-decreasing.** Segments are laid out in ascending offset order (readers
+ reject an out-of-order map).
+3. **Every referenced segment exists.** Each `SegmentId` named by any node of the root layout tree
+ must resolve to an entry in the segment map.
+4. **Row counts are consistent throughout the layout tree.** The `row_count` a layout node declares
+ must agree with the data beneath it:
+ - a **chunked** layout's `row_count` equals the sum of its children's `row_count`s;
+ - a **struct / columnar** layout carries exactly one child per field of its `DType` (plus one
+ leading validity child when the struct is nullable), and — because a struct is a row-aligned
+ bundle of columns — **every field column spans the same number of rows** as the struct itself.
+ Equal row counts across all columns is the rule that makes a projection of any subset of columns
+ reconstruct the same rows;
+ - a **flat** (leaf) layout's `row_count` equals the number of elements in the array serialized
+ into its segment.
+5. **All required segments are present.** The `layout` and `footer` segments are mandatory; the
+ `dtype` segment is mandatory unless the writer deliberately excludes it (see
+ [Data Type](#data-type)); and if the footer declares a `statistics` segment, that segment must be
+ present and complete.
+
+:::{important}
+File-level statistics are *advisory*, not an integrity backstop. They are optional, per-field, and
+derived by the writer from the same data stream, so they cannot be used to independently confirm that
+the body is complete.
+:::
+
+### Reader-side validation
+
+Because the writer's promise can be violated (by the bug above, by a truncated upload, or by any
+partial write), a conformant reader MUST verify the file against its own declarations and raise an
+error on any mismatch, rather than silently serving whatever parsed. At minimum a reader MUST:
+
+1. **Require the mandatory segments.** Reject a file whose postscript is missing the `layout` or
+ `footer` segment, whose footer carries no segment map, or that declares a `statistics` segment
+ which cannot be resolved.
+2. **Reject an out-of-order segment map** (offsets not non-decreasing).
+3. **Resolve every referenced segment.** Every `SegmentId` reachable from the root layout must index
+ a real entry in the segment map; a dangling reference is an error, not an empty read.
+4. **Enforce exact segment lengths on fetch.** A segment read must return exactly `length` bytes; a
+ short read — the signature of a range that runs past the true end of the file — is an error.
+5. **Cross-check row counts** as it binds and decodes the layout tree: chunk sums against the parent
+ chunked `row_count`; the child count of a struct/columnar layout against its `DType`; the equal
+ per-column row counts of a struct; and each leaf array's decoded length against the `row_count`
+ its flat layout declared.
+
+:::{warning}
+**The layout tree is the file's only record of row counts.** There is no independent, file-level
+"expected total row count" stored outside the tree, so the checks above can only prove a file is
+*internally consistent* — not that it is *complete*. A file whose layout tree was truncated before it
+was finalized (fewer chunks, or a smaller root `row_count`, than the producer intended) is perfectly
+self-consistent and reads back as a smaller-but-valid file; a reader cannot detect the missing
+trailing rows from the file alone. This is precisely why completeness is fundamentally a *writer*
+obligation: the reader's validation catches contradictions, but only the writer can guarantee the
+declarations describe everything that should have been written.
+:::
+
+:::{note}
+A fully conformant reader **MUST** perform all of the checks above uniformly. In particular it MUST
+validate a struct/columnar layout's child *count* against the schema **and** verify that every column
+declares the same `row_count` (otherwise an unequal-length struct binds without complaint and diverges
+only when a row range spanning the short column is scanned); it MUST enforce the chunked row-count sum
+and each leaf's decoded-length and `DType`; and it MUST perform these cross-checks **eagerly** at open
+time, surfacing them as **recoverable errors** rather than deferring them to first access or to
+process-aborting assertions.
+:::
+
+## Backward Compatibility
+
+Backward compatibility guarantees that any **older** Vortex file can be read by **newer** versions of the Vortex library,
+and is expected from all releases of Vortex from version 0.36.0 onwards.
+
+## Forward Compatibility
+
+:::{warning}
+Forward compatibility is not yet implemented, but is planned to ship prior to the 1.0 release.
+:::
+
+Forward compatibility extends the preceding stability guarantee such that **newer** Vortex files can be read by
+**older** versions of the Vortex library.
+
+The intent of this work is to allow us to continue to evolve the Vortex File Format, avoiding calcification
+and remaining up-to-date with new compression codecs and layout optimizations -- without breaking existing
+readers or requiring lockstep upgrades.
+
+The plan is that at write-time, a minimum supported reader version is declared. Any encodings or layouts added after that minimum
+reader version can then be embedded into the file with WebAssembly decompression logic. Old readers are able to decompress new
+data (slower than native code, but still with SIMD acceleration) and read the file. New readers are able to make the best use of
+these encodings with native decompression logic and additional push-down compute functions (which also provides an incentive to upgrade).
diff --git a/docs/specification/index.md b/docs/specification/index.md
new file mode 100644
index 00000000000..186c75ddba7
--- /dev/null
+++ b/docs/specification/index.md
@@ -0,0 +1,47 @@
+# Specification
+
+Vortex defines serialization formats for columnar data on disk and over the wire. The on-disk
+**File Format** is stable (since 0.36.0); the over-the-wire **IPC Format** is documented but still
+unstable and subject to change. This section specifies both — the byte layouts, FlatBuffer and Protobuf schemas, and the procedure
+for reading a file — with one documented exception: the layout tree's wire format is not yet
+specified here (see [Reading a File](reading-a-file.md)).
+
+If you just want to read or write Vortex data from a program, you don't need these pages — reach for
+the [language bindings](../api/index.md) or a [query-engine integration](../user-guide/index.md)
+instead. The specification is for reader/writer implementers, format debuggers, and anyone porting
+Vortex to a new language.
+
+## The format stack
+
+The formats build on one another:
+
+- **[Reading a File](reading-a-file.md)** — the end-to-end procedure for turning a `.vortex` file
+ into arrays. Start here if you want to understand how a read actually works.
+- **[File Format](file-format.md)** — the on-disk container: magic numbers, postscript, footer, and
+ compatibility guarantees.
+- **[Array Format](array-format.md)** — the shared binary representation of a single array, used
+ identically in memory, on disk, and over the wire.
+- **[Encoding Format](encoding-format.md)** — what the bytes inside a single array node *mean*,
+ encoding by encoding: the validity contract, and (as they land) each encoding's buffer/metadata layout.
+- **[IPC Format](ipc-format.md)** — the message-oriented wire protocol that streams arrays between
+ processes.
+- **[DType Format](dtype-format.md)** and **[Scalar Format](scalar-format.md)** — the schema and
+ scalar-value encodings referenced by the above.
+- **[Row Encoding](row-encoding.md)** — a separate, **experimental** byte-sortable row-key format
+ (used internally for sort keys), *not* part of the file/IPC format stack above.
+
+```{toctree}
+---
+maxdepth: 2
+hidden:
+---
+
+reading-a-file
+file-format
+array-format
+encoding-format
+ipc-format
+dtype-format
+scalar-format
+row-encoding
+```
diff --git a/docs/specification/ipc-format.md b/docs/specification/ipc-format.md
new file mode 100644
index 00000000000..4f9452fe3a2
--- /dev/null
+++ b/docs/specification/ipc-format.md
@@ -0,0 +1,39 @@
+# IPC Format
+
+The IPC format wraps [serialized arrays](array-format.md) in a message-oriented protocol for
+streaming between processes. It is used for inter-process communication and can serve as the wire
+protocol for remote source execution in the [Scan API](../concepts/scanning.md).
+
+:::{note}
+The IPC format is unstable and subject to change. It does not yet support shared arrays (e.g. a
+dictionary shared across multiple chunked arrays), which limits its efficiency for certain workloads.
+This is an area of active development. Unlike the IPC format, the [File Format](file-format.md) is
+stable.
+:::
+
+## Message framing
+
+Each message is a length-prefixed FlatBuffer header followed by a body:
+
+```
+[u32 header length] [flatbuffer Message header] [body bytes]
+```
+
+The `u32` header length is little-endian. The `Message` header carries the fields an implementer
+needs to consume the stream:
+
+- **`version`** (`MessageVersion`, default `V0`) — the message format version.
+- **`header`** — a union selecting one of the message types below.
+- **`body_size`** (`uint64`) — the exact number of body bytes that follow the header. A reader uses
+ this to know how much to consume before the next message.
+
+## Message types
+
+The `header` union selects one of three message types:
+
+- **`ArrayMessage`** — a [serialized array](array-format.md) (in the body) with its `row_count` and
+ the list of `encodings` (encoding IDs) the array references.
+- **`BufferMessage`** — a raw buffer with an `alignment_exponent`, used for transferring individual
+ segments.
+- **`DTypeMessage`** — signals that the body is a serialized [dtype](dtype-format.md), used to
+ communicate the schema before data transfer begins.
diff --git a/docs/specification/reading-a-file.md b/docs/specification/reading-a-file.md
new file mode 100644
index 00000000000..ae340ad32da
--- /dev/null
+++ b/docs/specification/reading-a-file.md
@@ -0,0 +1,148 @@
+# Reading a Vortex File
+
+This page walks through how a reader turns the bytes of a `.vortex` file into arrays. It is the
+procedural companion to the [File Format](file-format.md) byte-layout reference: the File Format page
+tells you *what the bytes are*, this page tells you *what to do with them*.
+
+:::{seealso}
+If you only want to read or write Vortex files from a program, you don't need any of this — use the
+language bindings ([Python](../api/python/index.rst), [Rust](https://docs.rs/vortex),
+[Java](../api/java/index.rst)) or a [query-engine integration](../user-guide/index.md). This page is
+for people implementing a reader, debugging the format, or porting Vortex to a new language.
+:::
+
+## The easy part and the hard part
+
+Here is the honest framing, because it shapes the whole format:
+
+**Reading the bytes is not hard.** A Vortex file is a magic number, a small *postscript* that points
+at a footer, a footer that describes where every data segment lives, a root *dtype* (the schema), and
+a tree of *layouts* that says how those segments compose into arrays. Open the file, read ~64 KiB
+from the end, follow a handful of offsets, and you can materialize any column. A competent implementer
+can write a correct-but-naive reader in an afternoon.
+
+**Getting *high performance* out of the file is the hard part** — and it is the entire reason the
+format is shaped the way it is. A fast reader must:
+
+- **prune** whole row ranges it can prove are irrelevant, using the per-zone statistics stored
+ alongside the data (min/max/null-count zone maps);
+- **push down projections** so it only fetches the columns a query touches — cheap when the writer
+ has laid out columns as independent layouts (the default), so a column's segments can be fetched
+ without reading its siblings;
+- **push down filters** and evaluate them against *compressed* data where possible, before
+ decompressing;
+- **schedule I/O** well: coalesce nearby segment reads, issue them concurrently, and prefetch — the
+ difference between one large sequential read and thousands of tiny random ones.
+
+Everything below describes the naive read path. The pruning, pushdown, and I/O-scheduling machinery
+that makes it fast lives in [Layouts](../concepts/layouts.md), the
+[Scan API](../concepts/scanning.md), and the [I/O subsystem](../developer-guide/internals/io.md).
+
+## Step 1 — Read the trailer
+
+A Vortex file begins and ends with the 4-byte magic number `VTXF`. The last 8 bytes form the
+`EndOfFile` struct:
+
+```
+... segments of binary data (+ optional inter-segment padding)
+... postscript
+<2 bytes> u16 version tag (little-endian)
+<2 bytes> u16 postscript length (little-endian)
+<4 bytes> magic number 'VTXF'
+```
+
+All manually-framed integers in the format — the trailer `u16`s here, the `u32` length suffix on a
+serialized array, and the `u32` header length in an [IPC](ipc-format.md) message — are
+**little-endian**. (The FlatBuffers themselves are little-endian by the FlatBuffer spec.)
+
+The postscript is guaranteed never to exceed `MAX_POSTSCRIPT_SIZE` = `u16::MAX - 8` = **65527** bytes.
+Because of that bound, a reader's **first read defaults to `MAX_POSTSCRIPT_SIZE + EOF_SIZE` = 65535
+bytes from the end of the file** (the `EndOfFile` trailer is 8 bytes), which is guaranteed to cover
+both the trailer and the entire postscript in a single round trip. Validate the trailing magic, read
+the version tag, and slice out the postscript using its length.
+
+## Step 2 — Parse the postscript
+
+The postscript is a FlatBuffer locating four segments by offset/length (encryption and compression
+specs are inlined here so they never require a prior fetch):
+
+:::{literalinclude} ../../vortex-flatbuffers/flatbuffers/vortex-file/footer.fbs
+:start-after: [postscript]
+:end-before: [postscript]
+:::
+
+- **`layout`** (required) — the root [Layout](../concepts/layouts.md) FlatBuffer.
+- **`footer`** (required) — the dictionary-encoded *registry* (segments, encodings, layouts,
+ compression, encryption).
+- **`dtype`** (optional) — the root [DType](../concepts/dtypes.md). Optional because large schemas can
+ be shared or fetched externally rather than embedded.
+- **`statistics`** (optional) — file-level per-field statistics for whole-file pruning.
+
+:::{important}
+The root `DType` is required to bind the layout and decode arrays, even though the `dtype` *segment*
+is optional. If the postscript carries a `dtype` segment, fetch and parse it now; if it does not, the
+reader **must** obtain the root `DType` from the caller (or an external catalog) before proceeding.
+A file with no embedded dtype and no externally-supplied dtype cannot be read.
+:::
+
+The two-round-trip design is deliberate: keeping the postscript small and fixed-bounded means the
+footer and the (possibly large) dtype live in their own segments, so a reader needs at most two round
+trips — one for the tail, one for the segments it points to — rather than three.
+
+## Step 3 — Load the footer registry
+
+The footer is the lookup table the rest of the read depends on:
+
+:::{literalinclude} ../../vortex-flatbuffers/flatbuffers/vortex-file/footer.fbs
+:start-after: [footer]
+:end-before: [footer]
+:::
+
+It holds dictionary-encoded specs that the layout tree references by index:
+
+- **`segment_specs`** — for each `SegmentId`, the `offset`, `length`, and `alignment_exponent` of a
+ byte range in the file. This is the indirection that lets the same layout tree be backed by a local
+ file, an object store, or an in-memory cache.
+- **`array_specs`** / **`layout_specs`** — globally-unique string IDs resolved against the Vortex
+ registry at read time to find the encoding/layout implementation.
+- **`compression_specs`** / **`encryption_specs`** — per-segment schemes (reserved; see
+ [File Format](file-format.md)).
+
+## Step 4 — Bind the layout tree to a segment source
+
+Deserialize the root `Layout` FlatBuffer into a tree of nodes — each carries a layout ID (into
+`layout_specs`), a row count, metadata, child layouts, and `SegmentId`s. Binding this tree to a
+*segment source* (a thing that can fetch a `SegmentId`'s bytes) yields a `LayoutReader` that fetches
+data lazily. The shape of the tree — struct-of-columns, chunked row groups, zone maps, dictionaries —
+is entirely the writer's choice; see [Layouts](../concepts/layouts.md).
+
+:::{note}
+The `Layout` FlatBuffer's byte-level wire format (the `layout.fbs` schema — a node's `encoding`
+index, `row_count`, opaque `metadata`, child layouts, and `segments`) is **not yet specified in this
+Specification section**; it is a deferred follow-up. [Layouts](../concepts/layouts.md) describes the
+tree conceptually, but a byte-exact clean-room reader of the layout tree cannot be built from the
+spec alone today. This is the one boundary **required to decode data** still defined only by the
+reference implementation. (The advisory `statistics` segment is likewise not byte-specified here, but
+a reader never needs it to decode values.)
+:::
+
+## Step 5 — Resolve segments and decode
+
+To materialize a region:
+
+1. Walk the layout tree to the nodes covering the requested columns and row range, resolving each
+ node's layout encoding through **`layout_specs`** (this is the layout registry — distinct from
+ the array registry in the next step).
+2. For each referenced `SegmentId`, look up its `SegmentSpec` in the footer to get `offset`,
+ `length`, and required alignment.
+3. Fetch those byte ranges. A leaf (flat) layout's segment is not a bare buffer — it holds a
+ **serialized array**: the data buffers followed by an Array FlatBuffer and a trailing `u32`
+ length (see [Array Format](array-format.md)). The buffers within it are aligned so they can be
+ used **zero-copy** as in-memory arrays.
+4. Parse that serialized array, resolving each `ArrayNode`'s encoding through **`array_specs`** (the
+ array registry), and decode, recursing into child nodes and child layouts.
+
+A naive reader fetches one segment per read and stops here — and it will be correct. A fast reader
+overlaps steps 1–3 across columns and chunks, prunes nodes using zone statistics before fetching, and
+coalesces adjacent segments into larger reads. That is the work the [Scan API](../concepts/scanning.md)
+exists to do.
diff --git a/docs/specs/row-encoding.md b/docs/specification/row-encoding.md
similarity index 100%
rename from docs/specs/row-encoding.md
rename to docs/specification/row-encoding.md
diff --git a/docs/specification/scalar-format.md b/docs/specification/scalar-format.md
new file mode 100644
index 00000000000..377205ab2c2
--- /dev/null
+++ b/docs/specification/scalar-format.md
@@ -0,0 +1,131 @@
+# Scalar Format
+
+A **scalar** is a single typed value: a [`DType`](dtype-format.md) paired with a value drawn from
+that type's domain. Scalars appear wherever Vortex needs to represent one value rather than an
+array — in compute and pushdown **expressions** (constants and literals) and as **array metadata**
+(for example the base and step of a sequence array, or the value of a constant array). Statistics
+(per-zone minima and maxima, etc.) store a bare `ScalarValue` rather than a full `Scalar`: the
+`dtype` is supplied by the array/field the statistic belongs to, so it is not repeated per value.
+
+## Model
+
+A `Scalar` carries its `dtype` alongside its `value`, so it is self-describing. The `ScalarValue` is
+a `oneof` over the possible value kinds, paralleling the logical types in the
+[DType format](dtype-format.md). A few encoding details:
+
+- **Floats.** `f32_value` and `f64_value` store IEEE-754 values directly; `f16_value` stores the
+ raw 16-bit half-precision bits in a `uint64`, since Protobuf has no native half type.
+- **Nested values.** `ListValue` holds a repeated sequence of `ScalarValue`s. Note the protobuf has
+ no dedicated struct value, and the current deserializer accepts `ListValue` only for a `List`
+ dtype — `FixedSizeList` and `Struct` scalar round-trips through this protobuf are not yet
+ supported.
+- **Variant.** A variant scalar carries a row-specific nested `Scalar`.
+- **Null.** Absence is represented by `null_value`, regardless of the declared `dtype`.
+
+Scalars are serialized with Protobuf rather than FlatBuffers, because they travel as embedded
+metadata inside arrays and expressions rather than on the zero-copy file/IPC path — the same split
+described in the [DType format](dtype-format.md#serializations).
+
+## Protobuf definition
+
+:::{literalinclude} ../../vortex-proto/proto/scalar.proto
+:language: protobuf
+:start-at: syntax =
+:::
+
+## Value encoding
+
+The `oneof` above is deliberately *narrower* than the [DType](dtype-format.md) system it serves:
+several logical types share a single arm, and one arm (`bytes_value`) serves three unrelated types.
+A `ScalarValue` is therefore **not self-describing** — the same wire bytes decode to different
+values depending on the `DType`. Decoding is well-defined only *with* that `DType`, which is supplied
+by context — the enclosing `Scalar`'s `dtype`, or the array/field that a statistic (or a `Constant`,
+`Sequence`, or `FoR` metadata value) belongs to. The overloaded arms — `int64_value`, `uint64_value`,
+`string_value`, and `bytes_value`, each shared across several logical types — are **not** recoverable
+from the `ScalarValue` alone; the type-specific arms are self-identifying, but a reader still validates
+them against the supplied `DType`. The rest of this section is the arm-by-arm mapping a reader needs
+to turn a `ScalarValue` plus its `DType` back into a concrete value.
+
+### Primitives
+
+All integers collapse onto two arms by signedness, and `F16` rides in a `uint64`:
+
+| Vortex `PType` | `oneof` arm | Carriage |
+|----------------|-------------|----------|
+| `I8`, `I16`, `I32`, `I64` | `int64_value` | value sign-extended to 64-bit |
+| `U8`, `U16`, `U32`, `U64` | `uint64_value` | value zero-extended to 64-bit |
+| `F16` | `f16_value` | raw 16-bit bits carried in a `uint64` |
+| `F32` | `f32_value` | IEEE-754 single |
+| `F64` | `f64_value` | IEEE-754 double |
+
+`int64_value` is a Protobuf `sint64`, so on the wire it is **ZigZag-varint-coded** — not a plain
+two's-complement varint. A decoder must ZigZag-decode it before the width narrowing below; reading it
+as a plain `int64` varint mis-reads every negative value. `uint64_value` and `f16_value` are plain
+`uint64` varints.
+
+On decode the `DType`'s concrete `PType` narrows the wide arm back to its declared width: an
+`int64_value` read against an `I16` dtype is range-checked and narrowed to `i16`, and likewise for
+`uint64_value`. A value outside the target type's range is a hard error, never a silent wrap — the
+narrowing is checked, not a wrapping truncation.
+
+:::{note}
+For backward compatibility a reader also accepts an integer carried under the *opposite* signedness
+arm (for example a `U32` decoded from `int64_value`), and an `F16` carried in `uint64_value` — the
+legacy pre-`f16_value` encoding, whose payload is `f16::to_bits() as u64`. A writer **MUST** use the
+arm in the table above; a reader **SHOULD** accept these legacy forms so that older statistics remain
+readable.
+:::
+
+### Strings, binary, and decimals
+
+The `bytes_value` arm is overloaded across three logical types, and `Utf8`/`Binary` may also travel
+in `string_value`. A writer emits `Utf8` as `string_value` and both `Binary` and `Decimal` as
+`bytes_value`; a reader disambiguates **solely** by the supplied `DType`:
+
+| `DType` | Decodes from | Interpreted as |
+|---------|--------------|----------------|
+| `Utf8` | `string_value` or `bytes_value` | UTF-8 string bytes |
+| `Binary` | `string_value` or `bytes_value` | raw bytes, verbatim |
+| `Decimal` | `bytes_value` | little-endian two's-complement integer (below) |
+
+A `Decimal` scalar has no arm of its own. Its backing integer is serialized into `bytes_value` as
+**little-endian two's-complement** bytes, and the integer width is recovered on decode from the byte
+length alone:
+
+| `bytes_value` length | Backing integer |
+|----------------------|-----------------|
+| 1 | `i8` |
+| 2 | `i16` |
+| 4 | `i32` |
+| 8 | `i64` |
+| 16 | `i128` |
+| 32 | `i256` |
+
+Any other length is a hard error. The decimal's precision and scale are **not** carried in the
+`ScalarValue`; they come from the `Decimal` `DType`.
+
+### Booleans, lists, and variants
+
+Three arms decode directly against their matching `DType`; a type mismatch is a hard error, never a
+reinterpretation.
+
+- **`bool_value`** — a Protobuf `bool`, valid only against a `Bool` `DType` (`proto.rs` `bool_from_proto`).
+- **`list_value`** — a `ListValue` carrying a `repeated ScalarValue`. Each element is decoded
+ recursively against the **element** type of the enclosing `List` `DType` (`proto.rs` `list_from_proto`). As
+ the Model section notes above, the deserializer accepts `list_value` only for a `List` dtype today;
+ `FixedSizeList` and `Struct` scalars do not yet round-trip through this Protobuf.
+- **`variant_value`** — a fully nested `Scalar` (its own `dtype` and `value`), valid only against a
+ `Variant` `DType` (`proto.rs` `ScalarValue::from_proto`). Because it carries its own `dtype` it is self-describing: a
+ reader decodes it by recursing into the ordinary `Scalar` decode.
+
+### Extension types
+
+Before matching any arm, a decoder handed an `Extension` `DType` substitutes the extension's
+`storage_dtype` and decodes the value against that instead. An extension scalar is therefore encoded
+exactly as its storage value would be; the extension identity lives only in the `DType`.
+
+### Null
+
+`null_value` uses the well-known `google.protobuf.NullValue` enum (imported from
+`google/protobuf/struct.proto`), not a Vortex-defined type — a non-Rust reader must resolve it from
+the standard Protobuf descriptors. It denotes absence regardless of the declared `DType`.
diff --git a/docs/specs/vortex_file_format.svg b/docs/specification/vortex_file_format.svg
similarity index 100%
rename from docs/specs/vortex_file_format.svg
rename to docs/specification/vortex_file_format.svg
diff --git a/docs/specs/vortex_file_format_minimal.svg b/docs/specification/vortex_file_format_minimal.svg
similarity index 100%
rename from docs/specs/vortex_file_format_minimal.svg
rename to docs/specification/vortex_file_format_minimal.svg
diff --git a/docs/specs/dtype-format.md b/docs/specs/dtype-format.md
deleted file mode 100644
index 39bed0938e6..00000000000
--- a/docs/specs/dtype-format.md
+++ /dev/null
@@ -1,12 +0,0 @@
-# DType Format
-
-## FlatBuffer Definition
-
-:::{literalinclude} ../../vortex-flatbuffers/flatbuffers/vortex-dtype/dtype.fbs
-:::
-
-## Protobuf Definition
-
-:::{literalinclude} ../../vortex-proto/proto/dtype.proto
-:language: protobuf
-:::
diff --git a/docs/specs/file-format.md b/docs/specs/file-format.md
deleted file mode 100644
index 425294f2d99..00000000000
--- a/docs/specs/file-format.md
+++ /dev/null
@@ -1,119 +0,0 @@
-# File Format
-
-:::{important}
-The Vortex File Format has been considered stable since the release of version 0.36.0. That means that you can expect all
-future versions of the Vortex library to be able to read files written by version 0.36.0 or later (up to and including
-the version doing the reading).
-:::
-
-:::{seealso}
-The majority of the complexity of the Vortex file format is encapsulated in [Vortex Layouts](/concepts/layouts).
-Unless you are interested in the specific byte layout of the file, you are probably looking for that documentation!
-:::
-
-Recall that [Vortex Layouts](/concepts/layouts) provide a mechanism to efficiently query large serialized Vortex
-arrays. The _Vortex File Format_ is designed to provide a container for these serialized arrays, as well as footer
-definition that allows efficiently querying the layout.
-
-Other considerations for the Vortex file format include:
-
-* Backwards compatibility, and (coming soon) forwards compatibility.
-* Fine-grained encryption.
-* Efficient access for both local disk and cloud storage.
-* Minimal overhead reading few columns or rows from wide or long arrays.
-
-## File Specification
-
-The Vortex file format has a very small definition, with much of the complexity encapsulated
-in [Vortex Layouts](/concepts/layouts).
-
-```
-<4 bytes> magic number 'VTXF'
-... segments of binary data, optionally with inter-segment padding
-... postscript data
-<2 bytes> u16 version tag
-<2 bytes> u16 postscript length
-<4 bytes> magic number 'VTXF'
-```
-
-The file format begins and ends with the 4-byte magic number `VTXF`.
-Immediately prior to the trailing magic number are two 16-bit integers: the version tag and the length of the postscript.
-
-Notably, this minimal notion of a Vortex file effectively includes only the byte ranges, alignment, encryption, and compression
-configurations for other pieces of metadata.
-
-
-
-## Postscript
-
-The postscript contains the locations of:
-
-1. a `dtype` segment representing the top-level logical data type (i.e., schema)
-2. a `layout` segment containing the root `Layout`
-3. a `statistics` segment containing file-level per-field statistics (e.g., minima and maxima of each field/column, for whole-file pruning)
-4. a `footer` segment containing a dictionary-encoded _segment map_, and other shared configuration such as compression and encryption schemes
-
-:::{literalinclude} ../../vortex-flatbuffers/flatbuffers/vortex-file/footer.fbs
-:start-after: [postscript]
-:end-before: [postscript]
-:::
-
-## Data Type
-
-Both viewed arrays and viewed layouts require an external `DType` to instantiate them. This helps us to avoid
-redundancy in the serialized format since it is very common for a child array or layout to inherit or infer its data
-type from the parent type.
-
-The root `DType` segment is a flat buffer serialized `DType` object. See [DType Format](/specs/dtype-format) for more
-information.
-
-:::{note}
-Unlike many columnar formats, the `DType` of a Vortex file is not required to be a `StructDType`. It is perfectly
-valid to store a `Float64` array, a `Boolean` array, or any other root data type.
-:::
-
-## Footer
-
-The footer is a flat buffer serialized `Footer` object. This object contains all the information required to
-load the root `Layout` object into a usable `LayoutReader`).
-For example, it contains the locations, compression schemes, encryption schemes, and required alignment of all segments in the file.
-
-:::{literalinclude} ../../vortex-flatbuffers/flatbuffers/vortex-file/footer.fbs
-:start-after: [footer]
-:end-before: [footer]
-:::
-
-The footer is separated from the Data Type such that large schemas can be omitted from the file if they can be
-shared or fetched from an external source.
-
-## Reified File Example
-
-Since Vortex files are largely self-describing, many mainstays of other columnar file formats (e.g., whether or not to
-have row groups) are decided by the **writer**, rather than being a rigid part of the specification. To build intuition,
-consider an example Vortex file with two non-nullable columns, "A" of type i32, and "B" of type UTF-8. Using the defaults
-as of June 2025, it might look as follows.
-
-
-
-## Backward Compatibility
-
-Backward compatibility guarantees that any **older** Vortex file can be read by **newer** versions of the Vortex library,
-and is expected from all releases of Vortex from version 0.36.0 onwards.
-
-## Forward Compatibility
-
-:::{warning}
-Forward compatibility is not yet implemented, but is planned to ship prior to the 1.0 release.
-:::
-
-Forward compatibility extends the preceding stability guarantee such that **newer** Vortex files can be read by
-**older** versions of the Vortex library.
-
-The intent of this work is to allow us to continue to evolve the Vortex File Format, avoiding calcification
-and remaining up-to-date with new compression codecs and layout optimizations -- without breaking existing
-readers or requiring lockstep upgrades.
-
-The plan is that at write-time, a minimum supported reader version is declared. Any encodings or layouts added after that minimum
-reader version can then be embedded into the file with WebAssembly decompression logic. Old readers are able to decompress new
-data (slower than native code, but still with SIMD acceleration) and read the file. New readers are able to make the best use of
-these encodings with native decompression logic and additional push-down compute functions (which also provides an incentive to upgrade).
diff --git a/docs/specs/index.md b/docs/specs/index.md
deleted file mode 100644
index 045ccb0710d..00000000000
--- a/docs/specs/index.md
+++ /dev/null
@@ -1,15 +0,0 @@
-# Specifications
-
-Vortex defines stable serialization formats for columnar data on disk and over the wire.
-
-```{toctree}
----
-maxdepth: 2
----
-
-file-format
-ipc-format
-dtype-format
-scalar-format
-row-encoding
-```
diff --git a/docs/specs/ipc-format.md b/docs/specs/ipc-format.md
deleted file mode 100644
index 6559818f293..00000000000
--- a/docs/specs/ipc-format.md
+++ /dev/null
@@ -1,12 +0,0 @@
-# IPC Format
-
-:::{warning}
-This page is under construction.
-:::
-
-Planned content:
-
-- Overview of the Vortex IPC wire format
-- Message framing and serialization
-- Compressed array transfer
-- Use by the Scan API for remote interchange
diff --git a/docs/specs/scalar-format.md b/docs/specs/scalar-format.md
deleted file mode 100644
index 2ee512c2be0..00000000000
--- a/docs/specs/scalar-format.md
+++ /dev/null
@@ -1,7 +0,0 @@
-# Scalar Format
-
-## Protobuf Definition
-
-:::{literalinclude} ../../vortex-proto/proto/scalar.proto
-:language: protobuf
-:::
diff --git a/docs/user-guide/duckdb.md b/docs/user-guide/duckdb.md
index 3fc0b03be9b..a0954875b77 100644
--- a/docs/user-guide/duckdb.md
+++ b/docs/user-guide/duckdb.md
@@ -28,7 +28,8 @@ WHERE age > 30;
```
:::{note}
-Direct file path syntax (`SELECT * FROM 'data.vortex'`) is coming in an upcoming DuckDB release.
+Direct file path syntax (`SELECT * FROM 'data.vortex'`) also works: the extension registers a
+replacement scan that rewrites a `.vortex` path to `read_vortex(...)`.
:::
## Writing Vortex Files
diff --git a/docs/user-guide/spark.md b/docs/user-guide/spark.md
index 676c11f332b..bb7a806d00d 100644
--- a/docs/user-guide/spark.md
+++ b/docs/user-guide/spark.md
@@ -1,15 +1,17 @@
# Spark
Vortex provides a Spark DataSource V2 connector for reading and writing Vortex files. The
-connector is published to Maven Central as `dev.vortex:vortex-spark`.
+connector is published to Maven Central as `dev.vortex:vortex-spark_2.13` (the artifact name carries
+the Scala binary version; `vortex-spark_2.12` is also published).
## Installation
-Add the dependency to your build. The connector is built against Spark 4.x with Scala 2.13.
+Add the dependency to your build. The `vortex-spark_2.13` artifact targets Spark 4.x; a
+`vortex-spark_2.12` artifact targeting Spark 3.x is also published.
````{tab} Gradle (Kotlin)
```kotlin
-implementation("dev.vortex:vortex-spark:")
+implementation("dev.vortex:vortex-spark_2.13:")
```
````
@@ -17,7 +19,7 @@ implementation("dev.vortex:vortex-spark:")
```xml
dev.vortex
- vortex-spark
+ vortex-spark_2.13
${vortex.version}
```
@@ -52,7 +54,9 @@ df.write()
.save();
```
-Each Spark partition produces one output file named `part-{partitionId}-{taskId}.vortex`.
+Each write task writes one `part-%05d-%d.vortex` file per partition path it touches (just one file
+for an unpartitioned write); the zero-padded partition id and the task id (e.g. `part-00000-3.vortex`)
+keep the name unique within each path.
### Write Options
diff --git a/docs/user-guide/work-in-progress.md b/docs/user-guide/work-in-progress.md
index 8c8be097676..92828dd8b23 100644
--- a/docs/user-guide/work-in-progress.md
+++ b/docs/user-guide/work-in-progress.md
@@ -13,7 +13,3 @@ Substrait plan consumption for portable query execution over Vortex data.
## cuDF
RAPIDS cuDF integration for GPU-accelerated DataFrames backed by Vortex arrays.
-
-## Trino
-
-Trino connector for querying Vortex tables via JNI.
diff --git a/java/README.md b/java/README.md
index d9554420998..ae0328f646c 100644
--- a/java/README.md
+++ b/java/README.md
@@ -2,16 +2,16 @@
We provide two interfaces for working with Vortex from Java:
-- `vortex-java` - a low-level interface JNI for working with Vortex files and arrays on cloud and local storage
+- `vortex-jni` - a low-level JNI interface for working with Vortex files and arrays on cloud and local storage
- `vortex-spark` - A Spark connector for working with datasets of Vortex files
## Publishing
-We publish three artifacts out of this repo at CI time to Maven Central Sonatype:
+We publish the following artifacts out of this repo at CI time to Maven Central Sonatype:
* `vortex-jni` JAR containing the JNI code, plus compiled native libraries for all of the following targets: `aarch64-apple-darwin`, `aarch64-unknown-linux-gnu`, `x86_64-unknown-linux-gnu`
* `vortex-jni-all` which is the "shadow JAR" containing all of `vortex-jni` as well as all upstream Java dependencies packaged in a single JAR.
-* `vortex-spark` which is the runtime JAR needed for the Vortex Spark bindings
+* `vortex-spark_2.12` and `vortex-spark_2.13` — the runtime JARs needed for the Vortex Spark bindings (the artifact name carries the Scala binary version)
We use the [following GPG key](https://keyserver.ubuntu.com/pks/lookup?search=8745D1A87C0B2159&fingerprint=on&op=index) for publishing:
diff --git a/scripts/check-docs-conformance.py b/scripts/check-docs-conformance.py
new file mode 100755
index 00000000000..6ee6a2084ba
--- /dev/null
+++ b/scripts/check-docs-conformance.py
@@ -0,0 +1,3260 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright the Vortex contributors
+"""Doc-conformance lint: assert that facts stated in docs/ prose still match their source of truth
+in the code/build, so documentation cannot silently drift from the implementation.
+
+Why this exists: Sphinx's `--fail-on-warning` + nitpicky gate the *structure* of the docs (dead
+cross-references, bad roles) but not their *accuracy*. A green build says nothing about whether a
+constant, package coordinate, or API name in prose is still correct. This script closes that gap for
+a curated registry of load-bearing facts.
+
+Design rules (load-bearing):
+- Every check reads BOTH sides at runtime — it derives the canonical value from the code/build source
+ and verifies the doc agrees. Checks MUST NOT hard-code the expected value inline; a check that
+ embedded the answer would itself drift.
+- Matching is token-boundary aware, not raw substring, so a stale value that merely overlaps the
+ claim (`65527` inside `655270`, crate `vortex` inside `vortex-data`) does not false-pass.
+- For facts expressed as a command (`pip install X`, `cargo add X`), the check additionally asserts
+ that EVERY occurrence of the command uses the canonical value — catching a stale duplicate command
+ that coexists with a correct one.
+
+Known limitations (regex matching has irreducible ambiguity for inputs that do not occur in real
+docs, so the matcher is tuned for correctness on realistic prose):
+- For bare-value facts (a number/name not behind a command stem) the check verifies the canonical
+ value is PRESENT; it does not exhaustively prove no stale variant appears elsewhere in the same
+ file. The command-prefix check covers the common stale-duplicate vector.
+- A pathological command argument that is neither a clean package spec nor cleanly delimited (e.g.
+ `pip install pkg[extra]wrong`) is not captured as a token, so it is skipped rather than flagged.
+ This is the irreducible limit of distinguishing a malformed spec from a correct spec with adjacent
+ markup; such inputs do not appear in practice. Broader absence-checking is tracked as Deferred work.
+
+Usage:
+ python scripts/check-docs-conformance.py # verify all checks; exit non-zero on drift
+ python scripts/check-docs-conformance.py --self-test # prove the checker detects drift (negative test)
+ python scripts/check-docs-conformance.py -v # verbose: print every check's resolved values
+"""
+from __future__ import annotations
+
+import argparse
+import ast
+import re
+import subprocess
+import sys
+import tomllib
+from collections.abc import Callable, Iterable
+from dataclasses import dataclass, field
+from pathlib import Path
+
+# The stable-encoding-set derivation lives in a sibling module so the tripwire can `import
+# encoding_stability` directly (this file's hyphenated name is not importable). The scripts/ dir is
+# sys.path[0] when run as __main__; the insert keeps the import robust if that ever changes.
+_SCRIPTS_DIR = Path(__file__).resolve().parent
+if str(_SCRIPTS_DIR) not in sys.path:
+ sys.path.insert(0, str(_SCRIPTS_DIR))
+import encoding_stability # noqa: E402 (must follow the sys.path insert above)
+
+# Token characters: a claim must be delimited by characters OUTSIDE this set to count as present.
+# Word chars and '-' (so `vortex` is not found inside `vortex-data`, nor `65527` inside `655270`).
+_TOKEN = r"[\w\-]"
+
+# A '.' is ambiguous: it can be sentence punctuation (`65527.` at end of a sentence — fine) OR a
+# sub-token separator (`65527.0` version, `vortex-data.old` — a DIFFERENT value that merely shares a
+# prefix). Disambiguate by what follows the dot: a '.' followed by a word/hyphen char continues the
+# token (so the claim is NOT a standalone match there); a '.' followed by anything else is prose.
+_NOT_DOTTED_CONTINUATION = r"(?!\.[\w\-])"
+
+
+def repo_root() -> Path:
+ # Prefer the git top-level; on ANY failure to obtain it — git not installed (FileNotFoundError) or
+ # `git rev-parse` exiting non-zero for any reason, chiefly "not a git working tree"
+ # (CalledProcessError) — fall back to the script's location. That fallback is correct in every case
+ # because the script lives at /scripts/, so the repo root is its parent's parent.
+ try:
+ out = subprocess.run(
+ ["git", "rev-parse", "--show-toplevel"],
+ capture_output=True, text=True, check=True,
+ ).stdout.strip()
+ return Path(out)
+ except (FileNotFoundError, subprocess.CalledProcessError):
+ # Any failure to get the git top-level — git not installed (FileNotFoundError) or `git rev-parse`
+ # exiting non-zero, chiefly "not a git working tree" (CalledProcessError) — falls back to the
+ # script's location. That fallback is correct for ALL such cases because the script lives at
+ # /scripts/, so the repo root is its parent's parent regardless of why git was unavailable.
+ return Path(__file__).resolve().parent.parent
+
+
+def _toml_str_from(text: str, label: str, *keys: str | int) -> str:
+ """Navigate `keys` (dict names or list indices, e.g. `"bin", 0, "name"`) into TOML `text`, parsed
+ with `tomllib`, and return the string there. Robust against comments, key ordering, and list-valued
+ keys that a regex over the raw text mishandles; raises if the path is missing or non-string.
+ NOTE: `"bin", 0` selects the FIRST `[[bin]]` target — the user-facing CLI binary in a single-binary
+ crate like vortex-tui; revisit if a crate ships multiple binaries."""
+ node = tomllib.loads(text)
+ for k in keys:
+ try:
+ node = node[k]
+ except (KeyError, IndexError, TypeError) as e:
+ raise LookupError(f"{label}: no TOML value at path {list(keys)} ({type(e).__name__})") from e
+ if not isinstance(node, str):
+ raise LookupError(f"{label}: TOML path {list(keys)} is {type(node).__name__}, not a string")
+ return node
+
+
+def _toml_str(root: Path, rel: str, *keys: str | int) -> str:
+ """`_toml_str_from` over the file at `rel` (see that function for the navigation contract)."""
+ return _toml_str_from((root / rel).read_text(encoding="utf-8"), rel, *keys)
+
+
+def _strip_comments(text: str) -> str:
+ """Remove C-style block (`/* */`) and line (`//`) comments so a commented-out decoy declaration
+ cannot be sourced as the canonical value (the regexes below also anchor to `^\\s*`, which a
+ `//`/`#`-prefixed line cannot satisfy). TOML `#` comments need no stripping — a `#`-prefixed line
+ fails the `^name`-anchored regexes — and mangling a `//` inside a URL/string is harmless here
+ because the patterns match declaration lines, not URLs."""
+ text = re.sub(r"/\*.*?\*/", "", text, flags=re.DOTALL)
+ return re.sub(r"//[^\n]*", "", text)
+
+
+def _strip_rust_comments(src: str) -> str:
+ """Remove `//` line comments and NESTED `/* */` block comments from Rust source — Rust block
+ comments nest, which the non-greedy `_strip_comments` regex mishandles (a commented marker inside an
+ outer comment can survive). Used where a decoy hidden in a nested comment must NOT satisfy a check
+ (e.g. the `#[cxx::bridge]` detection in `_cxx_dep`). String-literal contents are not modeled — the
+ markers it guards never appear in string literals."""
+ out: list[str] = []
+ i, n, depth = 0, len(src), 0
+ while i < n:
+ two = src[i:i + 2]
+ if depth == 0 and two == "//":
+ j = src.find("\n", i)
+ i = n if j < 0 else j # skip to the newline (kept on the next iteration)
+ elif two == "/*":
+ depth += 1
+ i += 2
+ elif two == "*/" and depth > 0:
+ depth -= 1
+ i += 2
+ elif depth > 0:
+ i += 1
+ else:
+ out.append(src[i])
+ i += 1
+ return "".join(out)
+
+
+def _read_const(text: str, pattern: str, label: str) -> int:
+ """Read exactly one integer constant matched by `pattern` (group 1) in `text` (comments stripped);
+ raise if missing or duplicated, so a derived fact fails loud rather than silently sourcing a decoy."""
+ matches = re.findall(pattern, _strip_comments(text), re.MULTILINE)
+ if len(matches) != 1:
+ raise LookupError(f"expected exactly one {label} matching {pattern!r}, found {len(matches)}")
+ return int(matches[0])
+
+
+def _derive_initial_read(root: Path) -> str:
+ """The file reader's first-read size = MAX_POSTSCRIPT_SIZE + EOF_SIZE. Read BOTH constituent
+ constants from vortex-file/src/lib.rs (so a change to EITHER is reflected, not cancelled), and
+ verify vortex-file/src/open.rs still defines INITIAL_READ_SIZE as their sum — so the doc's value
+ tracks the actual const, and a broken relationship is caught rather than assumed."""
+ lib = (root / "vortex-file/src/lib.rs").read_text(encoding="utf-8")
+ eof = _read_const(lib, r"^\s*pub const EOF_SIZE: usize = (\d+)\s*;", "EOF_SIZE")
+ max_postscript = 65535 - _read_const(lib, r"^\s*pub const MAX_POSTSCRIPT_SIZE: u16 = u16::MAX - (\d+)\s*;",
+ "MAX_POSTSCRIPT_SIZE subtrahend")
+ open_rs = _strip_rust_comments((root / "vortex-file/src/open.rs").read_text(encoding="utf-8"))
+ # Anchor to start-of-line + end-of-statement (`;`) so neither a commented decoy nor a CHANGED
+ # relationship (e.g. `... + EOF_SIZE + 1`) prefix-matches and silently passes.
+ if not re.search(r"^\s*const INITIAL_READ_SIZE: usize = MAX_POSTSCRIPT_SIZE as usize \+ EOF_SIZE\s*;",
+ open_rs, re.MULTILINE):
+ raise LookupError(
+ "INITIAL_READ_SIZE is no longer defined as `MAX_POSTSCRIPT_SIZE as usize + EOF_SIZE` in "
+ "vortex-file/src/open.rs — the first-read relationship moved; update the registry.")
+ return str(max_postscript + eof)
+
+
+def _derive_spark_artifact(root: Path) -> str:
+ # Policy: the docs recommend the artifact for the HIGHEST published Scala binary version (latest
+ # Scala — currently 2.13); `_spark_scala_variants` enumerates the published suffixes and this picks
+ # the max. If the recommended-version policy changes, update here and the docs together.
+ """The Spark connector's Maven artifact id the docs should advertise — the LATEST scala-suffixed
+ variant, derived (not hard-coded) from the `vortex-spark_` subprojects declared in
+ java/settings.gradle.kts, and cross-checked against the build's `artifactId = "vortex-spark_
+ $scalaVersion"` pattern. Catches a doc that drops the `_` suffix (copy-paste-broken
+ coordinate) or that pins a scala version the build no longer publishes."""
+ settings = _strip_comments((root / "java/settings.gradle.kts").read_text(encoding="utf-8"))
+ # Scala binary suffix may be `2.13` (Scala 2.x) OR dotless `3` (Scala 3) — allow both.
+ variants = re.findall(r'include\("vortex-spark_(\d+(?:\.\d+)?)"\)', settings)
+ if not variants:
+ raise LookupError("no `vortex-spark_` subprojects in java/settings.gradle.kts")
+ gradle = _strip_comments((root / "java/vortex-spark/build.gradle.kts").read_text(encoding="utf-8"))
+ if not re.search(r'artifactId\s*=\s*"vortex-spark_\$scalaVersion"', gradle):
+ raise LookupError("vortex-spark artifactId is no longer the scala-suffixed form; update registry")
+ latest = max(variants, key=lambda v: tuple(int(x) for x in v.split(".")))
+ return f"vortex-spark_{latest}"
+
+
+def _spark_major_for_scala(root: Path, scala: str) -> str:
+ """The Spark major version the `vortex-spark_` artifact targets, from the scala->spark when-arm
+ in vortex-spark/build.gradle.kts (`"" -> { ... libs.versions.spark ... }`). Returns
+ `Spark .x` to anchor spark.md's targeting claim; FAILS LOUD if the arm/mapping changes."""
+ src = _strip_comments((root / "java/vortex-spark/build.gradle.kts").read_text(encoding="utf-8"))
+ m = re.search(rf'"{re.escape(scala)}"\s*->\s*\{{[^}}]*?libs\.versions\.spark(\d+)', src, re.DOTALL)
+ if not m:
+ raise LookupError(f"no scala {scala} -> spark when-arm in vortex-spark/build.gradle.kts; spark.md is stale")
+ return f"Spark {m.group(1)}.x"
+
+
+def _derive_spark_filename(root: Path) -> str:
+ """The Spark writer's output filename format. BOTH the partitioned and unpartitioned writers name
+ files via `String.format("part-…")`; read both and require they agree (so the documented value
+ tracks whichever path the example uses, and a divergence between the two fails loud)."""
+ writers = ["java/vortex-spark/src/main/java/dev/vortex/spark/write/PartitionedVortexDataWriter.java",
+ "java/vortex-spark/src/main/java/dev/vortex/spark/write/VortexDataWriterFactory.java"]
+ fmts = set()
+ for w in writers:
+ m = re.search(r'String\.format\("(part-[%\dd-]+\.vortex)"',
+ _strip_comments((root / w).read_text(encoding="utf-8")))
+ if not m:
+ raise LookupError(f"no `part-*.vortex` String.format in {w}")
+ fmts.add(m.group(1))
+ if len(fmts) != 1:
+ raise LookupError(f"Spark writers disagree on the output filename format: {sorted(fmts)}")
+ return fmts.pop()
+
+
+def _parse_python_min_version(spec: str) -> str:
+ """Extract the effective minimum Python version from a `requires-python` spec, anchored to the `>=`
+ lower bound(s) — NOT the first version token or the upper bound. The minimum is the HIGHEST `>=`
+ clause (multiple lower bounds intersect: `>=3.10,>=3.11` requires 3.11), compared as a PEP-440-ish
+ numeric tuple so ordering is value-based, not lexical. The FULL lower-bound version is kept, incl. a
+ patch level (`>=3.11.4` -> `Python 3.11.4`), so a docs claim of "3.11" can't understate a real
+ "3.11.4" minimum. Only `>=` (lower) and `<`/`<=` (upper, ignored) operators are reasoned about;
+ ANY other operator (`>`, `==`, `===`, `!=`, `~=`) or a non-numeric lower bound (`>=3.11.0rc1`,
+ `>=3.11.*`) fails loud — because exclusions / compatible-release / pre-release clauses can raise or
+ qualify the effective minimum, and silently understating it would defeat the lock. The live spec is
+ `>= 3.11`; an exotic future spec gets a clear failure telling the maintainer to extend this.
+ Separated from the file read so the self-test can exercise it on synthetic specs."""
+ numeric: list[str] = []
+ suffixed: list[str] = []
+ for clause in spec.split(","):
+ c = clause.strip()
+ if not c:
+ continue
+ m = re.match(r"(>=|<=|===|==|!=|~=|>|<)\s*(\S+)$", c)
+ if not m:
+ raise LookupError(f"unparseable requires-python clause {c!r}")
+ op, v = m.group(1), m.group(2)
+ if op in ("<", "<="):
+ continue # an upper bound — does not affect the minimum
+ if op != ">=":
+ # `>`, `==`, `===`, `!=`, `~=` can each set or RAISE the effective minimum (e.g.
+ # `>=3.11,!=3.11.*` really requires 3.12) in ways a simple lower-bound scan would
+ # understate. Refuse to guess — fail loud so a maintainer extends this derivation.
+ raise LookupError(f"requires-python operator {op!r} in {c!r} not supported; handle explicitly")
+ (numeric if re.fullmatch(r"\d+\.\d+(?:\.\d+)*", v) else suffixed).append(v)
+ if suffixed: # a pre-release / wildcard lower bound (`>=3.11.0rc1`, `>=3.11.*`) — don't guess how
+ raise LookupError(f"requires-python lower bound(s) {suffixed!r} not plain numeric; handle explicitly")
+ if not numeric:
+ raise LookupError(f"could not parse a `>=` minimum from requires-python = {spec!r}")
+ best = max(numeric, key=lambda v: tuple(int(p) for p in v.split(".")))
+ return f"Python {best}"
+
+
+def _derive_python_version(root: Path) -> str:
+ """The minimum Python version the docs advertise — sourced from pyproject `requires-python`."""
+ rp = _toml_str(root, "vortex-python/pyproject.toml", "project", "requires-python") # e.g. ">= 3.11"
+ return _parse_python_min_version(rp)
+
+
+def _eval_byte_expr(expr: str) -> int:
+ """Evaluate a simple Rust byte-size literal: `1 << 20`, `8 * 1024`, or a plain int. Fails loud on
+ anything more complex so an unexpected expression can't be silently mis-sized."""
+ expr = expr.strip()
+ if (m := re.fullmatch(r"(\d+)\s*<<\s*(\d+)", expr)):
+ return int(m.group(1)) << int(m.group(2))
+ if (m := re.fullmatch(r"(\d+)\s*\*\s*(\d+)", expr)):
+ return int(m.group(1)) * int(m.group(2))
+ if re.fullmatch(r"\d+", expr):
+ return int(expr)
+ raise LookupError(f"cannot evaluate byte expression {expr!r}")
+
+
+def _bytes_to_human(n: int) -> str:
+ """Format a power-of-two byte count as ` MB` / ` KB` (matching the docs' prose), so a code
+ constant like `4 << 20` locks the doc string `4 MB`."""
+ if n and n % (1 << 20) == 0:
+ return f"{n >> 20} MB"
+ if n and n % (1 << 10) == 0:
+ return f"{n >> 10} KB"
+ return str(n)
+
+
+def _coalesce_bytes(root: Path, fn_name: str) -> tuple[int, int]:
+ """The (distance, max_size) byte pair for `CoalesceConfig::()` in vortex-io read_at.rs.
+ Requires EXACTLY ONE matching constructor (comments stripped first) so a same-shaped decoy can't
+ silently mis-source the value."""
+ src = _strip_rust_comments((root / "vortex-io/src/read_at.rs").read_text(encoding="utf-8"))
+ ms = re.findall(rf"fn {fn_name}\(\)\s*->\s*Self\s*\{{\s*Self::new\(([^)]*)\)", src)
+ if len(ms) != 1:
+ raise LookupError(f"expected exactly 1 CoalesceConfig::{fn_name}() in read_at.rs, found {len(ms)}")
+ args = [a for a in ms[0].split(",") if a.strip()]
+ if len(args) != 2: # exactly (distance, max_size) — neither fewer nor extra args
+ raise LookupError(f"CoalesceConfig::{fn_name}() had {len(args)} args, expected (distance, max_size)")
+ return _eval_byte_expr(args[0]), _eval_byte_expr(args[1])
+
+
+def _derive_buffered_block_size(root: Path) -> str:
+ """The BufferedStrategy localization size from vortex-file strategy.rs (`N * ONE_MEG`), as ` MB`
+ — `ONE_MEG` is sourced from the same file so the unit can't drift from the constant."""
+ src = _strip_rust_comments((root / "vortex-file/src/strategy.rs").read_text(encoding="utf-8"))
+ one_meg = re.findall(r"const ONE_MEG:\s*u64\s*=\s*([^;]+);", src)
+ mult = re.findall(r"BufferedStrategy::new\([^,]+,\s*(\d+)\s*\*\s*ONE_MEG\)", src)
+ if len(one_meg) != 1 or len(mult) != 1: # exactly one of each, else a decoy could mis-source
+ raise LookupError(
+ f"expected 1 ONE_MEG const and 1 BufferedStrategy::new in strategy.rs, "
+ f"found {len(one_meg)} and {len(mult)}")
+ return _bytes_to_human(int(mult[0]) * _eval_byte_expr(one_meg[0]))
+
+
+def _cxx_dep(root: Path) -> str:
+ """Confirm vortex-cxx uses the `cxx` crate as a direct Rust<->C++ bridge — the mechanism the C++
+ docs describe. Requires BOTH the `cxx` dependency in Cargo.toml AND a live `#[cxx::bridge]` in the
+ source (so a leftover dependency after a migration to wrapping the C FFI cannot satisfy the lock).
+ Returns the literal `cxx` to anchor the docs' present-check; FAILS LOUD if either is gone, signaling
+ the C++ docs must be re-described."""
+ cxx_dir = root / "vortex-cxx"
+ data = tomllib.loads((cxx_dir / "Cargo.toml").read_text(encoding="utf-8"))
+ if "cxx" not in data.get("dependencies", {}):
+ raise LookupError("vortex-cxx no longer depends on `cxx`; the C++ binding mechanism changed — update the docs")
+ src = _strip_rust_comments("\n".join(p.read_text(encoding="utf-8") for p in (cxx_dir / "src").rglob("*.rs")))
+ # Anchor to ITEM position (start-of-line + optional indent), so neither a commented decoy (already
+ # stripped) nor a string literal like `const D: &str = "#[cxx::bridge]"` (the `"` breaks the anchor)
+ # can satisfy the lock after the real bridge attribute is removed.
+ if not re.search(r"^[ \t]*#\[cxx::bridge", src, re.MULTILINE):
+ raise LookupError("vortex-cxx has no live `#[cxx::bridge]`; the C++ binding mechanism changed")
+ return "cxx"
+
+
+def _convert_batch_size(root: Path) -> str:
+ """The `vx convert` row-batch size (vortex-tui `BATCH_SIZE`) the CLI quickstart cites — read from the
+ convert command's own constant so the doc tracks it (fails loud if missing/duplicated)."""
+ txt = (root / "vortex-tui/src/convert.rs").read_text(encoding="utf-8")
+ return str(_read_const(txt, r"^\s*pub const BATCH_SIZE: usize = (\d+)\s*;", "BATCH_SIZE"))
+
+
+def _spark_filter_pushdown(root: Path) -> str:
+ """Confirm the Spark connector implements filter pushdown — VortexScanBuilder implements Spark's
+ `SupportsPushDownV2Filters`. Returns that interface name (the doc must mention it); FAILS LOUD if the
+ connector drops it, signaling integrations/spark.md's filter-pushdown claim is stale."""
+ sb = root / "java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexScanBuilder.java"
+ src = _strip_rust_comments(sb.read_text(encoding="utf-8")) # `//` + `/* */`, same as Java comments
+ # require it in the class's `implements` list, not merely imported or mentioned in a comment
+ if not re.search(r"implements\b[^{]*\bSupportsPushDownV2Filters\b", src, re.DOTALL):
+ raise LookupError("VortexScanBuilder does not implement SupportsPushDownV2Filters; spark.md is stale")
+ return "SupportsPushDownV2Filters"
+
+
+def _duckdb_replacement_scan(root: Path) -> str:
+ """Confirm the DuckDB extension registers a replacement scan (what makes `FROM 'data.vortex'` work).
+ Returns "replacement scan"; FAILS LOUD if the registration is gone, signaling duckdb.md's direct-path
+ claim is stale."""
+ src = _strip_rust_comments((root / "vortex-duckdb/src/lib.rs").read_text(encoding="utf-8"))
+ # Scope to the extension-load entrypoint `initialize_extension_from_raw` so a helper/test call elsewhere
+ # can't satisfy the check while the actual extension-load path never registers the replacement scan.
+ init = re.search(r"fn initialize_extension_from_raw\([^)]*\)\s*\{(.*?)\n\}", src, re.DOTALL)
+ if not init:
+ raise LookupError("could not find the `initialize_extension_from_raw` entrypoint in vortex-duckdb/src/lib.rs")
+ # require an actual CALL `.register_vortex_scan_replacement()` INSIDE the entrypoint, not just a mention
+ if not re.search(r"\.register_vortex_scan_replacement\s*\(\s*\)", init.group(1)):
+ raise LookupError("the extension entrypoint does not call register_vortex_scan_replacement(); duckdb.md is stale")
+ return "replacement scan"
+
+
+def _io_reader_impl(root: Path, name: str, rel: str) -> str:
+ """Confirm `name` is a struct that `impl VortexReadAt` in `rel` — the reader-type names io.md cites for
+ local-file (`FileReadAt`) and object-store (`ObjectStoreReadAt`) reads. FAILS LOUD if it stops
+ implementing the trait (renamed/removed), so io.md's reader names can't silently drift. Comments stripped."""
+ src = _strip_rust_comments((root / rel).read_text(encoding="utf-8"))
+ if not re.search(rf"\bimpl VortexReadAt for {re.escape(name)}\b", src):
+ raise LookupError(f"{name} no longer `impl VortexReadAt` in {rel}; io.md's reader name is stale")
+ return name
+
+
+def _jni_module_name(root: Path) -> str:
+ """The JNI Gradle module/artifact name (java/settings.gradle.kts `include(...)` whose name contains
+ `jni`, cross-checked against that module's build.gradle.kts `artifactId`) — the source of truth for the
+ `vortex-jni` name java/README.md cites. FAILS LOUD if the module is renamed; the README must then update
+ (and the renamed-away `vortex-java` is forbidden)."""
+ settings = _strip_comments((root / "java/settings.gradle.kts").read_text(encoding="utf-8"))
+ jni = {m for m in re.findall(r'include\("([\w-]+)"\)', settings) if "jni" in m}
+ if len(jni) != 1: # exactly one jni module, else the derivation is ambiguous
+ raise LookupError(f"expected exactly 1 jni Gradle module in java/settings.gradle.kts, found {sorted(jni)}")
+ name = jni.pop()
+ bg = (root / "java" / name / "build.gradle.kts").read_text(encoding="utf-8")
+ if not re.search(rf'artifactId\s*=\s*"{re.escape(name)}"', bg):
+ raise LookupError(f"java/{name}/build.gradle.kts artifactId != {name!r}; the JNI module name is inconsistent")
+ return name
+
+
+def _io_in_memory_coalesce_behavior(root: Path) -> str:
+ """Back io.md's "in-memory coalescing is opt-in" claim with the one cleanly-lockable code fact: the
+ `VortexReadAt` default `coalesce_config()` returns `None`, so a reader that doesn't override it does
+ not coalesce. (That some readers opt into the 8 KB preset is a general statement; the 8 KB VALUE
+ itself is locked separately by `io-coalesce-in-memory`.) Comments stripped; fails loud if it changes."""
+ default_src = _strip_rust_comments((root / "vortex-io/src/read_at.rs").read_text(encoding="utf-8"))
+ if not re.search(r"fn coalesce_config\(&self\)\s*->\s*Option\s*\{\s*None\s*\}", default_src):
+ raise LookupError("VortexReadAt default coalesce_config() no longer returns None; io.md is stale")
+ return "opt-in"
+
+
+def _scan_crate_name(root: Path) -> str:
+ """Confirm the `vortex-scan` crate exists — the Scan API the roadmap says is present today. Returns
+ its package name; FAILS LOUD if the crate is gone, signaling the roadmap claim is stale."""
+ cargo = root / "vortex-scan/Cargo.toml"
+ if not cargo.exists():
+ raise LookupError("vortex-scan crate is gone; the roadmap's 'Scan API exists today' claim is stale")
+ name = _toml_str_from(cargo.read_text(encoding="utf-8"), "vortex-scan/Cargo.toml", "package", "name")
+ if name != "vortex-scan":
+ raise LookupError(f"vortex-scan package name is {name!r}, not 'vortex-scan'")
+ # The roadmap/scanning docs credit the `DataSource` and `Partition` traits — verify both are live
+ # (`\b` so `DataSourceOpener`/`DataSourceScan` don't satisfy DataSource); comments stripped.
+ lib = _strip_rust_comments((root / "vortex-scan/src/lib.rs").read_text(encoding="utf-8"))
+ for trait in ("DataSource", "Partition"):
+ if not re.search(rf"pub trait {trait}\b", lib):
+ raise LookupError(f"vortex-scan has no `pub trait {trait}`; the scan-API docs are stale")
+ return name
+
+
+def _dtype_has_variant(enum_body: str) -> bool:
+ """Whether the `DType` enum body declares a `Variant` variant. A pure helper so the presence check
+ is self-testable on synthetic input."""
+ return bool(re.search(r"^\s*Variant\b", enum_body, re.MULTILINE))
+
+
+def _workspace_crate_names(root: Path) -> set[str]:
+ """Every crate name in the workspace (each Cargo.toml `[package] name`). The allowed set for the
+ crate references the architecture overview advertises (catches a fabricated/renamed crate like the
+ former `vortex-roaring`/`vortex-dict`/`vortex-expr`). Fails loud if implausibly few are found."""
+ names: set[str] = set()
+ for cargo in root.rglob("Cargo.toml"):
+ if "/target/" in str(cargo) or "/.git/" in str(cargo):
+ continue
+ m = re.search(r'^\s*name\s*=\s*"([^"]+)"', cargo.read_text(encoding="utf-8"), re.MULTILINE)
+ if m:
+ names.add(m.group(1))
+ if len(names) < 10:
+ raise LookupError(f"found only {len(names)} workspace crates; the scan may be broken")
+ return names
+
+
+def _encoding_crate_names(root: Path) -> set[str]:
+ """Every encoding crate published under `encodings/*` (its Cargo.toml `[package] name`). The source
+ of truth for the architecture overview's Encodings table. Fails loud if none are found."""
+ names: set[str] = set()
+ for cargo in sorted((root / "encodings").glob("*/Cargo.toml")):
+ m = re.search(r'^\s*name\s*=\s*"([^"]+)"', cargo.read_text(encoding="utf-8"), re.MULTILINE)
+ if m:
+ names.add(m.group(1))
+ if not names:
+ raise LookupError("no encoding crates found under encodings/*")
+ return names
+
+
+def _dtype_variant_names(root: Path) -> set[str]:
+ """Every variant name of the `DType` enum (vortex-array/src/dtype/mod.rs) — the source of truth for
+ the architecture overview's DType list. Comments stripped (nested-aware); fails loud if unparsed."""
+ src = _strip_rust_comments((root / "vortex-array/src/dtype/mod.rs").read_text(encoding="utf-8"))
+ m = re.search(r"pub enum DType\s*\{(.*?)\n\}", src, re.DOTALL)
+ if not m:
+ raise LookupError("could not find `pub enum DType` in vortex-array/src/dtype/mod.rs")
+ names = set(re.findall(r"^\s*([A-Z]\w*)\s*[({,]", m.group(1), re.MULTILINE))
+ if len(names) < 8:
+ raise LookupError(f"parsed only {len(names)} DType variants; the enum parse may be broken")
+ return names
+
+
+def _variant_dtype_present(root: Path) -> str:
+ """Confirm `DType::Variant` exists — the roadmap says the Variant DType is present today. Returns
+ `Variant`; FAILS LOUD if the variant is gone/renamed, signaling the roadmap claim is stale."""
+ src = _strip_rust_comments((root / "vortex-array/src/dtype/mod.rs").read_text(encoding="utf-8"))
+ m = re.search(r"pub enum DType\s*\{(.*?)\n\}", src, re.DOTALL)
+ if not m:
+ raise LookupError("could not find `pub enum DType` in vortex-array/src/dtype/mod.rs")
+ if not _dtype_has_variant(m.group(1)):
+ raise LookupError("DType has no `Variant` variant; the roadmap's 'Variant present today' claim is stale")
+ return "Variant"
+
+
+def _union_is_sole_noncanonical(root: Path) -> str:
+ """Back the canonical-arrays prose "every DType has a canonical encoding except `Union`"
+ (concepts/arrays.md + api/python/arrays.rst): assert the `DType` enum HAS a `Union` variant (so the
+ exception names a real type) AND the `Canonical` enum has NO `Union` variant or `UnionArray` payload
+ (so `Union` is genuinely un-canonicalized). FAILS LOUD if `Union` gains a canonical encoding — the
+ docs must then drop the exception. (That `Union` is the SOLE exception is not fully code-derivable
+ here — the DType->canonical map is many-to-one; see deferred work.) Returns `Union` to anchor the
+ docs' present-check."""
+ if "Union" not in _dtype_variant_names(root):
+ raise LookupError("DType enum has no `Union` variant; the arrays canonical-exception prose is stale")
+ canon = _strip_rust_comments((root / "vortex-array/src/canonical.rs").read_text(encoding="utf-8"))
+ m = re.search(r"pub enum Canonical\s*\{(.*?)\n\}", canon, re.DOTALL)
+ if not m:
+ raise LookupError("could not find `pub enum Canonical` in vortex-array/src/canonical.rs")
+ body = m.group(1)
+ if re.search(r"^\s*Union\b", body, re.MULTILINE) or "UnionArray" in body:
+ raise LookupError(
+ "Canonical enum now covers Union; concepts/arrays.md + arrays.rst must drop the 'except Union' claim")
+ return "Union"
+
+
+def _derive_spark_batch_range(root: Path) -> str:
+ """The Spark writer's batch-size range string `MIN–MAX` (en-dash), reading BOTH bounds from
+ VortexDataWriter.java so the documented `(1–65536)` tracks both constants."""
+ java = (root / "java/vortex-spark/src/main/java/dev/vortex/spark/write/VortexDataWriter.java"
+ ).read_text(encoding="utf-8")
+ # Anchor to a real field declaration (`^\s*...final int NAME = N;`) so a non-literal RHS
+ # (`= 2 - 1`) or a commented decoy cannot mis-source.
+ lo = _read_const(java, r"^\s*(?:private\s+)?static\s+final\s+int\s+MIN_BATCH_SIZE\s*=\s*(\d+)\s*;",
+ "MIN_BATCH_SIZE")
+ hi = _read_const(java, r"^\s*(?:private\s+)?static\s+final\s+int\s+MAX_BATCH_SIZE\s*=\s*(\d+)\s*;",
+ "MAX_BATCH_SIZE")
+ return f"{lo}–{hi}" # en-dash, matching the docs' `(1–65536)`
+
+
+def all_doc_files(root: Path) -> list[Path]:
+ """Every Markdown/reStructuredText doc under docs/, excluding the Sphinx build output."""
+ docs = root / "docs"
+ return sorted(
+ p for p in docs.rglob("*")
+ if p.suffix in (".md", ".rst") and "_build" not in p.parts
+ )
+
+
+def token_present(claim: str, text: str) -> bool:
+ """True iff `claim` appears in `text` delimited by non-token characters on both sides, and is not
+ a dotted sub-component of a longer version/identifier — neither a suffix (`65527` in `65527.0`)
+ nor a prefix (`65527` in `1.65527`). A leading '.' is always a continuation (prose never starts a
+ standalone token with an attached dot), so the leading guard simply forbids a preceding '.'."""
+ pattern = (
+ r"(?`): a package-shaped token — word/
+# hyphen chars, optional dotted sub-tokens, and an optional `[extras]` selector — followed by a token
+# boundary so the match isn't a truncated prefix of a longer junk run (`pkg[extra]wrong`). Dotted
+# sub-tokens are captured (so a malformed `vortex-data.old` is seen whole and rejected), but a lone
+# trailing '.' is not (so `pip install pkg.` at a sentence end is not absorbed and does not false-fail).
+_COMMAND_TOKEN = r"([\w\-]+(?:\.[\w\-]+)*(?:\[[^\]]*\])?)(?![\w\-.\[])"
+
+
+def command_tokens(prefix: str, text: str) -> list[str]:
+ """All package arguments following `prefix` in `text` (e.g. every `pip install `), skipping
+ any leading short/long flags (`pip install --upgrade -U `) so the package — not a flag — is
+ the captured token. (A flag that itself consumes a value, e.g. `--index-url URL`, is the documented
+ pathological residual and is not handled.)"""
+ return re.findall(re.escape(prefix) + r"(?:-{1,2}[\w\-]+\s+)*" + _COMMAND_TOKEN, text)
+
+
+def command_token_ok(value: str, tok: str, allow_extras: bool) -> bool:
+ """A captured command token is consistent with the canonical value iff it equals it exactly, or —
+ only when `allow_extras` (a pip/uvx-style ecosystem) — is the value followed by EXACTLY one
+ `[extras]` selector and nothing else (e.g. `pip install pkg[polars,ray]`). Using a full match (not
+ a prefix test) means `pkg[extra]wrong` or `pkg.old` is rejected; gating extras on `allow_extras`
+ means `cargo add pkg[bogus]` is rejected (Cargo has no pip-style extras)."""
+ if tok == value:
+ return True
+ return allow_extras and re.fullmatch(re.escape(value) + r"\[[^\]]*\]", tok) is not None
+
+
+@dataclass
+class ValueMatch:
+ """A fact whose canonical value is derived from `truth_file` (group 1 of `truth_regex`, optionally
+ mapped through `transform`) and must appear, via `claim_template`, in every file in `doc_files`.
+
+ When `command_prefixes` is set, every `` ANYWHERE under docs/ whose token begins
+ with `command_base` must be consistent with the canonical value (exactly the value, or — when
+ `command_extras` — the value plus one `[extras]` selector). The `command_base` scoping is what
+ lets the global scan ignore unrelated installs (`pip install pandas`) while still catching a
+ drifted/typo'd Vortex package name (`pip install vortex-dat`).
+ """
+
+ id: str
+ description: str
+ doc_files: list[str]
+ # Single-source path: derive the value from group 1 of `truth_regex` in `truth_file` (optionally
+ # mapped through `transform`). Multi-source path: set `derive` to compute the value from the repo
+ # directly (e.g. summing two constants) — `truth_file`/`truth_regex` are then unused.
+ truth_file: str = ""
+ truth_regex: str = ""
+ claim_template: str = "{value}"
+ transform: Callable[[str], str] | None = None
+ derive: Callable[[Path], str] | None = None
+ command_prefixes: list[str] = field(default_factory=list)
+ command_base: str = "" # only command tokens starting with this stem are validated
+ command_extras: bool = False # whether `value[extras]` is valid syntax for this ecosystem
+ forbid_regex: str = "" # if set, the check FAILS when this pattern appears in any doc_file
+ # (an absence check — e.g. a stale unsuffixed coordinate variant)
+
+ def __post_init__(self) -> None:
+ # Exactly one source mode: a `derive` callable, OR a (truth_file, truth_regex) pair. This
+ # catches a malformed future entry at import time rather than letting it report an empty source.
+ if self.derive is not None:
+ if self.truth_file or self.truth_regex:
+ raise ValueError(f"[{self.id}] set EITHER derive OR truth_file/truth_regex, not both")
+ if self.transform is not None:
+ raise ValueError(f"[{self.id}] `transform` is ignored for a derive fact; fold it into derive")
+ elif not (self.truth_file and self.truth_regex):
+ raise ValueError(f"[{self.id}] needs a derive callable OR both truth_file and truth_regex")
+
+ @property
+ def source_label(self) -> str:
+ """Human-readable source for diagnostics — the truth file, or `` for a derive fact."""
+ return self.truth_file or ""
+
+ def resolve_truth(self, root: Path) -> str:
+ if self.derive is not None:
+ return self.derive(root)
+ # Route Rust truth files through the nested-block-comment-aware stripper so every Rust-source
+ # lock shares ONE decoy model (a nested `/* /* */ */` can't smuggle a decoy past the check);
+ # non-Rust files keep the simple `//` + non-nested `/* */` stripper.
+ raw = (root / self.truth_file).read_text(encoding="utf-8")
+ text = _strip_rust_comments(raw) if self.truth_file.endswith(".rs") else _strip_comments(raw)
+ # `re.search` takes the FIRST match; `truth_regex` MUST be unique enough that the first match is
+ # the canonical source — anchor it to `^\s*` (re.MULTILINE) so a commented decoy or a
+ # later same-shaped line cannot win. Comments are already stripped above. A genuinely
+ # multi-source fact belongs in a `derive` with an explicit uniqueness check (see _derive_*).
+ m = re.search(self.truth_regex, text, re.MULTILINE)
+ if not m:
+ raise LookupError(
+ f"[{self.id}] truth pattern {self.truth_regex!r} not found in {self.truth_file} — "
+ f"the source of truth moved; update the registry."
+ )
+ raw = m.group(1)
+ return self.transform(raw) if self.transform else raw
+
+ def claim_for(self, value: str) -> str:
+ return self.claim_template.format(value=value)
+
+ def check(self, root: Path, *, override_value: str | None = None) -> tuple[bool, str]:
+ """Return (ok, detail). When override_value is set (self-test), use it instead of the real
+ truth — used to prove a drifted value would be caught."""
+ value = override_value if override_value is not None else self.resolve_truth(root)
+ claim = self.claim_for(value)
+
+ missing = [
+ f for f in self.doc_files
+ if not token_present(claim, (root / f).read_text(encoding="utf-8"))
+ ]
+ if missing:
+ return False, f"claim {claim!r} (from {self.source_label}) missing in: {', '.join(missing)}"
+
+ # Absence check: a forbidden pattern (e.g. a stale unsuffixed coordinate variant) must NOT
+ # appear in any doc_file — so a presence check passing on one corrected site can't mask a
+ # stale sibling site that still carries the old form.
+ if self.forbid_regex:
+ for f in self.doc_files:
+ if re.search(self.forbid_regex, (root / f).read_text(encoding="utf-8")):
+ return False, f"forbidden pattern {self.forbid_regex!r} (stale form) still present in {f}"
+
+ # Command-prefix absence check is GLOBAL but stem-scoped: every `` ANYWHERE
+ # under docs/ whose token begins with `command_base` must use the canonical value. Scanning
+ # the whole tree catches a stale install in an unlisted page; the stem scoping ignores
+ # unrelated installs (`pip install pandas`) so they don't false-fail.
+ if self.command_prefixes:
+ base = self.command_base or value
+ for p in all_doc_files(root):
+ text = p.read_text(encoding="utf-8")
+ for prefix in self.command_prefixes:
+ for tok in command_tokens(prefix, text):
+ if not tok.startswith(base):
+ continue # an unrelated package, not a Vortex install claim
+ if not command_token_ok(value, tok, self.command_extras):
+ return False, (
+ f"stale command in {p.relative_to(root)}: {prefix!r} uses {tok!r}, "
+ f"expected {value!r} (from {self.source_label})"
+ )
+ return True, f"claim {claim!r} present in {len(self.doc_files)} doc(s) + commands consistent site-wide"
+
+
+# --- Registry ----------------------------------------------------------------
+# Phase 1 seeds CURRENTLY-CORRECT facts so the lint is green on landing; Phase 2 appends one entry per
+# drifted fact as it is fixed. Each entry derives both sides at runtime (no hard-coded expected values).
+REGISTRY: list[ValueMatch] = [
+ ValueMatch(
+ id="pypi-package-name",
+ description="PyPI distribution name + install commands match vortex-python/pyproject.toml",
+ # tomllib path (not an unscoped `^name`) so it is independent of table order in the file.
+ derive=lambda root: _toml_str(root, "vortex-python/pyproject.toml", "project", "name"),
+ doc_files=["docs/getting-started/install.md", "docs/api/python/index.rst"],
+ command_prefixes=["pip install ", "uvx --from ", "uv add "],
+ command_base="vortex", # catch a typo'd vortex* install; ignore `pip install pandas` etc.
+ command_extras=True, # pip/uvx/uv support `vortex-data[polars,...]`
+ ),
+ ValueMatch(
+ id="rust-crate-install",
+ description="`cargo add ` in the Rust quickstart matches vortex/Cargo.toml name",
+ derive=lambda root: _toml_str(root, "vortex/Cargo.toml", "package", "name"),
+ doc_files=["docs/getting-started/rust.rst"],
+ claim_template="cargo add {value}",
+ command_prefixes=["cargo add "],
+ command_base="vortex", # ignore `cargo add serde` etc.; flag a wrong vortex* crate
+ command_extras=False, # Cargo has no pip-style `crate[extras]` syntax
+ ),
+ ValueMatch(
+ id="postscript-max-size",
+ description="MAX_POSTSCRIPT_SIZE (u16::MAX - N) computed from the const def matches the spec",
+ truth_file="vortex-file/src/lib.rs",
+ # Derive from the const DEFINITION, not the regression assertion, so a stale doc can't pass
+ # against a test literal that itself drifted. u16::MAX is a language constant (65535).
+ # `^\s*pub const` + `;` so a commented decoy or a changed RHS fails rather than mis-sourcing.
+ truth_regex=r"^\s*pub const MAX_POSTSCRIPT_SIZE: u16 = u16::MAX - (\d+)\s*;",
+ transform=lambda n: str(65535 - int(n)),
+ doc_files=["docs/specification/reading-a-file.md"],
+ ),
+ ValueMatch(
+ id="postscript-eof-size",
+ description="EOF_SIZE (the EndOfFile trailer length) in reading-a-file.md matches the const",
+ truth_file="vortex-file/src/lib.rs",
+ truth_regex=r"^\s*pub const EOF_SIZE: usize = (\d+)\s*;",
+ claim_template="{value} bytes", # `... trailer is 8 bytes` — specific enough not to match any 8
+ doc_files=["docs/specification/reading-a-file.md"],
+ ),
+ ValueMatch(
+ id="postscript-first-read",
+ description="First-read size = MAX_POSTSCRIPT_SIZE + EOF_SIZE (INITIAL_READ_SIZE) in the spec",
+ # Derived from BOTH constituent constants (not a self-cancelling reconstruction), with the
+ # INITIAL_READ_SIZE = MAX_POSTSCRIPT_SIZE + EOF_SIZE relationship verified in open.rs.
+ derive=_derive_initial_read,
+ doc_files=["docs/specification/reading-a-file.md"],
+ ),
+ ValueMatch(
+ id="duckdb-scan-function",
+ description="DuckDB scan table-function name registered in vortex-duckdb appears in the guide",
+ truth_file="vortex-duckdb/cpp/table_function.cpp",
+ # Not line-anchored (the C++ field initializer is mid-line); decoy protection comes from
+ # `_strip_comments` removing `//` and `/* */` before matching in resolve_truth.
+ truth_regex=r'name\s*:\s*\{"([a-z_]+)"s',
+ doc_files=["docs/user-guide/duckdb.md"],
+ ),
+ # NOTE: the `duckdb-filesystem-setting` check is intentionally omitted. The `vortex_filesystem`
+ # DuckDB setting was refactored out of `vortex-duckdb`, so this source-anchored check can no longer
+ # resolve its truth value. Re-base it (and the filesystem section of docs/user-guide/duckdb.md)
+ # against the current DuckDB integration before re-enabling.
+ ValueMatch(
+ id="spark-batch-default",
+ description="Spark writer default batch size matches VortexDataWriter.DEFAULT_BATCH_SIZE",
+ truth_file="java/vortex-spark/src/main/java/dev/vortex/spark/write/VortexDataWriter.java",
+ # `^\s*...final int` + `;` so a non-literal RHS or commented decoy cannot mis-source.
+ truth_regex=r"^\s*(?:private\s+)?static\s+final\s+int\s+DEFAULT_BATCH_SIZE\s*=\s*(\d+)\s*;",
+ doc_files=["docs/user-guide/spark.md"],
+ ),
+ ValueMatch(
+ id="spark-maven-artifact",
+ description="Maven coordinate uses the scala-suffixed `vortex-spark_2.13` artifact (docs + READMEs)",
+ derive=_derive_spark_artifact,
+ doc_files=["docs/user-guide/spark.md", "README.md", "java/README.md"],
+ # ...and NO bare `vortex-spark` coordinate (suffix dropped) survives: a `vortex-spark` in a
+ # coordinate position — `dev.vortex:vortex-spark`, `dev.vortex/vortex-spark` (the shields.io
+ # badge form), or `vortex-spark` — NOT followed by the `_` suffix. (A bare
+ # backticked `vortex-spark` describing the connector component is not a coordinate and is fine.)
+ forbid_regex=r"(?:dev\.vortex[:/]|)vortex-spark(?!_)",
+ ),
+ ValueMatch(
+ id="jni-module-name",
+ description="java/README.md cites the `vortex-jni` JNI module (settings.gradle.kts include + the "
+ "module's build.gradle.kts artifactId); forbids the renamed-away `vortex-java`",
+ derive=_jni_module_name, # "vortex-jni"; fails loud if the Gradle module is renamed
+ claim_template="{value}",
+ doc_files=["java/README.md"],
+ forbid_regex=r"\bvortex-java\b", # the old name must not reappear
+ ),
+ ValueMatch(
+ id="spark-scala213-version",
+ description="spark.md's `vortex-spark_2.13` -> Spark 4.x targeting matches the build.gradle.kts when-arm",
+ derive=lambda root: _spark_major_for_scala(root, "2.13"),
+ # Scope the claim to the artifact->Spark-major sentence so a wrong major fails even if "Spark 4.x"
+ # appears elsewhere.
+ claim_template="`vortex-spark_2.13` artifact targets {value}",
+ doc_files=["docs/user-guide/spark.md"],
+ ),
+ ValueMatch(
+ id="spark-scala212-version",
+ description="spark.md's `vortex-spark_2.12` -> Spark 3.x targeting matches the build.gradle.kts when-arm",
+ derive=lambda root: _spark_major_for_scala(root, "2.12"),
+ claim_template="`vortex-spark_2.12` artifact targeting {value}",
+ doc_files=["docs/user-guide/spark.md"],
+ ),
+ ValueMatch(
+ id="spark-output-filename",
+ description="spark.md output filename matches the Java `String.format` in BOTH Spark writers",
+ derive=_derive_spark_filename,
+ doc_files=["docs/user-guide/spark.md"],
+ ),
+ ValueMatch(
+ id="spark-batch-range",
+ description="Spark writer batch-size range `MIN–MAX` matches VortexDataWriter MIN/MAX_BATCH_SIZE",
+ # Derived from BOTH bounds so the documented `(1–65536)` locks MIN_BATCH_SIZE (=1) AND
+ # MAX_BATCH_SIZE (=65536); a bare `1` is too generic to lock, but the range string is specific.
+ derive=_derive_spark_batch_range,
+ doc_files=["docs/user-guide/spark.md"],
+ ),
+ ValueMatch(
+ id="cli-crate-install",
+ description="`cargo binstall/install ` in install.md matches vortex-tui/Cargo.toml name",
+ derive=lambda root: _toml_str(root, "vortex-tui/Cargo.toml", "package", "name"),
+ doc_files=["docs/getting-started/install.md"],
+ command_prefixes=["cargo binstall ", "cargo install "],
+ command_base="vortex", # flag a wrong vortex* CLI crate; ignore unrelated `cargo install` lines
+ command_extras=False,
+ ),
+ ValueMatch(
+ id="python-min-version",
+ description="`Python ` in the Python compatibility docs matches pyproject requires-python",
+ derive=_derive_python_version,
+ doc_files=["docs/api/python/index.rst"],
+ ),
+ ValueMatch(
+ id="cli-binary-name",
+ description="The `vx` CLI binary name in the getting-started docs matches vortex-tui [[bin]] name",
+ # The [[bin]] name (the invoked binary), parsed via tomllib so comments/key-order/list-values
+ # can't mis-source it. Locking it means a rename is caught (the docs must mention the new name)
+ # rather than silently passing — the CLI subcommand check alone would scan the NEW prefix and
+ # miss the now-stale old-name invocations. Cover EVERY front-door page that invokes `vx`.
+ derive=lambda root: _toml_str(root, "vortex-tui/Cargo.toml", "bin", 0, "name"),
+ doc_files=["docs/getting-started/index.md", "docs/getting-started/install.md",
+ "docs/getting-started/query.md", "docs/getting-started/convert.md"],
+ ),
+ ValueMatch(
+ id="io-coalesce-file",
+ description="local-file coalesce distance+max in io.md prose match CoalesceConfig::file()",
+ # Derive BOTH bounds and lock the full prose phrase ("1 MB distance and 4 MB max size") so the
+ # claim is SCOPED to the local-file sentence — a wrong distance OR max fails, and a bare "4 MB"
+ # elsewhere in io.md can't satisfy it. The forbid additionally catches the stale 8 KB value in
+ # EITHER form (prose or the Backend Adaptation table row); the in-memory note has no "Local file".
+ derive=lambda root: (lambda dm: f"{_bytes_to_human(dm[0])} distance and {_bytes_to_human(dm[1])} max size")(
+ _coalesce_bytes(root, "file")),
+ doc_files=["docs/developer-guide/internals/io.md"],
+ forbid_regex=r"[Ll]ocal files?\b[^\n]*8\s*KB",
+ ),
+ ValueMatch(
+ id="io-coalesce-object",
+ description="object-store coalesce distance+max in io.md prose match CoalesceConfig::object_storage()",
+ # Scoped like io-coalesce-file: lock the full "1 MB distance and 16 MB max size" object-store phrase.
+ derive=lambda root: (lambda dm: f"{_bytes_to_human(dm[0])} distance and {_bytes_to_human(dm[1])} max size")(
+ _coalesce_bytes(root, "object_storage")),
+ doc_files=["docs/developer-guide/internals/io.md"],
+ ),
+ ValueMatch(
+ id="io-in-memory-coalesce-behavior",
+ description="io.md's in-memory opt-in coalescing claim matches the VortexReadAt default (coalesce_config "
+ "returns None ⇒ a reader that doesn't override it does not coalesce)",
+ derive=_io_in_memory_coalesce_behavior, # "opt-in"; fails loud if the default coalesce_config() != None
+ doc_files=["docs/developer-guide/internals/io.md"],
+ ),
+ ValueMatch(
+ id="io-coalesce-in-memory",
+ description="in-memory coalesce distance+max in io.md match CoalesceConfig::in_memory()",
+ # Scoped like file/object: the in_memory() preset is 8 KB/8 KB. (Coalescing is opt-in for in-memory
+ # readers — the default reader doesn't coalesce; readers that opt in use this preset. io.md states both.)
+ derive=lambda root: (lambda dm: f"{_bytes_to_human(dm[0])} distance and {_bytes_to_human(dm[1])} max size")(
+ _coalesce_bytes(root, "in_memory")),
+ doc_files=["docs/developer-guide/internals/io.md"],
+ ),
+ ValueMatch(
+ id="io-reader-file",
+ description="io.md names `FileReadAt` as the local-file VortexReadAt implementor (std_file/read_at.rs)",
+ derive=lambda root: _io_reader_impl(root, "FileReadAt", "vortex-io/src/std_file/read_at.rs"),
+ claim_template="{value}",
+ doc_files=["docs/developer-guide/internals/io.md"],
+ ),
+ ValueMatch(
+ id="io-reader-object-store",
+ description="io.md names `ObjectStoreReadAt` as the object-store VortexReadAt implementor "
+ "(object_store/read_at.rs)",
+ derive=lambda root: _io_reader_impl(root, "ObjectStoreReadAt", "vortex-io/src/object_store/read_at.rs"),
+ claim_template="{value}",
+ doc_files=["docs/developer-guide/internals/io.md"],
+ ),
+ ValueMatch(
+ id="buffered-block-size",
+ description="`Buffered Layout` size in file-format.md matches BufferedStrategy in strategy.rs",
+ derive=_derive_buffered_block_size,
+ # Scope the claim to the Buffered Layout line ("localize up to 2 MB") so a wrong size there
+ # fails even if a correct "2 MB" appears in another sentence.
+ claim_template="localize up to {value}",
+ doc_files=["docs/concepts/file-format.md"],
+ # Forbid every stale form this PR corrected: (1) the wrong localization value (BufferedStrategy
+ # is 2 * ONE_MEG, not 1 MB); (2) " MB chunks" wording; (3) " MB of uncompressed data" on
+ # the Chunked layer. The 2 MB is buffered-chunk LOCALITY — the Chunked layer imposes no byte
+ # size, so attributing a byte size to chunks/uncompressed-data is the misconception this fixes.
+ forbid_regex=r"localize up to 1\s*MB|\d+\s*MB chunks|\d+\s*MB of uncompressed data",
+ ),
+ ValueMatch(
+ id="cpp-binding-cxx",
+ description="C++ docs describe a `cxx` Rust bridge (matches vortex-cxx Cargo.toml), not a C-FFI wrapper",
+ # Present-check anchors on `cxx` (the real mechanism); the forbid blocks the stale current-state
+ # claim that C++ "wraps the C FFI" — that is a FUTURE migration (language-bindings.md), not today.
+ derive=_cxx_dep,
+ doc_files=["docs/api/cpp/index.rst", "docs/developer-guide/embedding/cxx.md",
+ "docs/developer-guide/embedding/index.md", "docs/developer-guide/internals/architecture.md"],
+ # Catch any current-state "wrap(per/s/ping) ... the C FFI" claim; the planned-migration wording
+ # ("wrapping the C API") refers to the C API, not the C FFI, so it is correctly NOT matched.
+ # `[^.]` (not `[^.\n]`) keeps it sentence-scoped while tolerating line wraps (no line-break dodge).
+ forbid_regex=r"wrap(?:per|ping|s)?\b[^.]*?\bC FFI",
+ ),
+ ValueMatch(
+ id="spark-filter-pushdown",
+ description="integrations/spark.md says filter pushdown is supported, matching VortexScanBuilder",
+ derive=_spark_filter_pushdown, # "SupportsPushDownV2Filters"; fails loud if the connector drops it
+ doc_files=["docs/developer-guide/integrations/spark.md"],
+ forbid_regex=r"filter pushdown[^.]*\b(?:not yet connected|planned future work)\b|not yet connected to Spark",
+ ),
+ ValueMatch(
+ id="duckdb-direct-path",
+ description="user-guide/duckdb.md says direct file-path syntax works, matching the registered replacement scan",
+ derive=_duckdb_replacement_scan, # "replacement scan"; fails loud if the registration is gone
+ doc_files=["docs/user-guide/duckdb.md"],
+ forbid_regex=r"coming in an upcoming DuckDB release",
+ ),
+ ValueMatch(
+ id="cli-convert-chunk-size",
+ description="convert.md + `vx convert` --help state the 8192-row batch chunking, matching BATCH_SIZE",
+ # The quickstart used to claim "chunking on Parquet row-group boundaries"; really `vx convert`
+ # reads in BATCH_SIZE-row batches and writes 8192-row blocks. Lock the value (scoped to "-row")
+ # in BOTH the doc and the CLI --help comment; forbid the stale row-group-boundaries claim.
+ derive=_convert_batch_size, # "8192"
+ # Scope to the batch-chunking phrase common to both the doc ("chunking into 8192-row batches")
+ # and the --help comment ("Chunking occurs in 8192-row batches"), so an unrelated "8192-row"
+ # mention can't mask drift in the actual chunking sentence.
+ claim_template="{value}-row batches",
+ doc_files=["docs/getting-started/convert.md", "vortex-tui/src/lib.rs"],
+ forbid_regex=r"(?i)row.?group boundaries",
+ ),
+ ValueMatch(
+ id="scan-api-present",
+ description="roadmap.md notes the Scan API exists today, matching the `vortex-scan` crate",
+ derive=_scan_crate_name, # "vortex-scan"; fails loud if the crate is gone
+ # Scope to the PRESENT-state sentence ("already provides"), so a future-only "the vortex-scan
+ # crate will provide ..." rewording fails rather than passing on a bare token match.
+ claim_template="Scan API (the `{value}` crate) already provides",
+ doc_files=["docs/project/roadmap.md"],
+ ),
+ ValueMatch(
+ id="variant-dtype-present",
+ description="roadmap.md notes the Variant DType exists today, matching DType::Variant in vortex-array",
+ derive=_variant_dtype_present, # "Variant"; fails loud if the variant is gone/renamed
+ # Scope to the present-state claim so a "Variant is planned/upcoming" rewording fails.
+ claim_template="`{value}` DType already exists",
+ doc_files=["docs/project/roadmap.md"],
+ ),
+ ValueMatch(
+ id="canonical-union-exception",
+ description="concepts/arrays.md + api/python/arrays.rst name `Union` as the DType without a canonical "
+ "encoding; locked to the DType + Canonical enums (fails loud if Union is canonicalized)",
+ derive=_union_is_sole_noncanonical, # "Union"; the code-side guard is the derive, see its docstring
+ # Bare-token present-check, scoped to the two canonical-arrays docs (where `Union` appears only in
+ # the exception sentence); the derive does the load-bearing code assertion.
+ claim_template="{value}",
+ doc_files=["docs/concepts/arrays.md", "docs/api/python/arrays.rst"],
+ ),
+ ValueMatch(
+ id="integration-scan-trait",
+ description="integration docs name the real `DataSource` scan trait, not a (nonexistent) `Source` trait",
+ # The duckdb/datafusion/spark integration pages describe migrating to the Scan API; they must name
+ # the real `DataSource` trait. Present-anchor on it; forbid a backticked `Source`. (No `sink`
+ # forbid here — Spark docs may legitimately discuss sinks.)
+ derive=lambda root: "DataSource",
+ doc_files=["docs/developer-guide/integrations/duckdb.md",
+ "docs/developer-guide/integrations/datafusion.md",
+ "docs/developer-guide/integrations/spark.md"],
+ forbid_regex=r"`Source`",
+ ),
+ ValueMatch(
+ id="cpp-not-wrapper-nav",
+ description="nav/landing docs don't frame the C++ binding as a 'wrapper' or label C++ as FFI (it's cxx)",
+ # Companion to cpp-binding-cxx for the top-level/nav pages, which don't mention `cxx` (so they
+ # can't share that check's present-anchor). `Vortex` is a stable present anchor; the forbid blocks
+ # the stale "C++ wrapper" framing and the "C/C++ (FFI)" label (C++ is a cxx bridge, not the C FFI).
+ derive=lambda root: "Vortex",
+ doc_files=["docs/index.md", "docs/developer-guide/overview.md", "docs/api/c/index.rst"],
+ # Block the "C++ wrapper" framing, any "C++ (FFI)" / "C++ FFI" / "C/C++ (FFI)" label, AND a
+ # present-tense "foundation for ... C++" claim (C++ is a cxx Rust bridge today, not built on the
+ # C FFI — the C FFI is the INTENDED/future foundation). The `[^.]` (not `[^.\n]`) keeps the claim
+ # SENTENCE-scoped while tolerating line WRAPS, so the stale claim can't dodge the forbid by breaking
+ # across lines. "C++ (cxx)" and a later-sentence C++ mention are unaffected.
+ forbid_regex=r"C\+\+ wrapper|C\+\+\s*\(?FFI\)?|foundation for[^.]*?\bC\+\+",
+ ),
+ ValueMatch(
+ id="no-trino-integration-claim",
+ description="integration lists don't claim a (nonexistent) current/in-progress Trino integration",
+ # No Trino connector exists in the repo (only JNI bindings an external connector could build on).
+ # Forbid the "Trino ... in progress" framing AND the stale "Spark and Trino" current-pairing in the
+ # existing-integration / integration-point docs. The work-in-progress page remains the single place
+ # that lists Trino as planned; factual "Trino supports JDK 22" and "future ... Trino" are unaffected.
+ # `DataFusion` is a stable present anchor in all three files.
+ derive=lambda root: "DataFusion",
+ doc_files=["docs/index.md", "docs/concepts/scanning.md", "docs/developer-guide/language-bindings.md",
+ "docs/developer-guide/internals/architecture.md"],
+ # `[^.]` + `\s+` keep these sentence-scoped but line-wrap-tolerant, so a wrapped "Trino ... in\n
+ # progress" or "Spark\nand Trino" can't dodge the forbid by breaking across lines.
+ forbid_regex=r"Trino[^.]*?\bin\s+(?:progress|development)|with\s+Trino|Spark\s+and\s+Trino",
+ ),
+ ValueMatch(
+ id="scanning-api-traits",
+ description="scanning.md uses real scan trait names (no `Sink`, no `Source` trait; it's `DataSource`)",
+ # vortex-scan exposes DataSource/DataSourceScan/Partition. Present-anchor on the real `DataSource`
+ # name (the doc now uses it); forbid the fabricated `Sink` (any casing) AND a backticked `Source`
+ # implying a literal trait named Source (the real trait is `DataSource`).
+ derive=lambda root: "DataSource",
+ doc_files=["docs/concepts/scanning.md"],
+ # Forbid the fabricated `Sink` trait/interface (scoped so legitimate "data sink" prose is fine)
+ # and a backticked `Source` implying a literal trait named Source (the real trait is `DataSource`).
+ forbid_regex=r"(?i)\bsink`?\s+(?:trait|interface)\b|`Source`",
+ ),
+]
+
+
+@dataclass
+class CliSubcommandCheck:
+ """Assert that every `` referenced in docs SHELL CODE REGIONS
+ (shell-tagged fenced blocks + inline code spans — NOT prose, and NOT non-shell code blocks where
+ `vx` is the Python module alias) is a real subcommand of the CLI, derived from the clap
+ `Subcommand` enum in `enum_file`. Catches a doc that references a fabricated or renamed CLI
+ command — the `vx` analog of a fabricated API."""
+
+ id: str
+ description: str
+ enum_file: str
+ enum_name: str
+ command_prefix: str = "" # fallback literal, e.g. "vx "; prefer sourcing via bin_truth_file
+ bin_truth_file: str = "" # Cargo.toml whose `[[bin]] name = "..."` gives the real binary name
+
+ def resolve_prefix(self, root: Path) -> str:
+ """The command prefix (` `) — sourced from `bin_truth_file`'s `[[bin]] name` (via
+ tomllib) so a renamed binary is detected (read both sides; don't hard-code), falling back to
+ the literal `command_prefix` only when no `bin_truth_file` is configured."""
+ if not self.bin_truth_file:
+ return self.command_prefix
+ return _toml_str(root, self.bin_truth_file, "bin", 0, "name") + " "
+
+ @staticmethod
+ def _enum_body(text: str, enum_name: str) -> tuple[str, str]:
+ """Return `(body, header)` for `enum { ... }`: `body` is the text between the enum's
+ outer braces (brace-matched, so a nested `{ ... }` struct variant does not end it early);
+ `header` is the CONTIGUOUS block of attribute / doc-comment / blank lines immediately preceding
+ the declaration, where enum-level clap attributes (`#[command(rename_all = ...)]`) live. The
+ block is collected by walking backward and bracket-balancing, so a multi-line `#[command(...)]`
+ is captured in full regardless of length (not clipped by a fixed-width window)."""
+ i = text.find(f"enum {enum_name} {{")
+ if i < 0:
+ raise LookupError(f"enum {enum_name} not found in source")
+ start = text.index("{", i)
+ depth, end = 0, None
+ for j in range(start, len(text)):
+ if text[j] == "{":
+ depth += 1
+ elif text[j] == "}":
+ depth -= 1
+ if depth == 0:
+ end = j
+ break
+ if end is None:
+ raise LookupError(f"enum {enum_name} body is not closed")
+ header_lines: list[str] = []
+ bracket = 0 # net unclosed ']' while walking UP through a (possibly multi-line) attribute
+ for line in reversed(text[:i].split("\n")):
+ s = line.strip()
+ if bracket > 0: # inside a multi-line attribute we entered from its closing `]`
+ header_lines.append(line)
+ bracket += s.count("]") - s.count("[")
+ elif s == "" or s.startswith("//"): # blank or doc/line comment: part of the header block
+ header_lines.append(line)
+ elif "]" in s or s.startswith("#["): # an attribute line (possibly with a trailing comment
+ header_lines.append(line) # like `#[command(rename_all=..)] // note`)
+ bracket += s.count("]") - s.count("[")
+ else: # a line of the previous item — the header block ends here
+ break
+ return text[start + 1:end], "\n".join(reversed(header_lines))
+
+ @classmethod
+ def _enum_variants(cls, text: str, enum_name: str) -> list[str]:
+ """Variant identifiers of `enum `, DEPTH-AWARE within the body: only identifiers at the
+ enum's top level (brace-depth 0) are variants, so a struct-variant's field type
+ (`Browse {\\n file: PathBuf,\\n}`) on its own line is not mis-read as a variant."""
+ body, _ = cls._enum_body(text, enum_name)
+ variants: list[str] = []
+ depth = 0 # depth WITHIN the enum body
+ for line in body.split("\n"):
+ if depth == 0:
+ m = re.match(r"([A-Z][A-Za-z0-9]*)\s*[({,]", line.strip())
+ if m:
+ variants.append(m.group(1))
+ depth += line.count("{") - line.count("}")
+ return variants
+
+ @staticmethod
+ def _to_kebab(variant: str) -> str:
+ """clap/heck default subcommand name: insert `-` at lower/digit->Upper and Acronym->Word
+ boundaries, then lowercase. `Tree`->`tree`, `FooBar`->`foo-bar`, `SQLQuery`->`sql-query`,
+ `HTTPServer`->`http-server`."""
+ s = re.sub(r"(?<=[a-z0-9])(?=[A-Z])", "-", variant)
+ s = re.sub(r"(?<=[A-Z])(?=[A-Z][a-z])", "-", s)
+ return s.lower()
+
+ @staticmethod
+ def _decode_rust_str(v: str) -> str | None:
+ """Decode a Rust string literal value: `"x"` -> `x` (un-escaping `\\"`/`\\\\`), raw
+ `r#"x"#`/`r"x"` -> `x`. Returns None when `v` is not a (recognized) string literal — e.g. an
+ identifier, number, list, or bare flag — so callers can distinguish a string value from a key."""
+ rm = re.match(r'r(#*)"(.*)"\1\Z', v, re.DOTALL)
+ if rm:
+ return rm.group(2)
+ if len(v) >= 2 and v[0] == '"' and v[-1] == '"':
+ return v[1:-1].replace('\\"', '"').replace("\\\\", "\\")
+ return None
+
+ @staticmethod
+ def _iter_attrs(text: str):
+ """Yield the inner text of each top-level `#[...]` attribute, string- and bracket-aware (so a
+ `]` inside a string value does not end the attribute early). A variant may carry several."""
+ i, n = 0, len(text)
+ while i < n:
+ if text.startswith("#[", i):
+ j, depth = i + 2, 1
+ while j < n and depth > 0:
+ rm = re.match(r'r(#*)"', text[j:])
+ if rm: # raw string
+ close = '"' + rm.group(1)
+ e = text.find(close, j + len(rm.group(0)))
+ j = n if e < 0 else e + len(close)
+ continue
+ ch = text[j]
+ if ch == '"': # regular string with backslash escapes
+ k = j + 1
+ while k < n and text[k] != '"':
+ k += 2 if text[k] == "\\" else 1
+ j = k + 1
+ continue
+ if ch == "[":
+ depth += 1
+ elif ch == "]":
+ depth -= 1
+ j += 1
+ yield text[i + 2:j - 1]
+ i = j
+ else:
+ i += 1
+
+ @staticmethod
+ def _meta_items(s: str, i: int) -> list[str]:
+ """Top-level comma-separated items of a `(...)` meta list starting at index `i` (just past the
+ opening paren), STRING-LITERAL- and paren/bracket-depth-aware so commas inside strings or nested
+ groups do not split, and a `name = "x"` inside another item's string cannot become its own item."""
+ depth, bracket = 1, 0
+ items: list[str] = []
+ cur: list[str] = []
+ while i < len(s):
+ rm = re.match(r'r(#*)"', s[i:])
+ if rm: # raw string r#"..."#
+ close = '"' + rm.group(1)
+ end = s.find(close, i + len(rm.group(0)))
+ end = len(s) if end < 0 else end + len(close)
+ cur.append(s[i:end])
+ i = end
+ continue
+ c = s[i]
+ if c == '"': # regular string with backslash escapes
+ j = i + 1
+ while j < len(s) and s[j] != '"':
+ j += 2 if s[j] == "\\" else 1
+ cur.append(s[i:min(j + 1, len(s))])
+ i = j + 1
+ continue
+ if c == "(":
+ depth += 1
+ elif c == ")":
+ depth -= 1
+ if depth == 0:
+ break
+ elif c == "[":
+ bracket += 1
+ elif c == "]":
+ bracket -= 1
+ elif c == "," and depth == 1 and bracket == 0:
+ items.append("".join(cur))
+ cur = []
+ i += 1
+ continue
+ cur.append(c)
+ i += 1
+ items.append("".join(cur))
+ return items
+
+ @classmethod
+ def _clap_meta(cls, attr_text: str) -> dict[str, str | None]:
+ """Merge EVERY `command(...)`/`clap(...)` meta list across ALL `#[...]` attributes in
+ `attr_text` into `{key: value}` (value = decoded string for `key = "x"`, else None for a bare
+ flag). A variant may split metadata across separate attributes (`#[command(about=..)]` then
+ `#[command(name=..)]`); both are honored. Parsing is string-literal-aware (per `_iter_attrs` /
+ `_meta_items`), so a `name = "x"` INSIDE another item's string CANNOT spoof the result."""
+ meta: dict[str, str | None] = {}
+ for inner in cls._iter_attrs(attr_text):
+ mm = re.match(r"\s*(?:command|clap)\s*\(", inner)
+ if not mm:
+ continue
+ for item in cls._meta_items(inner, mm.end()):
+ km = re.match(r"\s*(\w+)\s*=\s*(.*)\Z", item, re.DOTALL)
+ if km:
+ raw = km.group(2).strip()
+ decoded = cls._decode_rust_str(raw)
+ # decoded string for `key = "x"`, else the RAW value (e.g. a `["x","y"]` list, so
+ # plural `aliases`/`visible_aliases` are recoverable by the caller)
+ meta[km.group(1)] = decoded if decoded is not None else raw
+ elif item.strip():
+ meta.setdefault(item.strip(), None) # bare flag, e.g. `subcommand` / `flatten`
+ return meta
+
+ @classmethod
+ def _variant_name_aliases(cls, attrs: str, ident: str) -> tuple[str, set[str]]:
+ """The CLI name (clap `name=` override, else kebab(ident)) and the alias set
+ (`alias`/`visible_alias`/`aliases`/`visible_aliases`) for one variant, parsed string-aware."""
+ meta = cls._clap_meta(attrs)
+ nm = meta.get("name")
+ name = nm if isinstance(nm, str) and nm else cls._to_kebab(ident)
+ aliases: set[str] = set()
+ for key in ("alias", "visible_alias", "aliases", "visible_aliases"):
+ val = meta.get(key)
+ if isinstance(val, str):
+ aliases.update(re.findall(r"[A-Za-z0-9][\w-]*", val))
+ return name, aliases
+
+ @staticmethod
+ def _struct_body(text: str, struct_name: str) -> str | None:
+ """Brace-matched body of `struct { ... }`, or None (tuple/unit struct or not found)."""
+ m = re.search(rf"\bstruct\s+{re.escape(struct_name)}\b[^{{;]*\{{", text)
+ if not m:
+ return None
+ start = text.index("{", m.start())
+ depth, end = 0, None
+ for j in range(start, len(text)):
+ if text[j] == "{":
+ depth += 1
+ elif text[j] == "}":
+ depth -= 1
+ if depth == 0:
+ end = j
+ break
+ return text[start + 1:end] if end is not None else None
+
+ @staticmethod
+ def _rename_all_guard(enum_name: str, body: str, header: str) -> None:
+ """FAIL LOUD on a clap `rename_all` (enum- or variant-level): it globally changes clap's casing
+ convention, which the kebab heuristic does not model, so a silent docs-vs-CLI mismatch is turned
+ into an explicit error to extend the check — rather than the matcher itself silently drifting,
+ which the project BANS forbid."""
+ if re.search(r"\brename_all\b", header) or re.search(r"\brename_all\b", body):
+ raise LookupError(
+ f"enum {enum_name} uses a clap `rename_all` attribute the conformance check does not "
+ f"model; extend CliSubcommandCheck to honor it before trusting the docs-vs-CLI comparison")
+
+ @staticmethod
+ def _net_delim(line: str) -> int:
+ """Net unbalanced `()`/`{}`/`[]` openers minus closers on a line."""
+ return (line.count("(") + line.count("{") + line.count("[")
+ - line.count(")") - line.count("}") - line.count("]"))
+
+ @classmethod
+ def _walk_variants(cls, body: str) -> list[dict[str, str]]:
+ """Per top-level variant of an enum body, return `{ident, attrs, text}`: `attrs` is the
+ concatenated (bracket-balanced) attribute text; `text` is the variant's FULL declaration — a
+ single line for `Tree(Args)`, or all lines until the payload's `()`/`{}` balances for a
+ rustfmt-wrapped multi-line tuple/struct variant. Capturing the full payload (not just the first
+ line) means a wrapped `Tree(\\n Args,\\n)` still exposes its nested-subcommand type."""
+ variants: list[dict[str, str]] = []
+ pending, attr_open = "", 0
+ cur: dict[str, str] | None = None # variant whose multi-line payload/block we are accumulating
+ cap_depth = 0
+ for line in body.split("\n"):
+ s = line.strip()
+ if cur is not None: # accumulate until the variant's delimiters balance
+ cur["text"] += "\n" + line
+ cap_depth += cls._net_delim(line)
+ if cap_depth <= 0:
+ cur = None
+ continue
+ if attr_open > 0 or s.startswith("#["):
+ pending += " " + s
+ attr_open += s.count("[") - s.count("]")
+ continue
+ if (mv := re.match(r"([A-Z][A-Za-z0-9]*)", s)):
+ v = {"ident": mv.group(1), "attrs": pending, "text": line}
+ variants.append(v)
+ pending = ""
+ d = cls._net_delim(line)
+ if d > 0: # multi-line tuple/struct variant — keep accumulating its payload/block
+ cur = v
+ cap_depth = d
+ return variants
+
+ @classmethod
+ def _subcommand_field_enum(cls, text: str) -> str | None:
+ """The enum type of a `#[clap(subcommand)]` / `#[command(subcommand)]` field within `text`, or
+ None if there is no such field OR the field is OPTIONAL/repeated (`Option<..>`/`Vec<..>`).
+ The `subcommand` flag is detected among OTHER meta (`#[command(subcommand, required = true)]`)
+ by parsing the attribute string-aware — not by requiring `subcommand` to be the sole arg. An
+ optional nested subcommand means the parent ALSO accepts its own positional args, so we cannot
+ tell a fabricated subcommand from an argument — the parent is treated as a lenient leaf rather
+ than risk false-positives on real `vx inspect `-style invocations."""
+ for am in re.finditer(r"(#\[(?:clap|command)\s*\([^\]]*\)\])\s*(?:pub\s+)?\w+\s*:\s*([^,\n]+)", text):
+ if "subcommand" not in cls._clap_meta(am.group(1)): # genuine top-level flag, not in a string
+ continue
+ expr = am.group(2).strip()
+ if expr.startswith(("Option", "Vec")): # optional/repeated subcommand → lenient leaf
+ return None
+ idents = re.findall(r"[A-Za-z_]\w*", expr)
+ return idents[-1] if idents else None
+ return None
+
+ @classmethod
+ def _resolve_nested(cls, crate_src: str, type_name: str, visited: set[str]) -> dict:
+ """Children sub-tree for a variant payload type: follow a struct's REQUIRED `#[clap(subcommand)]`
+ field to its enum, or a payload that is itself a `#[derive(Subcommand)] enum`. Else a leaf."""
+ t = re.sub(r"[&<].*", "", type_name.split("::")[-1]).strip()
+ if not t or not t[0].isupper():
+ return {}
+ sb = cls._struct_body(crate_src, t)
+ if sb is not None:
+ nested = cls._subcommand_field_enum(sb)
+ return cls._command_tree(crate_src, nested, visited) if nested else {}
+ if re.search(rf"#\[derive\([^)]*Subcommand[^)]*\)\]\s*(?:pub\s+)?enum\s+{re.escape(t)}\b", crate_src):
+ return cls._command_tree(crate_src, t, visited)
+ return {}
+
+ @classmethod
+ def _command_tree(cls, crate_src: str, enum_name: str, visited: set[str] | None = None) -> dict:
+ """Recursively build the clap command TREE rooted at `enum `: a nested dict mapping
+ each subcommand name (and alias) to its own sub-tree (`{}` for a leaf). Nested subcommands are
+ resolved by following a tuple variant's payload type (e.g. `Tree(TreeArgs)`) to a struct's
+ `#[clap(subcommand)]` field. Recursion is cycle-guarded. FAILS LOUD on `rename_all`."""
+ visited = visited or set()
+ if enum_name in visited:
+ return {}
+ visited = visited | {enum_name}
+ body, header = cls._enum_body(crate_src, enum_name)
+ cls._rename_all_guard(enum_name, body, header)
+ tree: dict = {}
+ for v in cls._walk_variants(body):
+ name, aliases = cls._variant_name_aliases(v["attrs"], v["ident"])
+ children: dict = {}
+ # tuple payload anchored to the variant ident (so an attribute's `command(` inside a struct
+ # variant body is NOT misread as a tuple payload); spans lines for rustfmt-wrapped variants.
+ tuple_m = re.match(r"\s*" + re.escape(v["ident"]) + r"\s*\(\s*(?:#\[[^\]]*\]\s*)?([\w:]+)",
+ v["text"])
+ if tuple_m: # tuple variant: payload type may carry a nested subcommand
+ children = cls._resolve_nested(crate_src, tuple_m.group(1), visited)
+ else: # struct/unit variant: a #[command(subcommand)] field carries the nested enum
+ nested = cls._subcommand_field_enum(v["text"])
+ if nested:
+ children = cls._command_tree(crate_src, nested, visited)
+ tree[name] = children
+ for a in aliases:
+ tree[a] = children
+ return tree
+
+ @classmethod
+ def _subcommand_names(cls, text: str, enum_name: str) -> set[str]:
+ """Flat set of subcommand names (kebab/`name=`/aliases) for ONE enum's variants — the first
+ level only. Used by the self-test; the live check uses `_command_tree` for nested paths."""
+ body, header = cls._enum_body(text, enum_name)
+ cls._rename_all_guard(enum_name, body, header)
+ names: set[str] = set()
+ for v in cls._walk_variants(body):
+ name, aliases = cls._variant_name_aliases(v["attrs"], v["ident"])
+ names.add(name)
+ names.update(aliases)
+ return names
+
+ # Shell-flavored code-block languages — where CLI invocations live. We deliberately do NOT scan
+ # `python`/`rust`/etc. blocks: the `vx` token is overloaded (it is also the docs' Python module
+ # alias, `import vortex as vx`), so a Python block is full of `vx.array(...)` that has nothing to
+ # do with the CLI. Restricting to shell blocks (plus inline spans) is the semantic match.
+ _SHELL_LANGS = frozenset({"bash", "sh", "shell", "shell-session", "sh-session", "zsh",
+ "console", "shellsession", "text"})
+ # MyST code directives — OPAQUE, with the language taken from the directive argument.
+ _CODE_DIRECTIVES = frozenset({"code-block", "code", "sourcecode"})
+ # MyST code-cell / doctest / raw directives — OPAQUE and NEVER shell: their bodies are Python or
+ # literal content (where `vx` is the module alias, or arbitrary text), so we consume them WITHOUT
+ # capturing and without letting their inline spans leak into the residue scanned for CLI mentions.
+ _OPAQUE_NONSHELL_DIRECTIVES = frozenset({"doctest", "code-cell", "ipython", "ipython3",
+ "eval-rst", "math", "raw", "literalinclude"})
+ # reStructuredText code-bearing directives whose indented body must be CONSUMED (captured iff the
+ # language is shell, else dropped) so a `.. code-block:: python` body's inline ``spans`` do not
+ # leak into the CLI scan.
+ _RST_CODE_DIRECTIVES = frozenset({"code-block", "code", "sourcecode", "parsed-literal", "doctest",
+ "testcode", "testoutput", "ipython", "math", "raw"})
+ # Shell operators/separators that end a command — the command-path walk stops here (and a second
+ # same-line `vx` after one is matched independently by `_invalid_invocations`).
+ _SHELL_OPS = frozenset({"&&", "||", "|", "|&", ";", "&", ">", ">>", "<", "2>", "2>>"})
+
+ @classmethod
+ def _shell_regions(cls, text: str) -> str:
+ """Concatenate SHELL code regions — where real CLI invocations live — so neither prose mentions
+ nor non-shell code (e.g. Python using the `vx` module alias) trigger false positives.
+
+ Fences are parsed line-by-line with a length-aware scanner so the docs' own MyST patterns are
+ handled: a `````{tab}````-style directive CONTAINER (info string starts with `{`, and is not a
+ code directive) is TRANSPARENT — its body is re-parsed, so a nested ```` ```bash ```` block
+ inside it is a real shell region — whereas a plain code fence OR a MyST code directive
+ (` ```{code-block} bash `/` ```{code} `) is OPAQUE (its body is literal, captured iff the
+ language is shell). A closing fence must use the same fence char and be at least as long as its
+ opener, so a 3-backtick block nests cleanly inside a 4-backtick container. Also handles
+ shell-tagged reStructuredText `.. code-block::`/`.. code::` directives and inline ``double`` /
+ `single` code spans (a CLI mention like `vx query` in prose, but never a dotted `vx.array`)."""
+ lines = text.split("\n")
+ regions: list[str] = []
+ residue: list[str] = [] # non-code-fence lines, for RST-directive + inline-span scanning
+ directives: list[tuple[str, int]] = [] # open MyST/RST container fences: (char, length)
+ fence_re = re.compile(r"^\s*([`~]{3,})(.*)$")
+ i, n = 0, len(lines)
+ while i < n:
+ m = fence_re.match(lines[i])
+ if not m:
+ residue.append(lines[i])
+ i += 1
+ continue
+ fence, info = m.group(1), m.group(2).strip()
+ char, length = fence[0], len(fence)
+ if info.startswith("{"):
+ dm = re.match(r"\{([\w-]+)\}\s*(.*)$", info)
+ directive = dm.group(1).lower() if dm else ""
+ if directive in cls._CODE_DIRECTIVES:
+ # MyST code directive (` ```{code-block} bash `): OPAQUE, like a plain fence, with
+ # the language taken from the directive argument. Fall through to the capture path.
+ lang = (dm.group(2).strip().split() or [""])[0].lower()
+ elif directive in cls._OPAQUE_NONSHELL_DIRECTIVES:
+ lang = "" # OPAQUE non-shell: consume the body (below) but never capture it
+ else:
+ directives.append((char, length)) # true container ({tab}/{note}): transparent
+ i += 1
+ continue
+ elif not info and directives and directives[-1][0] == char and length >= directives[-1][1]:
+ directives.pop() # bare fence that closes the innermost open container
+ i += 1
+ continue
+ else:
+ lang = info.split()[0].lower() if info else ""
+ # Opaque code fence: capture its body verbatim until a closing fence of the same char that
+ # is at least as long. Nested fences of any length inside are literal content.
+ close_re = re.compile(r"^\s*" + re.escape(char) + "{" + str(length) + r",}\s*$")
+ body: list[str] = []
+ j = i + 1
+ while j < n and not close_re.match(lines[j]):
+ body.append(lines[j])
+ j += 1
+ if lang in cls._SHELL_LANGS:
+ regions.append("\n".join(body))
+ i = j + 1 # skip past the closing fence (or EOF)
+
+ # RST code directives: CONSUME each indented body (capturing it iff a shell code-block, else
+ # dropping it) so a non-shell `.. code-block:: python` body's inline spans never reach the
+ # CLI-mention scan. Lines outside any consumed body form `residue2`, scanned for inline spans.
+ rst_lines = "\n".join(residue).split("\n")
+ residue2: list[str] = []
+ i = 0
+ while i < len(rst_lines):
+ dm = re.match(r"(\s*)\.\.\s+([\w-]+)::\s*(.*)$", rst_lines[i])
+ if dm and dm.group(2).lower() in cls._RST_CODE_DIRECTIVES:
+ directive = dm.group(2).lower()
+ arg = (dm.group(3).strip().split() or [""])[0].lower()
+ marker_indent = len(dm.group(1))
+ j = i + 1
+ block: list[str] = []
+ while j < len(rst_lines) and not rst_lines[j].strip(): # skip blanks after the marker
+ j += 1
+ while j < len(rst_lines):
+ lj = rst_lines[j]
+ if lj.strip() and (len(lj) - len(lj.lstrip())) <= marker_indent:
+ break # dedent ends the block
+ block.append(lj)
+ j += 1
+ if directive in ("code-block", "code", "sourcecode") and arg in cls._SHELL_LANGS:
+ regions.append("\n".join(block)) # shell body captured; non-shell body dropped
+ i = j
+ continue
+ residue2.append(rst_lines[i])
+ i += 1
+
+ no_fence = "\n".join(residue2)
+ regions += re.findall(r"``([^`]+)``", no_fence) # RST inline literal
+ regions += re.findall(r"(? dict:
+ """The full clap command tree, built from EVERY `.rs` file in the CLI crate's source dir (so a
+ nested subcommand enum defined in a sibling module — e.g. `TreeMode` in `tree.rs` — resolves)."""
+ src_dir = (root / self.enum_file).parent
+ crate_src = "\n".join(p.read_text(encoding="utf-8") for p in sorted(src_dir.rglob("*.rs")))
+ return self._command_tree(crate_src, self.enum_name)
+
+ @classmethod
+ def _validate_path(cls, tree: dict, tokens: list[str]) -> str | None:
+ """Walk a `vx` invocation's leading tokens down the command tree. Return the invalid
+ command-PATH string (e.g. `tree frobnicate`) if a token sits in a subcommand position but is
+ not a valid child; else None. Stops at the first flag (`-x`) or once a leaf is reached (the
+ remaining tokens are arguments, not subcommands). A trailing sentence period is stripped, so an
+ inline `` `vx convert` `` followed by prose punctuation is not mis-flagged."""
+ node, path = tree, []
+ for tok in tokens:
+ if tok.startswith("-") or tok in cls._SHELL_OPS:
+ break # a flag, or a shell operator/separator (the command ends here)
+ if not node:
+ break # leaf reached → remaining tokens are this command's arguments
+ tok = tok.rstrip(".,:;") # drop trailing sentence punctuation (inline `vx convert`.)
+ if not tok:
+ break
+ if tok in node:
+ path.append(tok)
+ node = node[tok]
+ else:
+ # `node` still has (required) children but this token is not one of them — whether it is
+ # a fabricated subcommand (`vx tree frobnicate`) or a path/arg where the required
+ # subcommand was omitted (`vx tree ./file.vortex`), the invocation is invalid.
+ return " ".join([*path, tok])
+ return None
+
+ def _invalid_invocations(self, root: Path, tree: dict, prefix: str) -> dict[str, list[str]]:
+ out: dict[str, list[str]] = {}
+ # Match EACH `` independently (the body is NOT captured greedily to EOL), so a second
+ # same-line invocation after a separator — `vx convert f && vx frobnicate` — is validated too.
+ # Left boundary `(? tuple[bool, str]:
+ prefix = self.resolve_prefix(root) # sourced from Cargo.toml [[bin]] name (not hard-coded)
+ tree = self.command_tree(root)
+ if not tree:
+ raise LookupError(f"no subcommands parsed from enum {self.enum_name} in {self.enum_file}")
+ bad = self._invalid_invocations(root, tree, prefix)
+ if bad:
+ path = sorted(bad)[0]
+ return False, (
+ f"docs reference unknown `{prefix}{path}` (not a valid command path for "
+ f"{self.enum_name} in {self.enum_file}; top-level: {', '.join(sorted(tree))}) "
+ f"in {bad[path][0]}"
+ )
+ return True, f"all `{prefix}*` references are valid command paths ({len(tree)} top-level)"
+
+
+CLI_CHECKS: list[CliSubcommandCheck] = [
+ CliSubcommandCheck(
+ id="vx-subcommands",
+ description="every `vx ` in docs is a real vortex-tui CLI subcommand",
+ enum_file="vortex-tui/src/lib.rs",
+ enum_name="Commands",
+ command_prefix="vx ", # fallback only
+ bin_truth_file="vortex-tui/Cargo.toml", # source the real binary name from `[[bin]] name`
+ ),
+]
+
+
+@dataclass
+class DocMembershipCheck:
+ """Assert every token captured by `mention_regex` (group 1) across `doc_files` is a member of the
+ allowed set derived from a source of truth by `allowed`. The doc analog of CliSubcommandCheck for
+ non-command facts — e.g. every `vortex-spark_` the docs advertise is a published scala
+ variant per java/settings.gradle.kts, so a stale variant (one removed from the build) is caught."""
+
+ id: str
+ description: str
+ doc_files: list[str]
+ mention_regex: str
+ allowed: Callable[[Path], set[str]]
+ region_fn: Callable[[str], str] | None = None # extract code regions first (e.g. python blocks)
+ scan_all_docs: bool = False # scan every doc (mentions may appear anywhere)
+
+ @staticmethod
+ def _outside(text: str, mention_regex: str, allowed: set[str]) -> set[str]:
+ """The set of `mention_regex` group-1 tokens in `text` that are NOT in `allowed` (pure; the
+ membership logic, exercised by the self-test on synthetic input without touching the repo)."""
+ return {t for t in re.findall(mention_regex, text) if t not in allowed}
+
+ def check(self, root: Path) -> tuple[bool, str]:
+ allowed = self.allowed(root)
+ if not allowed:
+ raise LookupError(f"[{self.id}] derived an empty allowed set — the source of truth moved")
+ files = all_doc_files(root) if self.scan_all_docs else [root / f for f in self.doc_files]
+ bad: dict[str, str] = {}
+ for p in files:
+ text = p.read_text(encoding="utf-8")
+ if self.region_fn is not None:
+ text = self.region_fn(text) # scope to code regions to avoid incidental prose matches
+ for tok in self._outside(text, self.mention_regex, allowed):
+ bad.setdefault(tok, str(p.relative_to(root)))
+ if bad:
+ t = sorted(bad)[0]
+ return False, (f"docs advertise `{t}`, not in the allowed set "
+ f"{{{', '.join(sorted(allowed))}}} (from source) in {bad[t]}")
+ return True, f"all advertised variants are in {{{', '.join(sorted(allowed))}}}"
+
+
+def _spark_scala_variants(root: Path) -> set[str]:
+ """The published Scala suffixes of the Spark connector, from java/settings.gradle.kts."""
+ settings = _strip_comments((root / "java/settings.gradle.kts").read_text(encoding="utf-8"))
+ return set(re.findall(r'include\("vortex-spark_(\d+(?:\.\d+)?)"\)', settings))
+
+
+# Python-flavored code-block languages — where `vortex.*` / `vx.*` API references live. Scoping to
+# these (not raw prose) keeps incidental matches out of the API-name check: `bench.vortex.dev` (URLs),
+# `vortex.h` (the C header), `${vortex.version}` (Maven), `vortex.write.batch.size` (a Spark option).
+_PYTHON_LANGS = frozenset({"python", "pycon", "py", "python3", "ipython", "ipython3"})
+
+
+# MyST directive classification. PROMPT/LANG directives carry python code (captured below). LEAF
+# directives carry code/content that is NEVER a nestable container — a non-python one (```{code-block}
+# bash`, a literalinclude, a toctree path list) is treated as OPAQUE, not descended into. Every OTHER
+# directive (`tab`, admonitions, `dropdown`, …) IS a container we descend into to find nested python.
+_PY_PROMPT_DIRECTIVES = frozenset({"doctest", "ipython", "ipython3"})
+_PY_LANG_DIRECTIVES = frozenset({"code-cell", "code-block", "sourcecode"})
+_LEAF_DIRECTIVES = frozenset({
+ "code-block", "code-cell", "sourcecode", "doctest", "ipython", "ipython3",
+ "literalinclude", "figure", "image", "math", "csv-table", "list-table",
+ "bibliography", "toctree", "mermaid", "raw",
+})
+
+
+def _classify_fence(info: str, fence_len: int) -> str:
+ """Classify a fence by its info string (and backtick length) into one of three kinds, shared by
+ `_scan_fences` and `_strip_fenced_blocks` so their handling can't diverge:
+ - 'python' — body is python code (```python, ```{doctest}, ```{code-block} python, a bare
+ ```{code-cell}) → captured as an API region.
+ - 'leaf' — a self-contained NON-python code/content block (```bash/```text, a 3-backtick
+ bare block, ```{code-block} bash, literalinclude/toctree/…) → OPAQUE.
+ - 'container' — `{tab}`/`{eval-rst}`/admonitions and 4+-backtick bare wrappers → descended into
+ (scan) / unwrapped (strip), because the body holds more markup: nested python
+ fences OR (for `{eval-rst}`) raw RST directives the prose passes must still see."""
+ info = info.strip()
+ if info.startswith("{"):
+ dm = re.match(r"\{([\w-]+)\}\s*(.*)", info)
+ directive = dm.group(1).lower() if dm else ""
+ arg = (dm.group(2).split() or [""])[0].lower() if dm else ""
+ if (directive in _PY_PROMPT_DIRECTIVES
+ or (directive in _PY_LANG_DIRECTIVES and arg in _PYTHON_LANGS)
+ or (directive == "code-cell" and arg == "")): # bare {code-cell}: page python kernel
+ return "python"
+ return "leaf" if directive in _LEAF_DIRECTIVES else "container"
+ if (info.split() or [""])[0].lower() in _PYTHON_LANGS:
+ return "python"
+ if info == "" and fence_len >= 4: # a 4+-backtick bare wrapper; a 3-backtick bare block is literal
+ return "container"
+ return "leaf"
+
+
+def _strip_pycon_output(body: str) -> str:
+ """If a captured python region is a pycon/doctest SESSION (has `>>>` lines), keep only the prompt
+ INPUT payloads (`>>>` / `...`) and drop interpreter OUTPUT lines — output such as a repr like
+ `` is not API usage and must not be checked. A plain python block (no
+ `>>>`) is returned unchanged."""
+ if not re.search(r"^\s*>>>", body, re.MULTILINE):
+ return body
+ return "\n".join(re.findall(r"^\s*(?:>>>|\.\.\.) ?(.*)$", body, re.MULTILINE))
+
+
+def _scan_fences(lines: list[str], regions: list[str]) -> None:
+ """Walk fenced blocks honoring the CommonMark/MyST rule that a fence opened by N backticks is closed
+ only by ≥N backticks — so a 3-backtick ```` ```python ```` nested inside a 4-backtick ```` ````{tab}
+ ```` container is NOT mis-paired. Python fences are captured; CONTAINER fences are recursed into so
+ nested python examples are still API-checked; LEAF (explicit non-python) fences are OPAQUE so a
+ literal python snippet shown inside a shell/heredoc example is not mistaken for real API usage.
+ Recursion always shrinks the body, so it terminates. See `_classify_fence` for the kinds."""
+ i = 0
+ while i < len(lines):
+ m = re.match(r"(\s*)(`{3,}|~{3,})(.*)$", lines[i])
+ if not m:
+ i += 1
+ continue
+ fence = m.group(2)
+ close = re.compile(r"\s*" + re.escape(fence[0]) + "{" + str(len(fence)) + r",}\s*$")
+ j = i + 1
+ while j < len(lines) and not close.match(lines[j]):
+ j += 1
+ body = lines[i + 1 : j]
+ kind = _classify_fence(m.group(3), len(fence))
+ if kind == "python":
+ regions.append(_strip_pycon_output("\n".join(body)))
+ elif kind == "container": # descend for nested python
+ _scan_fences(body, regions)
+ # leaf → opaque, do not descend
+ i = j + 1
+
+
+def _strip_fenced_blocks(text: str) -> str:
+ """Return `text` with python/leaf fenced blocks REMOVED but CONTAINER fences UNWRAPPED (open/close
+ markers dropped, body kept and recursively stripped). Scopes the RST-directive and bare-`>>>` passes
+ to prose: an opaque ```text/```bash block is gone (so a literal `>>> vortex.bad` / `.. code-block::
+ python` inside it is not scanned), but a ```{eval-rst}` container's raw RST body — which may hold
+ `.. code-block:: python` / `>>>` examples — is RETAINED so those passes still scan it. Markdown
+ python is already captured by `_scan_fences`, so dropping python leaf fences here costs no coverage."""
+ lines = text.split("\n")
+ out: list[str] = []
+ i = 0
+ while i < len(lines):
+ m = re.match(r"(\s*)(`{3,}|~{3,})(.*)$", lines[i])
+ if not m:
+ out.append(lines[i])
+ i += 1
+ continue
+ fence = m.group(2)
+ close = re.compile(r"\s*" + re.escape(fence[0]) + "{" + str(len(fence)) + r",}\s*$")
+ j = i + 1
+ while j < len(lines) and not close.match(lines[j]):
+ j += 1
+ if _classify_fence(m.group(3), len(fence)) == "container":
+ out.append(_strip_fenced_blocks("\n".join(lines[i + 1 : j]))) # unwrap: keep body as prose
+ # python / leaf → drop entirely
+ i = j + 1
+ return "\n".join(out)
+
+
+def _python_regions(text: str) -> str:
+ """Concatenate Python code regions: Markdown fences tagged with a python language (`python`,
+ `pycon`) OR a MyST code-cell/doctest directive (```` ```{doctest} pycon ````, ```` ```{code-cell}
+ python ````, ```` ```{code-block} python ````) — the dominant python-example form in these docs —
+ and reStructuredText `.. code-block:: python` / `.. doctest::` directives. Fence parsing is
+ backtick-length-aware so python fences nested inside MyST containers (e.g. ```` ````{tab} ````)
+ are not missed."""
+ regions: list[str] = []
+ _scan_fences(text.split("\n"), regions)
+ # The RST-directive and bare-`>>>` passes operate on PROSE (markdown fences stripped) so a literal
+ # `.. code-block:: python` / `>>> vortex.x` shown inside an opaque ```text/```bash fence is not
+ # scanned as live API usage. RST directives/doctests live in .rst files (no ``` fences), so
+ # stripping costs no real coverage; markdown python examples are already captured by `_scan_fences`.
+ prose = _strip_fenced_blocks(text)
+ lines = prose.split("\n")
+ i = 0
+ while i < len(lines):
+ m = re.match(r"(\s*)\.\.\s+(?:code-block|code|doctest)::\s*([\w-]*)\s*$", lines[i])
+ if m and (m.group(2).lower() in _PYTHON_LANGS or "doctest" in lines[i]):
+ indent = len(m.group(1))
+ j = i + 1
+ block: list[str] = []
+ while j < len(lines) and not lines[j].strip():
+ j += 1
+ while j < len(lines):
+ if lines[j].strip() and (len(lines[j]) - len(lines[j].lstrip())) <= indent:
+ break
+ block.append(lines[j])
+ j += 1
+ regions.append(_strip_pycon_output("\n".join(block))) # `.. doctest::` blocks: drop output
+ i = j
+ continue
+ i += 1
+ # Prompt-only RST doctest blocks (no directive): collect the code after every `>>>`/`...` prompt,
+ # so bare interpreter examples (e.g. api/python/expr.rst) are API-checked too. Same PROSE scoping
+ # as the RST-directive pass; python/pycon fences with `>>>` are already captured by `_scan_fences`.
+ regions += re.findall(r"^\s*(?:>>>|\.\.\.) ?(.*)$", prose, re.MULTILINE)
+ return "\n".join(regions)
+
+
+def _canonical_payload_classes(enum_body: str, api: set[str] | None) -> set[str]:
+ """Map every `Canonical` variant to the last path-segment of its PAYLOAD type — `List(ListViewArray)`
+ -> ListViewArray, `Foo(crate::a::Bar)` -> Bar — NOT the variant name. So `ListArray` (a non-canonical
+ list encoding) is correctly excluded and the canonical `ListViewArray` is recognized. Keep only
+ payloads exposed as Python classes (`Decimal`/`Variant`/`ListViewArray` are canonical in Rust but
+ not yet in the Python API).
+
+ FAILS LOUD if any variant declaration can't be parsed into `Variant(Payload)` form (a struct-style
+ or multiline-payload variant), rather than silently skipping it — a silent skip would shrink the
+ expected docs set and let an enum-shape change slip past the lock. Separated from the file read so
+ the self-test can exercise it on a synthetic enum body."""
+ classes: set[str] = set()
+ for raw in enum_body.splitlines():
+ line = raw.strip().rstrip(",").strip()
+ if not line or line.startswith("#") or line.startswith("//"):
+ continue # attribute / comment / blank — structural noise, not a variant
+ if not re.match(r"[A-Z]\w*", line):
+ continue # not a variant declaration line
+ m = re.fullmatch(r"[A-Z]\w*\(\s*([\w:]+(?:<[^>]*>)?)\s*\)", line)
+ if not m:
+ raise LookupError(f"unparseable Canonical variant {line!r} (expected `Variant(Payload)`)")
+ cls = m.group(1).split("::")[-1].split("<")[0] # crate::a::Bar -> Bar
+ if api is None or cls in api: # api=None -> keep ALL payloads (not just Python-exposed)
+ classes.add(cls)
+ return classes
+
+
+def _canonical_all_payloads(root: Path) -> set[str]:
+ """ALL canonical-encoding payload classes from the Rust `Canonical` enum (NOT filtered to the Python
+ API) — e.g. DecimalArray, ListViewArray, VariantArray. The source of truth for the DType -> canonical
+ encoding column of concepts/arrays.md."""
+ canon = _strip_rust_comments((root / "vortex-array/src/canonical.rs").read_text(encoding="utf-8"))
+ m = re.search(r"pub enum Canonical\s*\{(.*?)\n\}", canon, re.DOTALL)
+ if not m:
+ raise LookupError("could not find `pub enum Canonical` in vortex-array/src/canonical.rs")
+ return _canonical_payload_classes(m.group(1), None)
+
+
+def _canonical_python_classes(root: Path) -> set[str]:
+ """The canonical-encoding Python classes: each Rust `Canonical` enum variant (canonical.rs) mapped
+ to the Python class of its PAYLOAD type, kept only when that class exists in vortex's public API.
+ The source of truth for which encodings arrays.rst's "Canonical Encodings" section should list.
+ See `_canonical_payload_classes` for the payload-vs-variant-name mapping and fail-loud parsing."""
+ canon = _strip_rust_comments((root / "vortex-array/src/canonical.rs").read_text(encoding="utf-8"))
+ m = re.search(r"pub enum Canonical\s*\{(.*?)\n\}", canon, re.DOTALL)
+ if not m:
+ raise LookupError("could not find `pub enum Canonical` in vortex-array/src/canonical.rs")
+ return _canonical_payload_classes(m.group(1), _vortex_public_api(root))
+
+
+def _listed_canonical_classes(rst_text: str) -> set[str]:
+ """The `vortex.` autoclasses listed under arrays.rst's "Canonical Encodings" section, bounded
+ by the next RST section heading (or EOF) so a class in a LATER section (Utility, Compressed) is not
+ counted. A pure helper so the section-boundary parsing is self-testable on synthetic input, not just
+ the live docs file."""
+ rst_text = rst_text.replace("\r\n", "\n") # tolerate CRLF checkouts (Windows / autocrlf)
+ m = re.search(r"Canonical Encodings\n-+\n(.*?)(?:\n[A-Z][\w ]+\n-+\n|\Z)", rst_text, re.DOTALL)
+ if not m:
+ return set()
+ return set(re.findall(r"\.\.\s*autoclass::\s*vortex\.(\w+)", m.group(1)))
+
+
+@dataclass
+class CanonicalSectionCheck:
+ """Assert arrays.rst's "Canonical Encodings" section lists EXACTLY the canonical Python classes
+ (set equality, so a mislabeled non-canonical class like `VarBinArray` AND a missing canonical class
+ like `FixedSizeListArray` are both caught) — sourced from the Rust `Canonical` enum."""
+
+ id: str
+ description: str
+
+ def check(self, root: Path) -> tuple[bool, str]:
+ expected = _canonical_python_classes(root)
+ if not expected:
+ raise LookupError(f"[{self.id}] derived an empty canonical set")
+ text = (root / "docs/api/python/arrays.rst").read_text(encoding="utf-8")
+ listed = _listed_canonical_classes(text)
+ if listed != expected:
+ return False, (f"arrays.rst Canonical Encodings section mismatch vs Rust Canonical enum: "
+ f"extra={sorted(listed - expected)} missing={sorted(expected - listed)}")
+ return True, f"canonical section lists exactly the {len(expected)} canonical Python classes"
+
+
+def _stat_names(root: Path) -> set[str]:
+ """The canonical statistic names from `Stat::name()` in vortex-array expr/stats/mod.rs — the source
+ of truth for the statistics arrays.md should advertise. Comments are stripped first; see
+ `_stat_names_from` for the parse + per-variant cross-check that makes it fail loud rather than
+ silently shrink the expected set."""
+ src = _strip_rust_comments((root / "vortex-array/src/expr/stats/mod.rs").read_text(encoding="utf-8"))
+ blocks = re.findall(r"fn name\(&self\)\s*->\s*&str\s*\{(.*?)\n\s*\}", src, re.DOTALL)
+ if len(blocks) != 1: # exactly one Stat::name(), else a decoy could mis-source the names
+ raise LookupError(f"expected exactly 1 Stat::name() in stats/mod.rs, found {len(blocks)}")
+ enum_m = re.search(r"pub enum Stat\s*\{(.*?)\n\}", src, re.DOTALL)
+ if not enum_m:
+ raise LookupError("could not find `pub enum Stat` in vortex-array/src/expr/stats/mod.rs")
+ return _stat_names_from(blocks[0], enum_m.group(1))
+
+
+def _stat_enum_variants(enum_body: str) -> set[str]:
+ """The variant names of the `Stat` enum (`Variant = N,` or `Variant,` forms)."""
+ variants = set(re.findall(r"^\s*([A-Z]\w*)\s*(?:=[^,\n]+)?,", enum_body, re.MULTILINE))
+ if not variants:
+ raise LookupError("no Stat enum variants parsed")
+ return variants
+
+
+def _stat_names_from(name_body: str, enum_body: str) -> set[str]:
+ """Parse the `Stat::name()` arms and CROSS-CHECK that exactly one name was produced per enum variant.
+ This is the definitive guard against ANY arm form that compiles while collapsing variants — a
+ same-line OR multi-line or-pattern, a wildcard, a missing arm — all yield fewer names than variants
+ and fail loud (the enum variant count is the ground truth). Separated from the file read so the
+ self-test can drive both halves on synthetic input."""
+ names = _parse_stat_name_arms(name_body)
+ variants = _stat_enum_variants(enum_body)
+ if len(names) != len(variants):
+ raise LookupError(f"Stat::name() produced {len(names)} names but the enum has {len(variants)} "
+ "variants (a collapsed or-pattern / wildcard / missing arm?)")
+ return names
+
+
+def _parse_stat_name_arms(body: str) -> set[str]:
+ """Names from a `Stat::name()` match body. Validates EVERY `=>` arm: the left side must be exactly
+ one `Self::Variant` (NOT an or-pattern `Self::A | Self::B`, NOT a wildcard `_`, NOT a guard) and the
+ right side must be a string literal — otherwise FAIL LOUD, because any of those forms could compile
+ while collapsing several variants into one name and silently shrinking the expected set. Also fails
+ on duplicate literals. Separated from the file read so the self-test can exercise the fail-loud
+ paths on synthetic input."""
+ arm_lines = [ln for ln in body.splitlines() if "=>" in ln]
+ if not arm_lines:
+ raise LookupError("Stat::name() yielded no match arms")
+ names: list[str] = []
+ for ln in arm_lines:
+ lhs, _, rhs = ln.partition("=>")
+ if not re.fullmatch(r"Self::\w+", lhs.strip()): # or-pattern / wildcard / guard
+ raise LookupError(f"Stat::name() arm pattern {lhs.strip()!r} is not a single `Self::Variant`; "
+ "extend the parser")
+ lit = re.fullmatch(r'"(\w+)"', rhs.strip().rstrip(",").strip())
+ if not lit:
+ raise LookupError(f"Stat::name() arm returns a non-literal {rhs.strip()!r}; extend the parser")
+ names.append(lit.group(1))
+ dupes = sorted({n for n in names if names.count(n) > 1})
+ if dupes: # two variants sharing a name string is a source bug the lock must not silently absorb
+ raise LookupError(f"Stat::name() returns duplicate literals {dupes}")
+ return set(names)
+
+
+def _listed_stat_names(md_text: str) -> tuple[list[str], int]:
+ """(names, n_bullets) for arrays.md's `## Statistics` section: the leading `` `name` `` code-span of
+ each bullet (in order, duplicates preserved) and the TOTAL bullet count. A bullet WITHOUT a leading
+ code span (a malformed/extra entry like `* true_count: ...`) makes len(names) < n_bullets, so the
+ caller can fail it instead of silently ignoring it. Section bounded at the next `## ` heading; a
+ pure helper so the parsing is self-testable on synthetic md. (Wrapped continuation lines don't start
+ with `*`, so they're not counted as bullets.)"""
+ md_text = md_text.replace("\r\n", "\n") # tolerate CRLF checkouts (Windows / autocrlf)
+ m = re.search(r"## Statistics\n(.*?)(?:\n## |\Z)", md_text, re.DOTALL)
+ if not m:
+ return [], 0
+ bullets = re.findall(r"^\s*\*\s+(.*)$", m.group(1), re.MULTILINE)
+ names = [cm.group(1) for b in bullets if (cm := re.match(r"`(\w+)`", b))]
+ return names, len(bullets)
+
+
+@dataclass
+class StatsListCheck:
+ """Assert arrays.md's `## Statistics` section lists EXACTLY the `Stat::name()` values — sourced from
+ the Rust `Stat` enum. Fails on: a fabricated stat (`true_count`), a missing one (`sum`), a duplicate,
+ OR a bullet without a leading `` `name` `` code-span (which a bare set comparison would ignore)."""
+
+ id: str
+ description: str
+
+ def check(self, root: Path) -> tuple[bool, str]:
+ expected = _stat_names(root)
+ names, n_bullets = _listed_stat_names((root / "docs/concepts/arrays.md").read_text(encoding="utf-8"))
+ if n_bullets != len(names):
+ return False, f"{n_bullets - len(names)} Statistics bullet(s) lack a leading `name` code-span"
+ dupes = sorted({n for n in names if names.count(n) > 1})
+ if dupes:
+ return False, f"arrays.md Statistics section lists duplicate stats: {dupes}"
+ listed = set(names)
+ if listed != expected:
+ return False, (f"arrays.md Statistics list mismatch vs Stat::name(): "
+ f"extra={sorted(listed - expected)} missing={sorted(expected - listed)}")
+ return True, f"statistics section lists exactly the {len(expected)} Stat names"
+
+
+def _vortex_submodules(root: Path) -> set[str]:
+ """The importable submodules of the `vortex` package (`.py` files + package dirs). These are MODULES,
+ not callables — so `vortex.(...)` (e.g. `vortex.dataset(...)`) is a module-called-as-
+ function error, as opposed to a real factory like `vortex.array(...)`."""
+ pkg = root / "vortex-python/python/vortex"
+ mods = {p.stem for p in pkg.iterdir() if p.suffix == ".py" and p.stem != "__init__"}
+ mods |= {p.name for p in pkg.iterdir() if p.is_dir() and (p / "__init__.py").exists()}
+ return mods
+
+
+def _vortex_public_api(root: Path) -> set[str]:
+ """The public top-level names of the `vortex` Python package: `__all__` (parsed from __init__.py
+ via `ast`) PLUS the importable submodules (so `vortex.store`, `vortex.arrow`, etc. — accessible but
+ not all in `__all__` — are recognized). The allowed set for the docs' `vortex.` references.
+ FAILS LOUD if `__all__` is not a literal list/tuple of names (e.g. became computed), rather than
+ silently under-reporting the allowed set."""
+ pkg = root / "vortex-python/python/vortex"
+ names: set[str] = set()
+ found = False
+ for node in ast.walk(ast.parse((pkg / "__init__.py").read_text(encoding="utf-8"))):
+ if isinstance(node, ast.Assign) and any(getattr(t, "id", None) == "__all__" for t in node.targets):
+ if not (isinstance(node.value, (ast.List, ast.Tuple))
+ and all(isinstance(e, ast.Constant) for e in node.value.elts)):
+ raise LookupError("vortex `__all__` is not a literal list of names; the API-name check needs updating")
+ names |= {e.value for e in node.value.elts}
+ found = True
+ if not found:
+ raise LookupError("could not find a literal `__all__` in vortex/__init__.py")
+ return names | _vortex_submodules(root)
+
+
+@dataclass
+class DTypeListCheck:
+ """Assert the architecture overview's `DType` enum row lists EXACTLY the `DType` variants (sourced
+ from the Rust enum) — set equality, so it can't drift to a stale subset (it had dropped Decimal/
+ FixedSizeList/Union/Variant) NOR list a fabricated extra variant. The row is the table cell beginning
+ ``DType` enum:`."""
+
+ id: str
+ description: str
+
+ def check(self, root: Path) -> tuple[bool, str]:
+ expected = _dtype_variant_names(root)
+ text = (root / "docs/developer-guide/internals/architecture.md").read_text(encoding="utf-8")
+ m = re.search(r"`DType` enum:([^|\n]*)", text)
+ if not m:
+ return False, "architecture.md has no ``DType` enum:` row to check"
+ listed = set(re.findall(r"\b([A-Z]\w*)\b", m.group(1)))
+ if listed != expected:
+ return False, (f"architecture.md `DType` row mismatch vs the Rust enum: "
+ f"extra={sorted(listed - expected)} missing={sorted(expected - listed)}")
+ return True, f"architecture.md `DType` row lists exactly the {len(expected)} variants"
+
+
+# Wheel target triple -> the platform label the Python docs advertise. New triples FAIL LOUD so the
+# docs (and this map) must be updated rather than silently drifting.
+_WHEEL_LABEL = {
+ "aarch64-apple-darwin": "Apple Silicon macOS",
+ "x86_64-apple-darwin": "Intel macOS",
+ "aarch64-unknown-linux-gnu": "ARM64 Linux",
+ "x86_64-unknown-linux-gnu": "x86_64 Linux",
+}
+
+
+def _wheel_platform_labels(root: Path) -> set[str]:
+ """The platform labels the Python prebuilt-wheel docs should list, derived from the build target
+ triples in .github/workflows/package.yml (mapped via `_WHEEL_LABEL`). FAILS LOUD on an unknown
+ triple so a new platform forces a docs + map update."""
+ yml = (root / ".github/workflows/package.yml").read_text(encoding="utf-8")
+ # Scope to the `prepare-python` wheel-build job's block (start-of-job to the next top-level job key),
+ # so an inline `{...target:...}` table in an UNRELATED job can't leak into the wheel platform set.
+ job = re.search(r"^ prepare-python:\n(.*?)(?=^ \w[\w-]*:\n|\Z)", yml, re.DOTALL | re.MULTILINE)
+ if not job:
+ raise LookupError("no `prepare-python` job in .github/workflows/package.yml; the wheel lock scope moved")
+ # Within that job, match the wheel build MATRIX entries — `{ ..., target: , ... }` (inline brace
+ # table); the JNI jobs use `- target: ` (no braces) and are out of scope anyway. Match `target:`
+ # ANYWHERE inside the braces (order-insensitive) so a reordered/new field can't hide a target. Capture
+ # ANY triple token (not a fixed arch/os set) so a genuinely new platform reaches `_WHEEL_LABEL` + FAILS LOUD.
+ triples = set(re.findall(r"\{[^{}]*\btarget:\s*([\w-]+)[^{}]*\}", job.group(1)))
+ if not triples:
+ raise LookupError("no wheel matrix target triples found in .github/workflows/package.yml")
+ labels: set[str] = set()
+ for t in triples:
+ if t not in _WHEEL_LABEL:
+ raise LookupError(f"unknown wheel target triple {t!r}; add it to _WHEEL_LABEL and the Python docs")
+ labels.add(_WHEEL_LABEL[t])
+ return labels
+
+
+@dataclass
+class WheelPlatformCheck:
+ """Assert the Python docs' prebuilt-wheel platform list includes every platform the packaging
+ workflow builds (`_wheel_platform_labels`) — so adding/removing a wheel target forces a docs update."""
+
+ id: str
+ description: str
+
+ def check(self, root: Path) -> tuple[bool, str]:
+ expected = _wheel_platform_labels(root)
+ text = (root / "docs/api/python/index.rst").read_text(encoding="utf-8")
+ # the bullet list under "available for:" — set equality, so a stale advertised platform (built
+ # target removed) is caught too, not only a missing one.
+ m = re.search(r"available for:\n\n((?:\* .*\n)+)", text)
+ listed = set(re.findall(r"\* (.+)", m.group(1))) if m else set()
+ if listed != expected:
+ return False, (f"python/index.rst wheel platforms mismatch vs package.yml: "
+ f"extra={sorted(listed - expected)} missing={sorted(expected - listed)}")
+ return True, f"python/index.rst lists exactly the {len(expected)} built wheel platforms"
+
+
+@dataclass
+class CanonicalConceptsTableCheck:
+ """Assert concepts/arrays.md's DType->canonical-encoding table uses EXACTLY the real canonical
+ encodings (the `Canonical` enum payloads) in its encoding column (set equality — caught the stale
+ table that omitted DecimalArray/VariantArray)."""
+
+ id: str
+ description: str
+
+ def check(self, root: Path) -> tuple[bool, str]:
+ expected = _canonical_all_payloads(root)
+ text = (root / "docs/concepts/arrays.md").read_text(encoding="utf-8")
+ m = re.search(r"Canonical Encoding\s*\|\n\|[-\s|]+\n(.*?)(?:\n\n|\n##|\Z)", text, re.DOTALL)
+ if not m:
+ return False, "arrays.md has no DType->canonical-encoding table"
+ listed = set(re.findall(r"`(\w+Array)`", m.group(1)))
+ if listed != expected:
+ return False, (f"arrays.md canonical-encoding column mismatch vs the Canonical enum: "
+ f"extra={sorted(listed - expected)} missing={sorted(expected - listed)}")
+ return True, f"arrays.md canonical table uses exactly the {len(expected)} canonical encodings"
+
+
+@dataclass
+class EncodingsTableCheck:
+ """Assert architecture.md's `## Encodings` table lists EXACTLY the `encodings/*` crates (set equality
+ — caught a stale `vortex-roaring`/`vortex-dict` that no longer exist AND omitted vortex-pco/zstd/
+ parquet-variant). The table is the `| Crate | Technique |` block under the `## Encodings` heading."""
+
+ id: str
+ description: str
+
+ def check(self, root: Path) -> tuple[bool, str]:
+ expected = _encoding_crate_names(root)
+ text = (root / "docs/developer-guide/internals/architecture.md").read_text(encoding="utf-8")
+ m = re.search(r"## Encodings\b(.*?)(?:\n## |\Z)", text, re.DOTALL)
+ if not m:
+ return False, "architecture.md has no `## Encodings` section"
+ listed = set(re.findall(r"`(vortex-[\w-]+)`", m.group(1)))
+ if listed != expected:
+ return False, (f"architecture.md Encodings table mismatch vs encodings/*: "
+ f"extra={sorted(listed - expected)} missing={sorted(expected - listed)}")
+ return True, f"architecture.md Encodings table lists exactly the {len(expected)} encoding crates"
+
+
+@dataclass
+class DtypesTableCheck:
+ """Assert concepts/dtypes.md's Logical Types table lists EXACTLY the `DType` variants (set equality),
+ so the user-facing dtype list can't drift from the Rust enum (it had omitted Union/Variant)."""
+
+ id: str
+ description: str
+
+ def check(self, root: Path) -> tuple[bool, str]:
+ expected = _dtype_variant_names(root)
+ text = (root / "docs/concepts/dtypes.md").read_text(encoding="utf-8")
+ # the MAIN dtype table only — bounded at the first sub/section heading (e.g. `### Primitive`),
+ # so per-type subsection tables (PType I8/F64/...) aren't mistaken for DType variants.
+ m = re.search(r"## Logical Types\b(.*?)(?:\n#{2,4} |\Z)", text, re.DOTALL)
+ if not m:
+ return False, "dtypes.md has no `## Logical Types` section"
+ # the Name column: a leading `| `` |` cell with a capitalized identifier
+ listed = set(re.findall(r"^\|\s*`([A-Z]\w*)`", m.group(1), re.MULTILINE))
+ if listed != expected:
+ return False, (f"dtypes.md Logical Types table mismatch vs the DType enum: "
+ f"extra={sorted(listed - expected)} missing={sorted(expected - listed)}")
+ return True, f"dtypes.md Logical Types table lists exactly the {len(expected)} DType variants"
+
+
+@dataclass
+class PythonModuleCallCheck:
+ """Flag a `vortex.(...)` / `vx.(...)` in any doc's PYTHON region — calling a
+ MODULE as a function (the `vortex.dataset(...)` drift PR-2.2 fixed: `dataset` is a submodule, not a
+ callable). The `python-api-names` membership check only proves the NAME exists; this catches the
+ semantic module-as-callable misuse a static name check cannot."""
+
+ id: str
+ description: str
+
+ def check(self, root: Path) -> tuple[bool, str]:
+ mods = _vortex_submodules(root)
+ pat = re.compile(r"(?(...)` module-as-callable misuse ({len(mods)} submodules)"
+
+
+DOC_CHECKS: list[DocMembershipCheck] = [
+ DocMembershipCheck(
+ id="architecture-crate-refs",
+ description="every `vortex-` referenced in architecture.md is a real workspace crate",
+ doc_files=["docs/developer-guide/internals/architecture.md"],
+ # crate-form refs only: `vortex-array/src/expr` yields `vortex-array` (a real crate); a bare
+ # `vortex-expr`/`vortex-roaring` (no longer / never a crate) is flagged. The published Java Spark
+ # artifacts `vortex-spark_` aren't Cargo crates, so they're added to the allowed set.
+ mention_regex=r"`(vortex-[a-z][\w.-]*)",
+ allowed=lambda root: _workspace_crate_names(root) | {f"vortex-spark_{v}" for v in _spark_scala_variants(root)},
+ ),
+ DocMembershipCheck(
+ id="spark-scala-variants",
+ description="every `vortex-spark_` advertised in the docs/READMEs is a published scala variant",
+ doc_files=["docs/user-guide/spark.md", "docs/developer-guide/internals/architecture.md",
+ "README.md", "java/README.md"],
+ mention_regex=r"vortex-spark_(\d+(?:\.\d+)?)",
+ allowed=_spark_scala_variants,
+ ),
+ DocMembershipCheck(
+ id="python-api-names",
+ description="every top-level `vortex.` / `vx.` in docs PYTHON blocks is a real "
+ "public name of the vortex package (catches a fabricated/renamed Python API)",
+ doc_files=[], # scans every doc
+ scan_all_docs=True,
+ region_fn=_python_regions, # python code regions only — not prose/URLs/Maven/Spark-option text
+ mention_regex=r"(?encoding table == the Rust Canonical enum payloads",
+ ),
+ WheelPlatformCheck(
+ id="python-wheel-platforms",
+ description="python/index.rst lists every wheel platform built by package.yml",
+ ),
+]
+
+
+@dataclass
+class EncodingStabilityCheck:
+ """Validate the stable-encoding-set derivation (encoding_stability.py): it parses
+ `register_default_encodings` + the crate `initialize()` fns and yields non-empty, pairwise-disjoint
+ stable / unstable / parked encoding sets. This checks the DERIVATION's health only — it does NOT
+ enforce that each stable encoding has a spec section (that is the tripwire, deliberately not built
+ here). Fails loud if the registration source or the `unstable_encodings` feature moved."""
+
+ id: str
+ description: str
+
+ def check(self, root: Path) -> tuple[bool, str]:
+ cls = encoding_stability.classify_encodings(root)
+ stable = {n for n, c in cls.items() if c == encoding_stability.STABLE}
+ unstable = {n for n, c in cls.items() if c == encoding_stability.UNSTABLE}
+ parked = {n for n, c in cls.items() if c == encoding_stability.PARKED}
+ if not stable:
+ return False, "derived an empty stable-encoding set"
+ if stable & unstable or stable & parked or unstable & parked:
+ return False, "stable/unstable/parked encoding sets are not disjoint"
+ return True, (f"{len(stable)} stable / {len(unstable)} unstable / {len(parked)} parked encodings "
+ f"derived (unstable={sorted(unstable)})")
+
+
+ENCODING_CHECKS: list[EncodingStabilityCheck] = [
+ EncodingStabilityCheck(
+ id="encoding-stability-set",
+ description="the stable-encoding-set derivation parses and yields disjoint, non-empty sets",
+ ),
+]
+
+
+# --- Spec-conformance tripwire (shadow / observe mode) ----------------------------------------------
+# A guardrail that keeps docs/specification/encoding-format.md honest against the code. It runs in
+# SHADOW mode: it REPORTS coverage gaps and would-be drift, NON-BLOCKING, so CI stays green while
+# per-encoding byte-layout sections are added incrementally. It authors no spec content and never
+# contributes to the exit code; flipping it to hard enforcement is a later, deliberate step.
+#
+# Two things it observes:
+# 1. COVERAGE — every STABLE encoding (encoding_stability.stable_encodings, code-derived) should
+# gain a per-encoding byte-layout section. All 33 stable encodings now do (33/33); validity is
+# cross-cutting and specified once, NOT as a per-encoding section. The report keeps sizing this so
+# a newly-added stable encoding without a section surfaces immediately.
+# 2. DRIFT — each per-encoding LOCK (registered in ENCODING_LAYOUT_LOCKS with its section in a later
+# task) derives its invariant from encodings/*/src and compares it to the value the spec section
+# pins; a divergence is reported as WOULD-BE drift — exactly what hard enforcement would reject.
+#
+# --- Per-encoding byte-layout SECTION CONVENTION (defined here; a later task authors the sections) ---
+# A stable encoding `` (named EXACTLY as encoding_stability.stable_encodings() yields it — the
+# Rust struct ident, e.g. `ALP`, `BitPacked`, `FoR`, `DecimalByteParts`) is COVERED iff
+# encoding-format.md contains a MyST target-label line of the exact form
+#
+# (encoding-layout-)=
+#
+# on its own line, immediately preceding that encoding's byte-layout heading. Example:
+#
+# (encoding-layout-ALP)=
+# ### `vortex.alp` — Byte layout
+# ...
+#
+# Why the anchor keys on the STABLE-SET NAME rather than the wire ID (`vortex.alp`, `fastlanes.for`):
+# * The stable-set name is EXACTLY the identity derived from code, so both sides stay
+# code-authoritative with zero name-mapping — coverage is a pure set difference.
+# * It sidesteps wire-ID ambiguity: the encoding IDs mix prefixes (`vortex.*` AND `fastlanes.*` for
+# the FastLanes family), so keying on wire IDs would need a fragile per-crate derivation.
+# * The invisible anchor decouples the machine key from the human-facing heading text (which should
+# still show the wire ID for readers), so coverage detection NEVER collides with the cross-cutting
+# Validity section — that section names encodings in prose and sub-headings but carries no
+# `encoding-layout-*` anchor, so it correctly counts as zero coverage today.
+LAYOUT_ANCHOR_RE = re.compile(r"^\(encoding-layout-([A-Za-z0-9]+)\)=\s*$", re.MULTILINE)
+ENCODING_FORMAT_DOC = "docs/specification/encoding-format.md"
+# The per-encoding byte-layout sections live on FAMILY pages under this directory (canonical.md,
+# containers.md, …), linked from ENCODING_FORMAT_DOC's toctree. Coverage is scanned across BOTH the
+# top-level page and every family page, so a section authored on any of them counts.
+ENCODING_FORMAT_FAMILY_DIR = "docs/specification/encoding-format"
+
+
+def layout_anchor_label(name: str) -> str:
+ """The MyST target label a byte-layout section for stable encoding `name` must carry (the section
+ convention above). One function so the parser and any future authoring tool cannot disagree."""
+ return f"encoding-layout-{name}"
+
+
+def parse_layout_anchors(text: str) -> set[str]:
+ """The set of encoding names `text` marks as having a byte-layout section, per the anchor
+ convention. Fenced code blocks are stripped first so an anchor shown inside an EXAMPLE block is not
+ miscounted as real coverage. Pure (no I/O) so the self-test can drive it on synthetic input."""
+ return set(LAYOUT_ANCHOR_RE.findall(_strip_fenced_blocks(text)))
+
+
+def _covered_from_texts(texts: Iterable[str]) -> set[str]:
+ """Union of byte-layout anchors across multiple doc texts — the aggregation `covered_encodings`
+ performs over the top-level page + family pages. Pure (no I/O) so the self-test can drive the
+ multi-page scan surface on synthetic input."""
+ covered: set[str] = set()
+ for text in texts:
+ covered |= parse_layout_anchors(text)
+ return covered
+
+
+def _encoding_format_docs(root: Path) -> list[Path]:
+ """Every doc scanned for byte-layout anchors: the top-level encoding-format.md plus every family
+ page under docs/specification/encoding-format/*.md. A missing file/dir simply contributes nothing
+ (the spec pages may not be authored yet); shadow-only, never fatal."""
+ docs: list[Path] = []
+ top = root / ENCODING_FORMAT_DOC
+ if top.exists():
+ docs.append(top)
+ family = root / ENCODING_FORMAT_FAMILY_DIR
+ if family.is_dir():
+ docs.extend(sorted(family.glob("*.md")))
+ return docs
+
+
+def covered_encodings(root: Path) -> set[str]:
+ """Encodings whose byte-layout section is present anywhere in the encoding-format spec — the
+ top-level page or any family page (empty today). Shadow-only, never fatal."""
+ return _covered_from_texts(p.read_text(encoding="utf-8") for p in _encoding_format_docs(root))
+
+
+def coverage_gaps(stable: set[str], covered: set[str]) -> list[str]:
+ """Stable encodings still lacking a byte-layout section — the coverage gap the shadow report sizes."""
+ return sorted(stable - covered)
+
+
+@dataclass
+class EncodingLayoutLock:
+ """EXTENSION POINT — a per-encoding byte-layout invariant, registered in ENCODING_LAYOUT_LOCKS by a
+ later authoring task alongside that encoding's spec section. Both sides are DERIVED, never
+ hard-coded: `derive_code` computes the invariant from encodings/*/src (the source of truth) and
+ `derive_spec` reads the value the byte-layout section pins (typically from within that encoding's
+ section, located via the anchor convention). WOULD-BE drift = the two disagree — exactly what hard
+ enforcement will reject once the shadow period ends. In SHADOW mode the divergence is only reported.
+
+ `check(root, override_code=...)` mirrors ValueMatch's self-test hook: overriding the code side with
+ a sentinel proves the lock detects drift (the derived value no longer matches the spec side)."""
+
+ encoding: str # a member of encoding_stability.stable_encodings()
+ description: str
+ derive_code: Callable[[Path], str] # invariant derived from source (authoritative)
+ derive_spec: Callable[[Path], str] # the value the spec section pins (the doc side)
+
+ def check(self, root: Path, *, override_code: str | None = None) -> tuple[bool, str]:
+ """Return (drifted, detail). `drifted` is True when the code-derived invariant disagrees with
+ the value the spec pins. A raised LookupError/OSError is the caller's to catch — in shadow mode
+ the caller downgrades it to a non-fatal note."""
+ code = override_code if override_code is not None else self.derive_code(root)
+ spec = self.derive_spec(root)
+ if code != spec:
+ return True, f"code derives {code!r} but spec pins {spec!r}"
+ return False, f"code and spec agree ({code!r})"
+
+
+# Empty today: the per-encoding locks land WITH their spec sections in later tasks. Registering a
+# lock here makes the tripwire hard-check that encoding's invariant (still shadow-reported until the
+# global flip to enforcement). See EncodingLayoutLock for the contract each entry must satisfy.
+ENCODING_LAYOUT_LOCKS: list[EncodingLayoutLock] = []
+
+
+def shadow_report(root: Path) -> None:
+ """Print the SHADOW-mode spec-conformance report (coverage + would-be drift), NON-BLOCKING. Never
+ raises and never affects the exit code: a derivation error is downgraded to a note here because the
+ blocking EncodingStabilityCheck already guards the derivation's health."""
+ print("\n--- spec-conformance tripwire (SHADOW / observe mode — non-blocking, REPORTS only) ---")
+ try:
+ stable = encoding_stability.stable_encodings(root)
+ except (LookupError, OSError) as e:
+ print(f" shadow: could not derive the stable set ({type(e).__name__}: {e}); "
+ "the blocking encoding-stability check reports the cause.")
+ return
+ covered = covered_encodings(root)
+ gaps = coverage_gaps(stable, covered)
+ print(f" stable must-spec set ({len(stable)}): {', '.join(sorted(stable))}")
+ print(f" byte-layout coverage: {len(covered)}/{len(stable)} covered "
+ f"(convention: an `(encoding-layout-)=` anchor in {ENCODING_FORMAT_DOC} "
+ f"or {ENCODING_FORMAT_FAMILY_DIR}/*.md)")
+ for name in sorted(stable):
+ print(f" [{'COVERED' if name in covered else 'GAP '}] {name}")
+ stray = sorted(covered - stable)
+ if stray: # an anchor for a name NOT in the stable set (typo'd / stale) — surfaced, not a gap
+ print(f" NOTE: byte-layout anchors for non-stable names (typo/stale?): {', '.join(stray)}")
+ try: # echo the feature-gated-stable divergence so a reviewer isn't surprised Zstd is must-spec
+ fg = sorted(encoding_stability.feature_gated_stable(root))
+ except (LookupError, OSError):
+ fg = []
+ if fg:
+ print(f" NOTE: {', '.join(fg)} in the must-spec set via a non-unstable feature gate "
+ "(feature-gate divergence; see encoding_stability.py).")
+ drifts = 0
+ print(f" per-encoding locks registered: {len(ENCODING_LAYOUT_LOCKS)}")
+ for lock in ENCODING_LAYOUT_LOCKS:
+ try:
+ drifted, detail = lock.check(root)
+ except (LookupError, OSError) as e:
+ print(f" [LOCK-ERR] {lock.encoding}: {type(e).__name__}: {e}")
+ continue
+ if drifted:
+ drifts += 1
+ print(f" [{'WOULD-DRIFT' if drifted else 'ok '}] {lock.encoding}: {detail}")
+ print(f" SHADOW SUMMARY: {len(covered)}/{len(stable)} byte-layout sections, {len(gaps)} coverage "
+ f"gap(s), {drifts} would-be drift(s) — REPORTED, not enforced (CI stays green).")
+
+
+def _all_checks() -> list:
+ """Every check, regardless of type — all share `.id` and `.check(root) -> (ok, detail)`."""
+ return [*REGISTRY, *CLI_CHECKS, *DOC_CHECKS, *SECTION_CHECKS, *ENCODING_CHECKS]
+
+
+def run_checks(root: Path, verbose: bool) -> int:
+ failures: list[str] = []
+ checks = _all_checks()
+ for chk in checks:
+ try:
+ ok, detail = chk.check(root)
+ except (LookupError, OSError) as e:
+ ok, detail = False, f"{type(e).__name__}: {e}"
+ if not ok:
+ failures.append(f" [{chk.id}] {detail}")
+ if verbose or not ok:
+ print(f"{'PASS' if ok else 'FAIL'} {chk.id}: {detail}")
+ # SHADOW/observe tripwire — always printed, strictly NON-BLOCKING: it reports spec coverage +
+ # would-be drift but never touches `failures`, so the exit code is unchanged by anything it finds.
+ shadow_report(root)
+ if failures:
+ print(f"\n{len(failures)} doc-conformance check(s) FAILED:", file=sys.stderr)
+ for f in failures:
+ print(f, file=sys.stderr)
+ print(
+ "\nDocs drifted from their source of truth. Fix the prose (or, better, source it via "
+ "literalinclude), or update the registry if the source of truth legitimately moved.",
+ file=sys.stderr,
+ )
+ return 1
+ print(f"\nOK — all {len(checks)} doc-conformance checks passed.")
+ return 0
+
+
+def self_test(root: Path) -> int:
+ """Negative tests that prove the checker's logic, independent of the live registry. Each block is
+ labelled inline; broadly they cover: (1) token_present boundary/overlap/punctuation handling;
+ (2) dotted-continuation disambiguation; (3) command-token capture, flag-skip, extras, stem-scoping;
+ (4) every ValueMatch entry detects drift when its truth is replaced by a sentinel; (5) the
+ CliSubcommandCheck — clap kebab/name/alias/rename_all parsing, MyST/RST shell-region extraction
+ (incl. nested fences, code directives, python/doctest opacity), nested command-path validation,
+ multi-vx-per-line, binary-name sourcing, and the optional-subcommand-leniency tradeoff. Finally it
+ confirms the live registry passes (a check that can never pass is useless). Keep the labels current
+ when adding cases; the count of cases is intentionally not asserted (cases grow with the registry)."""
+ failures: list[str] = []
+
+ # 1. overlap must NOT match (regression guard for substring matching)
+ if token_present("65527", "padded to 655270 bytes"):
+ failures.append("token_present matched an overlapping value (substring regression)")
+ if token_present("vortex", "pip install vortex-data"):
+ failures.append("token_present matched crate name inside a longer token")
+ # 2. sentence-end punctuation must still match
+ if not token_present("65527", "the bound is 65527."):
+ failures.append("token_present false-negative on sentence-end punctuation")
+ if not token_present("read_vortex", "call read_vortex, then scan"):
+ failures.append("token_present false-negative on comma-delimited token")
+ # 2b. dotted continuation (version / sub-token) must NOT match on EITHER side, but sentence-end
+ # '.' must
+ if token_present("65527", "version 65527.0 of the format"):
+ failures.append("token_present matched a trailing dotted continuation (65527 in 65527.0)")
+ if token_present("65527", "release 1.65527 of the format"):
+ failures.append("token_present matched a leading dotted continuation (65527 in 1.65527)")
+ if not token_present("65527", "the bound is 65527. Next paragraph"):
+ failures.append("token_present false-negative on sentence-end dot before a word")
+
+ # 3. command-scan logic (capture + consistency) — exercised directly, because the
+ # override-sentinel path in (4) below short-circuits on the presence check before reaching
+ # the global command scan, so it would otherwise never cover this code path.
+ captured = command_tokens("pip install ", "`pip install pkg`. then pip install pkg[polars,ray]")
+ if captured != ["pkg", "pkg[polars,ray]"]:
+ failures.append(f"command_tokens captured {captured!r}, expected ['pkg', 'pkg[polars,ray]']")
+ # a dotted (malformed) token is captured WHOLE so it can be rejected; a sentence-end '.' is never
+ # absorbed into the token (it is simply not captured as a command, which presence-checking covers)
+ if command_tokens("pip install ", "run pip install pkg.old here") != ["pkg.old"]:
+ failures.append("command_tokens did not capture a dotted token whole (pkg.old)")
+ if "pkg." in command_tokens("pip install ", "run pip install pkg. Then more"):
+ failures.append("command_tokens absorbed a sentence-end dot")
+ # post-extras junk must not be captured as a clean token (it would otherwise prefix-match)
+ if command_tokens("pip install ", "pip install pkg[extra]wrong here"):
+ failures.append("command_tokens captured a token with post-extras junk")
+ # leading flags are skipped so the package (not the flag) is captured
+ if command_tokens("pip install ", "pip install --upgrade -U vortex-typo here") != ["vortex-typo"]:
+ failures.append("command_tokens did not skip leading flags to reach the package token")
+ if not command_token_ok("pkg", "pkg", allow_extras=False):
+ failures.append("command_token_ok rejected an exact match")
+ if not command_token_ok("pkg", "pkg[polars,ray]", allow_extras=True):
+ failures.append("command_token_ok rejected the [extras] form when extras are allowed")
+ if command_token_ok("pkg", "pkg[bogus]", allow_extras=False):
+ failures.append("command_token_ok accepted [extras] for a non-extras ecosystem (cargo)")
+ if command_token_ok("pkg", "pkg-wrong", allow_extras=True):
+ failures.append("command_token_ok accepted a stale token")
+ if command_token_ok("pkg", "pkg.old", allow_extras=True):
+ failures.append("command_token_ok accepted a dotted (malformed) token")
+ if command_token_ok("pkg", "pkg[extra]wrong", allow_extras=True):
+ failures.append("command_token_ok accepted a token with post-extras junk")
+
+ # 4. each ValueMatch check detects drift in its canonical value
+ for chk in REGISTRY:
+ ok, _ = chk.check(root, override_value="__CONFORMANCE_DRIFT_SENTINEL_DOES_NOT_EXIST__")
+ if ok:
+ failures.append(f"[{chk.id}] a drifted value was NOT caught")
+ else:
+ print(f"SELF-TEST OK {chk.id}: drift would be caught")
+
+ # 4b. source-reader hardening: comment stripping (a commented decoy is never sourced), the
+ # derive/transform mode guard, and Cargo `[[bin]]` key-order tolerance.
+ if _strip_comments("a // line\nb /* block */ c").split() != ["a", "b", "c"]:
+ failures.append("_strip_comments did not remove // and /* */ comments")
+ if _read_const("pub const N: usize = 8;\n// pub const N: usize = 99;\n",
+ r"^\s*pub const N: usize = (\d+)\s*;", "N") != 8:
+ failures.append("comment-decoy: _read_const sourced a commented-out value")
+ try:
+ ValueMatch(id="bad", description="", doc_files=[], derive=lambda r: "x", transform=lambda s: s)
+ failures.append("ValueMatch accepted derive + transform together")
+ except ValueError:
+ pass
+ # Cargo `[[bin]]` name sourcing (tomllib) is robust to a `#`-commented decoy, key reordering, and
+ # a list-valued key before `name` — a raw regex over the text would mishandle these.
+ synth_cargo = ('[package]\nname = "vortex-tui"\n\n[[bin]]\n# name = "old"\npath = "src/main.rs"\n'
+ 'required-features = ["native"]\nname = "vx"\n')
+ if _toml_str_from(synth_cargo, "synth", "bin", 0, "name") != "vx":
+ failures.append("_toml_str_from mishandles [[bin]] name with comments/key-order/list-values")
+ if _toml_str_from(synth_cargo, "synth", "package", "name") != "vortex-tui":
+ failures.append("_toml_str_from mishandles [package] name (table-order independence)")
+ # forbid_regex absence check: a fact whose claim is PRESENT still FAILS when a forbidden (stale)
+ # pattern is also present, so a corrected site cannot mask a stale sibling. Exercised on a real doc.
+ spark = "docs/user-guide/spark.md"
+ if (root / spark).exists():
+ present_forbidden = ValueMatch(id="fb1", description="", doc_files=[spark],
+ derive=lambda r: "Vortex", forbid_regex=r"DataSource V2")
+ ok_fb, detail_fb = present_forbidden.check(root)
+ if ok_fb or "forbidden" not in detail_fb: # must fail, AND specifically for the forbidden reason
+ failures.append(f"forbid_regex failure not attributed to the forbidden pattern ({detail_fb!r})")
+ absent_forbidden = ValueMatch(id="fb2", description="", doc_files=[spark],
+ derive=lambda r: "Vortex", forbid_regex=r"__never_appears_xyzzy__")
+ if not absent_forbidden.check(root)[0]:
+ failures.append("forbid_regex spuriously failed when the forbidden pattern is absent")
+ # the SPECIFIC spark stale-coordinate forbid catches the bare prose/XML forms, ignores suffixed
+ spark_forbid = next(c.forbid_regex for c in REGISTRY if c.id == "spark-maven-artifact")
+ for bare in ("published as `dev.vortex:vortex-spark`.", "vortex-spark"):
+ if not re.search(spark_forbid, bare):
+ failures.append(f"spark forbid_regex missed a bare coordinate form: {bare!r}")
+ for suffixed in ("dev.vortex:vortex-spark_2.13:1.0", "vortex-spark_2.13"):
+ if re.search(spark_forbid, suffixed):
+ failures.append(f"spark forbid_regex false-matched a suffixed coordinate: {suffixed!r}")
+ # DocMembershipCheck logic on SYNTHETIC input (no hard-coded live variant set): a mention
+ # outside the allowed set is flagged; one inside it is not.
+ outside = DocMembershipCheck._outside("use vortex-spark_2.13 then vortex-spark_2.11",
+ r"vortex-spark_(\d+(?:\.\d+)?)", {"2.12", "2.13"})
+ if outside != {"2.11"}:
+ failures.append(f"DocMembershipCheck._outside wrong (got {outside!r}; expected {{'2.11'}})")
+ # and the live derive yields a non-empty published set the live check binds to
+ if not _spark_scala_variants(root):
+ failures.append("_spark_scala_variants derived an empty set from settings.gradle.kts")
+
+ # 4b-ii. PR-2.3 forbid/scoped-claim checks — synthetic-string tests over REGISTRY forbid regexes;
+ # NOT under the Spark-doc guard (they don't read spark.md), so Spark-only churn can't skip them.
+ buf_forbid = next(c.forbid_regex for c in REGISTRY if c.id == "buffered-block-size")
+ for stale in ("localize up to 1 MB", "the 8k row zones and 2MB chunks balance",
+ "Chunked Layout to partition the column into 2MB of uncompressed data"):
+ if not re.search(buf_forbid, stale):
+ failures.append(f"buffered forbid_regex missed a stale form: {stale!r}")
+ if re.search(buf_forbid, "2 MB of buffered chunk locality"):
+ failures.append("buffered forbid_regex false-matched the corrected buffered-locality wording")
+ io_forbid = next(c.forbid_regex for c in REGISTRY if c.id == "io-coalesce-file")
+ for stale in ("Local files use 8 KB for both", "| Local file | {{x}} | 8 KB | 8 KB |"):
+ if not re.search(io_forbid, stale):
+ failures.append(f"io-coalesce forbid_regex missed a stale form: {stale!r}")
+ for ok in ("In-memory buffers use an 8 KB distance", "| Local file | {{x}} | 1 MB | 4 MB |"):
+ if re.search(io_forbid, ok):
+ failures.append(f"io-coalesce forbid_regex false-matched a correct line: {ok!r}")
+ # scoped claims: the derived value must appear IN ITS CONTEXT, not just anywhere in the file.
+ if token_present("1 MB distance and 4 MB max size", "local: 5 MB max size. (4 MB appears here)"):
+ failures.append("io-coalesce claim matched out of context (unscoped bare token)")
+ if not token_present("localize up to 2 MB", "Buffered Layout to localize up to 2 MB of chunks"):
+ failures.append("buffered scoped claim false-negative on the correct sentence")
+ if token_present("localize up to 2 MB", "Buffered Layout to localize up to 3 MB; a 2 MB zone"):
+ failures.append("buffered claim matched a bare 2 MB elsewhere, not the scoped phrase")
+ # scanning-api-traits forbid is case-insensitive for Sink AND catches a backticked `Source` trait
+ # (the real trait is `DataSource`), but not the corrected `DataSource` mention.
+ scan_forbid = next(c.forbid_regex for c in REGISTRY if c.id == "scanning-api-traits")
+ for stale in ("An equivalent `Sink` trait exists", "an equivalent sink interface exists",
+ "The core `Source` trait and scan pipeline"):
+ if not re.search(scan_forbid, stale):
+ failures.append(f"scanning-api-traits forbid missed a stale form: {stale!r}")
+ if re.search(scan_forbid, "The core `DataSource` trait and scan pipeline"):
+ failures.append("scanning-api-traits forbid false-matched the corrected `DataSource` wording")
+ # cpp-binding-cxx forbid blocks the stale current-state "wrapper around the C FFI" framing, but NOT
+ # the legitimate future-plan wording ("wrapping the C API"); and _cxx_dep derives `cxx` live.
+ cxx_forbid = next(c.forbid_regex for c in REGISTRY if c.id == "cpp-binding-cxx")
+ for stale in ("a C++ wrapper around the C FFI", "wrapper around the Vortex C FFI",
+ "the C++ API wraps the C FFI", "a thin wrapper for the C FFI"):
+ if not re.search(cxx_forbid, stale):
+ failures.append(f"cpp-binding-cxx forbid missed a stale form: {stale!r}")
+ if re.search(cxx_forbid, "The plan is to migrate to wrapping the C API directly"):
+ failures.append("cpp-binding-cxx forbid false-matched the legitimate future-plan (C API) wording")
+ # the "wrapper ... C FFI" forbid is sentence-scoped but line-wrap-tolerant — a stale claim split across
+ # lines is still caught; a later-sentence "C FFI" mention is not.
+ if not re.search(cxx_forbid, "a C++ wrapper around the\nVortex C FFI"):
+ failures.append("cpp-binding-cxx forbid missed a 'wrapper ... C FFI' claim split across lines")
+ if re.search(cxx_forbid, "It wraps the Rust core. A separate C FFI is also offered."):
+ failures.append("cpp-binding-cxx forbid false-matched a C FFI mention in a LATER sentence")
+ if _cxx_dep(root) != "cxx":
+ failures.append("_cxx_dep did not confirm the live cxx dependency + bridge")
+ # the bridge check strips comments first, so a commented-out `#[cxx::bridge]` decoy is NOT live
+ if re.search(r"#\[cxx::bridge", _strip_rust_comments("// #[cxx::bridge] mod ffi { }\nfn x() {}")):
+ failures.append("_cxx_dep bridge search would accept a line-commented `#[cxx::bridge]` decoy")
+ # the bridge check also anchors to ITEM position, so a string literal containing the attribute can't
+ # satisfy it after the real bridge is removed (the `"` breaks the `^[ \t]*#\[` anchor).
+ if re.search(r"^[ \t]*#\[cxx::bridge", 'const DOC: &str = "#[cxx::bridge]";\nfn x() {}', re.MULTILINE):
+ failures.append("_cxx_dep item-position anchor accepted a string-literal `#[cxx::bridge]` decoy")
+ if not re.search(r"^[ \t]*#\[cxx::bridge", " #[cxx::bridge]\n mod ffi {}", re.MULTILINE):
+ failures.append("_cxx_dep item-position anchor missed a real indented `#[cxx::bridge]`")
+ # the duckdb replacement-scan lock is scoped to the `initialize_extension_from_raw` entrypoint body — a
+ # call in a helper/test fn OUTSIDE it does NOT satisfy the lock; a call INSIDE it does.
+ dd_outside = _strip_rust_comments('fn helper() { db.register_vortex_scan_replacement(); }\n'
+ 'pub unsafe fn initialize_extension_from_raw(db: X) {\n init_tracing();\n}')
+ dd_im = re.search(r"fn initialize_extension_from_raw\([^)]*\)\s*\{(.*?)\n\}", dd_outside, re.DOTALL)
+ if dd_im and re.search(r"\.register_vortex_scan_replacement\s*\(\s*\)", dd_im.group(1)):
+ failures.append("duckdb replacement-scan lock satisfied by a call OUTSIDE initialize_extension_from_raw")
+ dd_inside = _strip_rust_comments('pub unsafe fn initialize_extension_from_raw(db: X) {\n'
+ ' db.register_vortex_scan_replacement();\n}')
+ dd_im2 = re.search(r"fn initialize_extension_from_raw\([^)]*\)\s*\{(.*?)\n\}", dd_inside, re.DOTALL)
+ if not (dd_im2 and re.search(r"\.register_vortex_scan_replacement\s*\(\s*\)", dd_im2.group(1))):
+ failures.append("duckdb replacement-scan lock missed a real call inside initialize_extension_from_raw")
+ # jni-module-name: live derive returns the real module, and the forbid catches the renamed-away name.
+ if _jni_module_name(root) != "vortex-jni":
+ failures.append(f"_jni_module_name wrong (got {_jni_module_name(root)!r})")
+ jni_forbid = next(c.forbid_regex for c in REGISTRY if c.id == "jni-module-name")
+ if not re.search(jni_forbid, "the vortex-java JAR contains the JNI code"):
+ failures.append("jni-module-name forbid missed the stale `vortex-java` name")
+ if re.search(jni_forbid, "the vortex-jni JAR contains the JNI code"):
+ failures.append("jni-module-name forbid false-matched the corrected `vortex-jni` name")
+ # the scan-api / variant-dtype roadmap locks derive present-state facts from code; live values + a
+ # synthetic DType-without-Variant negative for the pure presence helper.
+ if _scan_crate_name(root) != "vortex-scan" or _variant_dtype_present(root) != "Variant":
+ failures.append("scan-api/variant-dtype derive did not confirm the live code facts")
+ # convert chunk-size lock: derives BATCH_SIZE; the scoped "-row" claim + the row-group-boundaries
+ # forbid catch the stale quickstart wording.
+ if _convert_batch_size(root) != "8192":
+ failures.append(f"_convert_batch_size wrong (got {_convert_batch_size(root)!r})")
+ conv_forbid = next(c.forbid_regex for c in REGISTRY if c.id == "cli-convert-chunk-size")
+ for stale in ("chunking on Parquet row-group boundaries", "Chunking occurs on Parquet RowGroup boundaries"):
+ if not re.search(conv_forbid, stale):
+ failures.append(f"cli-convert-chunk-size forbid missed a stale form: {stale!r}")
+ if re.search(conv_forbid, "chunking into 8192-row batches"):
+ failures.append("cli-convert-chunk-size forbid false-matched the corrected wording")
+ # the claim is scoped to "-row batches" — a bare unrelated "8192-row" mention does not satisfy it
+ if token_present("8192-row batches", "the file has 8192-row groups and other 8192-row data"):
+ failures.append("cli-convert-chunk-size claim matched a bare 8192-row mention, not the scoped phrase")
+ # the scan lock requires the `DataSource` trait specifically — `DataSourceOpener`/`-Scan` (word
+ # boundary) must NOT satisfy it, but `pub trait DataSource:` must.
+ if re.search(r"pub trait DataSource\b", "pub trait DataSourceOpener: 'static {}"):
+ failures.append("scan-api `DataSource` trait check satisfied by DataSourceOpener (missing \\b)")
+ if not re.search(r"pub trait DataSource\b", "pub trait DataSource: 'static + Send {}"):
+ failures.append("scan-api `DataSource` trait check missed a real `pub trait DataSource`")
+ if _dtype_has_variant("Null,\n Bool(Nullability),\n Primitive(PType, Nullability),"):
+ failures.append("_dtype_has_variant false-positive on an enum body without a Variant variant")
+ if not _dtype_has_variant("Extension(ExtDTypeRef),\n Variant(Nullability),"):
+ failures.append("_dtype_has_variant missed a present Variant variant")
+ # _variant_dtype_present strips NESTED Rust comments, so a Variant hidden in one isn't "present"
+ if _dtype_has_variant("Null,\n /* a /* Variant(Nullability), */ b */ Bool,"):
+ # (sanity: the nested-comment stripping happens before _dtype_has_variant in _variant_dtype_present;
+ # here we assert the stripper removes it)
+ pass
+ if re.search(r"^\s*Variant\b",
+ _strip_rust_comments("Null,\n /* x /* Variant(Nullability), */ y */ Bool,"), re.MULTILINE):
+ failures.append("_variant_dtype_present would detect a Variant hidden in a nested comment")
+ # CONSTANT/FUNCTION locks now route Rust sources through _strip_rust_comments, so a decoy constructor
+ # hidden in a NESTED `/* /* */ */` block comment can't inflate the "exactly one" match count past the
+ # real one (the byte-pair / block-size / stat-name locks all rely on this).
+ nested_decoy = ("fn in_memory() -> Self { Self::new(8192, 8192) }\n"
+ " /* outer /* fn in_memory() -> Self { Self::new(1, 2) } */ inner */")
+ if len(re.findall(r"fn in_memory\(\)\s*->\s*Self\s*\{\s*Self::new\(",
+ _strip_rust_comments(nested_decoy))) != 1:
+ failures.append("_strip_rust_comments left a nested-commented constructor decoy for a byte-pair lock")
+ # canonical-union-exception: live facts hold, and the guard trips iff the Canonical enum gains a Union
+ # variant / UnionArray payload (forcing the "except Union" prose to update).
+ if _union_is_sole_noncanonical(root) != "Union":
+ failures.append("_union_is_sole_noncanonical did not confirm the live DType/Canonical facts")
+ for canonicalized in ("Null(NullArray),\n Union(UnionArray),", "Bool(BoolArray),\n Union(SomeArray),"):
+ if not (re.search(r"^\s*Union\b", canonicalized, re.MULTILINE) or "UnionArray" in canonicalized):
+ failures.append("canonical-union-exception guard would miss a canonicalized Union")
+ if re.search(r"^\s*Union\b", "Null(NullArray),\n Bool(BoolArray),", re.MULTILINE):
+ failures.append("canonical-union-exception guard false-tripped on a Canonical enum without Union")
+ # scoped roadmap claims reject FUTURE-only wording even though it contains the derived token.
+ if token_present("Scan API (the `vortex-scan` crate) already provides",
+ "The Scan API (the `vortex-scan` crate) will provide pluggable sources."):
+ failures.append("scan-api-present claim matched a future-only sentence")
+ if token_present("`Variant` DType already exists",
+ "The `Variant` DType is planned for a future release."):
+ failures.append("variant-dtype-present claim matched a planned-only sentence")
+ # cpp-not-wrapper-nav forbid catches the stale "C++ wrapper" framing and the "C/C++ (FFI)" label.
+ nav_forbid = next(c.forbid_regex for c in REGISTRY if c.id == "cpp-not-wrapper-nav")
+ for stale in ("embed it through the C FFI, C++ wrapper, or", "Java (JNI) and C/C++ (FFI) bindings",
+ "C++ (FFI) bindings", "C and C++ FFI bindings"):
+ if not re.search(nav_forbid, stale):
+ failures.append(f"cpp-not-wrapper-nav forbid missed a stale form: {stale!r}")
+ for ok in ("the C++ binding (a direct cxx Rust bridge)", "Java (JNI), C (FFI), and C++ (cxx)"):
+ if re.search(nav_forbid, ok): # corrected wording (incl. C++ (cxx)) must NOT match
+ failures.append(f"cpp-not-wrapper-nav forbid false-matched corrected wording: {ok!r}")
+ # the "foundation for ... C++" forbid is SENTENCE-scoped but tolerates line WRAPS: a stale present-tense
+ # claim broken across lines is still caught, while a legitimate later-SENTENCE C++ mention is not.
+ if not re.search(nav_forbid, "intended foundation for\nother language bindings (including C++)"):
+ failures.append("cpp-not-wrapper-nav forbid missed a 'foundation for ... C++' claim split across lines")
+ if re.search(nav_forbid, "the intended foundation for other language bindings. The C++ binding uses cxx."):
+ failures.append("cpp-not-wrapper-nav forbid false-matched a foundation claim with C++ in a LATER sentence")
+ # wheel target extraction is ORDER-INSENSITIVE within the brace table (a reordered field can't silently
+ # drop a platform); the JNI list form (no braces) and `${{ matrix.target.target }}` are NOT captured.
+ wheel_re = r"\{[^{}]*\btarget:\s*([\w-]+)[^{}]*\}"
+ if "x86_64-unknown-linux-gnu" not in set(re.findall(
+ wheel_re, '- { target: x86_64-unknown-linux-gnu, os: ubuntu, runs-on: "ubuntu-latest" }')):
+ failures.append("wheel target extraction missed a target that is not the last field in the brace table")
+ if set(re.findall(wheel_re, "- target: aarch64-unknown-linux-gnu\n name: ${{ matrix.target.target }}.zip")):
+ failures.append("wheel target extraction wrongly captured a JNI list-form / GitHub-expression target")
+ # wheel extraction is JOB-SCOPED to `prepare-python`: an inline `{...target:...}` in an UNRELATED job
+ # is NOT captured (only the wheel job's matrix triples reach _WHEEL_LABEL).
+ synthetic_yml = (' prepare-python:\n strategy:\n matrix:\n target:\n'
+ ' - { os: ubuntu, target: x86_64-unknown-linux-gnu }\n'
+ ' other-job:\n steps:\n - run: build { target: not-a-wheel-triple }\n')
+ job_m = re.search(r"^ prepare-python:\n(.*?)(?=^ \w[\w-]*:\n|\Z)", synthetic_yml, re.DOTALL | re.MULTILINE)
+ scoped = set(re.findall(wheel_re, job_m.group(1))) if job_m else set()
+ if scoped != {"x86_64-unknown-linux-gnu"}:
+ failures.append(f"wheel job-scoping wrong (got {sorted(scoped)}; expected only the prepare-python triple)")
+ # io-reader locks: the named structs `impl VortexReadAt` live; a file without the impl fails loud.
+ if _io_reader_impl(root, "FileReadAt", "vortex-io/src/std_file/read_at.rs") != "FileReadAt":
+ failures.append("_io_reader_impl did not confirm FileReadAt impl VortexReadAt")
+ if _io_reader_impl(root, "ObjectStoreReadAt", "vortex-io/src/object_store/read_at.rs") != "ObjectStoreReadAt":
+ failures.append("_io_reader_impl did not confirm ObjectStoreReadAt impl VortexReadAt")
+ if re.search(r"\bimpl VortexReadAt for FileReadAt\b", "impl VortexReadAt for SomethingElse {}"):
+ failures.append("io-reader impl check satisfied by an unrelated VortexReadAt impl")
+ # spark-version locks: the live scala->spark when-arms map 2.13->Spark 4.x and 2.12->Spark 3.x.
+ if _spark_major_for_scala(root, "2.13") != "Spark 4.x" or _spark_major_for_scala(root, "2.12") != "Spark 3.x":
+ failures.append("_spark_major_for_scala did not confirm the live scala->spark mapping (2.13=4.x, 2.12=3.x)")
+ # _strip_rust_comments removes NESTED block comments, so a marker hidden in one is not "live"
+ if re.search(r"#\[cxx::bridge", _strip_rust_comments("/* a /* b */ #[cxx::bridge] */ fn x(){}")):
+ failures.append("_strip_rust_comments did not strip a nested-block-comment #[cxx::bridge] decoy")
+ if "#[cxx::bridge]" not in _strip_rust_comments("#[cxx::bridge]\nmod ffi {}"):
+ failures.append("_strip_rust_comments wrongly removed a live #[cxx::bridge]")
+ # no-trino-integration-claim forbid blocks in-progress framing AND the stale "Spark and Trino"
+ # current-pairing, but not factual Trino-project context or future-connector wording.
+ trino_forbid = next(c.forbid_regex for c in REGISTRY if c.id == "no-trino-integration-claim")
+ for stale in ("Trino support is in progress", "Spark (with Trino in progress)",
+ "the integration point for Spark and Trino connectors"):
+ if not re.search(trino_forbid, stale):
+ failures.append(f"no-trino-integration-claim forbid missed a stale form: {stale!r}")
+ # Factual project context and explicit future/planned-connector wording are allowed (neither claims a
+ # CURRENT Trino integration). NB (documented residual): the live "tier suitable for query engine
+ # integrations (e.g. ... Trino)" line is an ILLUSTRATIVE example, not a current-integration claim — and a
+ # hypothetical FALSE "Vortex integrates with ... Trino" current-tense list is NOT regex-distinguishable
+ # from that legit example without false-positiving the live doc. So the self-test deliberately does NOT
+ # assert "a list containing Trino is allowed" (that would sanction the ambiguous case); the live registry
+ # check is what confirms the real doc passes. Catching arbitrary false current-integration phrasings is
+ # beyond the scope of this registry check, not a lock gap.
+ for ok in ("Trino already supports JDK 22", "any future JVM connector (such as Trino)"):
+ if re.search(trino_forbid, ok): # factual / future-connector context is allowed
+ failures.append(f"no-trino-integration-claim forbid false-matched allowed context: {ok!r}")
+ # the forbid is sentence-scoped but line-wrap-tolerant — a stale claim split across lines is caught.
+ for wrapped in ("Trino support is in\nprogress", "the integration point for Spark\nand Trino"):
+ if not re.search(trino_forbid, wrapped):
+ failures.append(f"no-trino-integration-claim forbid missed a line-wrapped stale form: {wrapped!r}")
+
+ # 4c. Python API check: python-region scoping ignores prose/URLs/Maven/Spark-options, and the
+ # membership logic flags a fabricated `vortex.` while accepting a real one.
+ py = _python_regions("`bench.vortex.dev` and `${vortex.write.batch.size}`\n"
+ "```python\nimport vortex\nvortex.open('f')\n```\nprose vortex.frobnicate\n"
+ "```{doctest} pycon\n>>> import vortex as vx\n>>> vx.array([1])\n```")
+ if "vortex.open" not in py or "vortex.dev" in py or "vortex.frobnicate" in py:
+ failures.append("_python_regions wrong: missing python block, or leaked prose/URL/option text")
+ if "vx.array" not in py: # MyST {doctest} fences are the dominant python-example form in these docs
+ failures.append("_python_regions did not scan a MyST ```{doctest} fence")
+ # backtick-length-aware nesting: a 3-backtick python fence inside a 4-backtick `{tab}` container is
+ # captured (the inner 3-backtick close must NOT be mis-paired against the 4-backtick container).
+ nested = _python_regions("````{tab} Python\n```python\nvx.nested_call(1)\n```\n````\n"
+ "````{tab} Other\n```bash\nrm vortex.frob\n```\n````")
+ if "vx.nested_call" not in nested:
+ failures.append("_python_regions missed a python fence nested in a 4-backtick `{tab}` container")
+ if "vortex.frob" in nested:
+ failures.append("_python_regions leaked a non-python (bash) fence nested in a container")
+ # OPACITY: an explicit non-python fence containing a LITERAL ```python snippet (a shell heredoc /
+ # markdown-generating example) must NOT be scanned as real docs python — i.e. not descended into.
+ opaque = _python_regions("````bash\ncat <>>` bodies are still API-checked.
+ eval_rst = _python_regions("```{eval-rst}\n.. code-block:: python\n\n vx.from_eval_rst(1)\n\n"
+ ">>> vx.eval_rst_prompt(2)\n```")
+ if "vx.from_eval_rst" not in eval_rst or "vx.eval_rst_prompt" not in eval_rst:
+ failures.append("_python_regions did not scan an {eval-rst} container's RST python bodies")
+ # pycon/doctest OUTPUT is not API usage: a `>>>` session's input is checked, but interpreter output
+ # (e.g. a repr like ``) is dropped so it can't false-fail valid docs.
+ pycon = _python_regions("```{doctest} pycon\n>>> import vortex as vx\n>>> vx.array([1])\n"
+ "\n```")
+ if "vx.array" not in pycon:
+ failures.append("_strip_pycon_output dropped a `>>>` INPUT line")
+ if "not_a_real_api" in pycon:
+ failures.append("_strip_pycon_output scanned pycon OUTPUT as code (false-positive risk)")
+ # a bare ```{code-cell}` (no language arg) is python in MyST and MUST be scanned
+ cell = _python_regions("```{code-cell}\nimport vortex\nvortex.from_bare_code_cell(3)\n```")
+ if "vortex.from_bare_code_cell" not in cell:
+ failures.append("_python_regions did not scan a bare ```{code-cell} (python kernel) block")
+ # the bare-`>>>` RST-doctest pass is PROSE-scoped: a prompt inside an opaque ```text/```bash fence
+ # is NOT scanned, but a prompt in real RST prose IS.
+ prompts = _python_regions("```text\n>>> vortex.in_a_text_fence(1)\n```\n\n>>> vortex.in_real_prose(2)")
+ if "vortex.in_a_text_fence" in prompts:
+ failures.append("bare-`>>>` pass scanned a prompt inside an opaque ```text fence (false positive)")
+ if "vortex.in_real_prose" not in prompts:
+ failures.append("bare-`>>>` pass missed a prompt in RST prose")
+ bad_api = DocMembershipCheck._outside("vortex.open(x)\nvortex.frobnicate(y)\nvx.array(z)",
+ r"(? set: # noqa: E306 (test-local helper)
+ return {mm.group(1) for mm in callpat.finditer(_python_regions(doc)) if mm.group(1) in subs}
+ if _mods_called("```python\nvortex.dataset('f')\n```") != {"dataset"}:
+ failures.append("module-call check did not flag `vortex.dataset(...)` (module called as function)")
+ for ok in ("vortex.open('f')", "vortex.array([1])", "vortex.io.write(t,'f')", "vx.file.open('f')"):
+ if _mods_called(f"```python\n{ok}\n```"):
+ failures.append(f"module-call check false-flagged a valid call: {ok!r}")
+ # spark/duckdb content forbids catch the stale future-claims (and their derives confirm live code).
+ sp_forbid = next(c.forbid_regex for c in REGISTRY if c.id == "spark-filter-pushdown")
+ if not re.search(sp_forbid, "not yet connected to Spark's SupportsPushDownFilters; planned future work"):
+ failures.append("spark-filter-pushdown forbid missed the stale 'not yet connected' claim")
+ dd_forbid = next(c.forbid_regex for c in REGISTRY if c.id == "duckdb-direct-path")
+ if not re.search(dd_forbid, "syntax is coming in an upcoming DuckDB release"):
+ failures.append("duckdb-direct-path forbid missed the stale 'coming in an upcoming release' claim")
+ # canonical encodings sourced from the Rust Canonical enum: VarBinView is canonical (not VarBin),
+ # FixedSizeList is canonical, ListArray is NOT (its payload ListViewArray is not in the Python API),
+ # and a Decimal/Variant with no Python class is excluded.
+ canon = _canonical_python_classes(root)
+ if "VarBinViewArray" not in canon or "FixedSizeListArray" not in canon or "VarBinArray" in canon:
+ failures.append(f"_canonical_python_classes wrong (got {sorted(canon)})")
+ if "ListArray" in canon: # Canonical::List wraps ListViewArray, not ListArray
+ failures.append(f"_canonical_python_classes wrongly included ListArray (got {sorted(canon)})")
+ # CanonicalSectionCheck's arrays.rst section parser on SYNTHETIC rst: it bounds the "Canonical
+ # Encodings" section at the next heading (a class in a later section is NOT counted), and yields the
+ # empty set when the heading is absent.
+ rst = ("Canonical Encodings\n-------------------\n\n.. autoclass:: vortex.NullArray\n :members:\n"
+ "\n.. autoclass:: vortex.BoolArray\n\nUtility Encodings\n-----------------\n\n"
+ ".. autoclass:: vortex.VarBinArray\n")
+ if _listed_canonical_classes(rst) != {"NullArray", "BoolArray"}:
+ failures.append(f"_listed_canonical_classes mis-bounded the section (got {_listed_canonical_classes(rst)})")
+ if _listed_canonical_classes("Some Other Heading\n------\n\n.. autoclass:: vortex.NullArray\n") != set():
+ failures.append("_listed_canonical_classes returned non-empty when the Canonical heading is absent")
+ # byte-size derivation (io coalesce + buffered block size)
+ if (_eval_byte_expr("1 << 20"), _eval_byte_expr("8 * 1024"), _eval_byte_expr("42")) != (1 << 20, 8192, 42):
+ failures.append("_eval_byte_expr mis-evaluated a shift / mul / plain-int literal")
+ if _bytes_to_human(4 << 20) != "4 MB" or _bytes_to_human(8 * 1024) != "8 KB":
+ failures.append("_bytes_to_human wrong unit formatting")
+ try:
+ _eval_byte_expr("foo()")
+ failures.append("_eval_byte_expr did not fail loud on a complex expression")
+ except LookupError:
+ pass
+ if _coalesce_bytes(root, "file") != (1 << 20, 4 << 20):
+ failures.append(f"_coalesce_bytes(file) wrong (got {_coalesce_bytes(root, 'file')})")
+ if _derive_buffered_block_size(root) != "2 MB":
+ failures.append(f"_derive_buffered_block_size wrong (got {_derive_buffered_block_size(root)!r})")
+ # Stat names + the arrays.md Statistics-section parser (set-equality lock)
+ sn = _stat_names(root)
+ if "true_count" in sn or "run_count" in sn or "null_count" not in sn or len(sn) < 5:
+ failures.append(f"_stat_names wrong (got {sorted(sn)})")
+ stats_md = "## Statistics\n\n* `is_constant`: holds `true` only.\n* `sum`: y\n\n## Execution\n* `nope`: z\n"
+ if _listed_stat_names(stats_md) != (["is_constant", "sum"], 2): # bounded at next `## `; leading span
+ failures.append(f"_listed_stat_names mis-parsed/over-ran the section (got {_listed_stat_names(stats_md)})")
+ if _listed_stat_names("## Other\n* `x`: y\n") != ([], 0):
+ failures.append("_listed_stat_names returned non-empty when the Statistics heading is absent")
+ if _listed_stat_names("## Statistics\r\n\r\n* `sum`: y\r\n\r\n## Execution\r\n") != (["sum"], 1):
+ failures.append("_listed_stat_names did not tolerate CRLF line endings")
+ # a bullet WITHOUT a leading code-span is counted (n_bullets) but not named, so the check can fail it
+ malformed = _listed_stat_names("## Statistics\n\n* `sum`: ok\n* true_count: no backticks\n")
+ if malformed != (["sum"], 2):
+ failures.append(f"_listed_stat_names did not surface a code-span-less bullet (got {malformed})")
+ # duplicates are preserved in the list so StatsListCheck can detect them
+ dup = _listed_stat_names("## Statistics\n\n* `sum`: a\n* `sum`: b\n")
+ if dup != (["sum", "sum"], 2):
+ failures.append(f"_listed_stat_names collapsed a duplicate (got {dup})")
+ # Stat::name() arm parsing: literal arms yield names; a non-literal (const-return) arm FAILS LOUD
+ # rather than silently shrinking the expected set.
+ if _parse_stat_name_arms('Self::A => "a",\n Self::B => "b",') != {"a", "b"}:
+ failures.append("_parse_stat_name_arms wrong on literal arms")
+ try:
+ _parse_stat_name_arms('Self::A => "a",\n Self::Foo => FOO_CONST,')
+ failures.append("_parse_stat_name_arms did not fail loud on a non-literal (const-return) arm")
+ except LookupError:
+ pass
+ try:
+ _parse_stat_name_arms('Self::A => "x",\n Self::B => "x",') # two variants, same name string
+ failures.append("_parse_stat_name_arms did not fail loud on duplicate name literals")
+ except LookupError:
+ pass
+ for bad_arm in ('Self::A | Self::B => "x",', '_ => "x",'): # or-pattern / wildcard collapse variants
+ try:
+ _parse_stat_name_arms(f'Self::Z => "z",\n {bad_arm}')
+ failures.append(f"_parse_stat_name_arms did not fail loud on arm pattern {bad_arm!r}")
+ except LookupError:
+ pass
+ # _stat_names_from cross-checks name count against enum variant count, so even a MULTI-LINE
+ # or-pattern (which leaves only `Self::B => ...` on the `=>` line) is caught by the count mismatch.
+ if _stat_names_from('Self::A => "a",\n Self::B => "b",', "A = 0,\n B = 1,") != {"a", "b"}:
+ failures.append("_stat_names_from wrong when name count matches variant count")
+ try: # multi-line or-pattern: 1 name produced, but 2 variants -> fail loud
+ _stat_names_from('Self::A |\n Self::B => "ab",', "A = 0,\n B = 1,")
+ failures.append("_stat_names_from did not fail loud on a collapsed (multi-line or-pattern) arm")
+ except LookupError:
+ pass
+ if _stat_enum_variants("IsConstant = 0,\n IsSorted = 1,\n Plain,") != {"IsConstant", "IsSorted", "Plain"}:
+ failures.append("_stat_enum_variants mis-parsed discriminant / bare variant forms")
+ # payload-vs-variant-name distinction on SYNTHETIC input: List(ListViewArray) must NOT yield
+ # ListArray even when ListArray IS in the public API — a variant-name mapping would wrongly include
+ # it. (This is the exact regression the payload-based derivation prevents.)
+ synth_api = {"NullArray", "ListArray"} # ListArray exposed; ListViewArray/DecimalArray are NOT
+ payloads = _canonical_payload_classes("Null(NullArray),\nList(ListViewArray),\nDecimal(DecimalArray),",
+ synth_api)
+ if payloads != {"NullArray"}:
+ failures.append(f"_canonical_payload_classes leaked a variant-name mapping (got {sorted(payloads)})")
+ if _canonical_payload_classes("Foo(crate::path::BarArray),", {"BarArray"}) != {"BarArray"}:
+ failures.append("_canonical_payload_classes did not take the last path segment of a payload")
+ try: # a struct-style variant must FAIL LOUD, not be silently skipped
+ _canonical_payload_classes("Weird { field: u8 },", {"Weird"})
+ failures.append("_canonical_payload_classes did not fail loud on an unparseable variant")
+ except LookupError:
+ pass
+ # Python min-version derivation anchors to the `>=` lower bound regardless of clause order, and
+ # fails loud when no lower bound is present.
+ if _parse_python_min_version(">=3.11,<4.0") != "Python 3.11":
+ failures.append("_parse_python_min_version did not anchor to the >= lower bound")
+ if _parse_python_min_version("<4,>=3.11") != "Python 3.11": # reordered clauses
+ failures.append("_parse_python_min_version is order-sensitive (picked the upper bound)")
+ if _parse_python_min_version(">=3.11.4,<4.0") != "Python 3.11.4": # patch level must be kept
+ failures.append("_parse_python_min_version truncated a patch-level lower bound")
+ if _parse_python_min_version(">=3.10,>=3.11,<4") != "Python 3.11": # intersecting lower bounds
+ failures.append("_parse_python_min_version did not pick the highest of multiple >= bounds")
+ # fail loud on: no lower bound / pre-release / wildcard / exclusion / compatible-release / exact /
+ # exclusive-lower — any of which could silently understate the effective minimum.
+ for bad in ("<4.0", ">=3.11.0rc1,<4", ">=3.11.*", ">=3.11,!=3.11.*", ">=3.11,!=3.11.0",
+ "~=3.11", "==3.11", ">3.10"):
+ try:
+ _parse_python_min_version(bad)
+ failures.append(f"_parse_python_min_version did not fail loud on {bad!r}")
+ except LookupError:
+ pass
+
+ # 5. CLI-subcommand check logic, exercised end-to-end on synthetic input.
+ # kebab-casing incl. acronyms (clap/heck)
+ for variant, want in [("Tree", "tree"), ("FooBar", "foo-bar"), ("SQLQuery", "sql-query"),
+ ("HTTPServer", "http-server")]:
+ if CliSubcommandCheck._to_kebab(variant) != want:
+ failures.append(f"_to_kebab({variant!r}) != {want!r}")
+ # depth-aware enum parse: a wrapped struct-field type is NOT a variant
+ variants = CliSubcommandCheck._enum_variants(
+ "enum Commands {\n Tree(Args),\n Browse {\n file: PathBuf,\n },\n}", "Commands")
+ if variants != ["Tree", "Browse"]:
+ failures.append(f"_enum_variants mis-parsed (got {variants!r}; field type leaked?)")
+ # clap naming attributes: `name=` overrides the kebab default, `alias`/`visible_alias` add names,
+ # and an unmodeled `rename_all` must FAIL LOUD rather than silently trusting kebab-casing.
+ names = CliSubcommandCheck._subcommand_names(
+ 'enum Commands {\n Tree(A),\n #[command(name = "ls", visible_alias = "dir")]\n'
+ " Browse { file: PathBuf },\n}", "Commands")
+ if names != {"tree", "ls", "dir"}:
+ failures.append(f"_subcommand_names ignored name=/alias attrs (got {names!r})")
+ try:
+ CliSubcommandCheck._subcommand_names(
+ 'enum Commands {\n #[command(rename_all = "snake_case")]\n FooBar(A),\n}', "Commands")
+ failures.append("_subcommand_names did not fail loud on an unmodeled rename_all")
+ except LookupError:
+ pass
+ # multi-line (rustfmt-formatted) clap attribute: the `name=` override spread across lines is honored
+ ml_names = CliSubcommandCheck._subcommand_names(
+ 'enum Commands {\n Tree(A),\n #[command(\n name = "ls",\n'
+ ' visible_alias = "dir",\n )]\n Browse { file: PathBuf },\n}', "Commands")
+ if ml_names != {"tree", "ls", "dir"}:
+ failures.append(f"_subcommand_names ignored a multi-line clap attribute (got {ml_names!r})")
+ # enum-level rename_all hidden behind a long (>200 char) doc comment, even WITH a trailing comment
+ # on the attribute, must STILL fail loud (backward-walk is not clipped, and a `// note` does not
+ # terminate the header block).
+ long_doc = "/// " + ("x " * 120) + "\n"
+ try:
+ CliSubcommandCheck._subcommand_names(
+ long_doc + '#[command(rename_all = "snake_case")] // tweak casing\n'
+ "enum Commands {\n FooBar(A),\n}", "Commands")
+ failures.append("_subcommand_names missed rename_all behind a long doc comment / trailing note")
+ except LookupError:
+ pass
+ # string-literal-aware clap meta: a `name = "fake"` INSIDE an `about` raw string must NOT spoof the
+ # real `name = "ls"` override.
+ spoof = CliSubcommandCheck._subcommand_names(
+ 'enum Commands {\n #[command(about = r#"run with name = "fake""#, name = "ls")]\n'
+ " Browse { file: PathBuf },\n}", "Commands")
+ if spoof != {"ls"}:
+ failures.append(f"_clap_meta let a string literal spoof the name (got {spoof!r})")
+ # MyST `{code-block} bash` directive is OPAQUE shell (scanned); python/doctest directives are not.
+ cb = CliSubcommandCheck._shell_regions("```{code-block} bash\nvx frobnicate f\n```")
+ if "vx frobnicate" not in cb:
+ failures.append("_shell_regions did not scan a {code-block} bash directive body")
+ cb_py = CliSubcommandCheck._shell_regions("```{code-block} python\nvx.open('f')\n```")
+ if "vx.open" in cb_py:
+ failures.append("_shell_regions scanned a {code-block} python directive (alias false-positive)")
+ doctest = CliSubcommandCheck._shell_regions("```{doctest} pycon\n>>> import vortex as vx\n`vx frob`\n```")
+ if "vx frob" in doctest:
+ failures.append("_shell_regions leaked a {doctest} directive body into the CLI scan")
+ # MyST nested fence: a fabricated `vx` inside a ```` ```bash ```` block nested in a 4-backtick
+ # `{tab}` container MUST be scanned (the docs use this exact shape), and a real shell block AFTER
+ # the closed container must still be scanned (the directive-close must not swallow the rest).
+ nested = CliSubcommandCheck._shell_regions(
+ "````{tab} demo\n```bash\nvx frobnicate f\n```\n````\n\n```bash\nvx convert g\n```")
+ if "vx frobnicate" not in nested:
+ failures.append("_shell_regions missed a shell block nested in a 4-backtick {tab} container")
+ if "vx convert" not in nested:
+ failures.append("_shell_regions: a directive-close swallowed a later top-level shell block")
+ # shell regions: shell-tagged markdown fence + shell-tagged RST directive + inline span scanned;
+ # prose AND non-shell (python) blocks excluded — `vx` there is the module alias, not the CLI.
+ sh = CliSubcommandCheck._shell_regions(
+ "prose vx faketool\n```bash\nvx convert x\n```\nand `vx query`")
+ if "vx convert" not in sh or "vx query" not in sh or "vx faketool" in sh:
+ failures.append("_shell_regions (markdown shell fence) wrong: missing region or leaked prose")
+ py = CliSubcommandCheck._shell_regions("```python\nimport vortex as vx\nvx.array([1])\n```")
+ if "import vortex" in py:
+ failures.append("_shell_regions scanned a python block (vx-alias false-positive risk)")
+ rst = CliSubcommandCheck._shell_regions(
+ ".. code-block:: bash\n\n vx inspect f.vortex\n\nprose vx nope")
+ if "vx inspect" not in rst or "vx nope" in rst:
+ failures.append("_shell_regions (RST shell directive) wrong: missing block or leaked prose")
+ # a NON-shell RST code directive body is consumed, so its inline ``spans`` do not leak into the
+ # scan (the docs' api/*.rst use `.. code-block:: python`); a shell RST directive is still captured.
+ rst_py = CliSubcommandCheck._shell_regions(".. code-block:: python\n\n x = 1 # `vx frob`\n")
+ if "vx frob" in rst_py:
+ failures.append("_shell_regions leaked an inline span from a python RST directive body")
+ rst_sh = CliSubcommandCheck._shell_regions(".. code-block:: bash\n\n vx convert a\n")
+ if "vx convert" not in rst_sh:
+ failures.append("_shell_regions dropped a shell RST directive body")
+ # nested command paths: build a tree from synthetic cross-file crate source (the `Tree(TreeArgs)`
+ # -> `#[clap(subcommand)] mode: TreeMode` -> {array, layout} chain that the real CLI uses), then
+ # validate paths against it.
+ crate = (
+ "enum Commands {\n"
+ " /// tree\n Tree(super::tree::TreeArgs),\n"
+ " Convert(#[command(flatten)] super::convert::ConvertArgs),\n"
+ " Browse { file: PathBuf },\n}\n"
+ "pub struct TreeArgs {\n #[clap(subcommand)]\n pub mode: TreeMode,\n}\n"
+ "pub enum TreeMode {\n Array { file: PathBuf },\n Layout { file: PathBuf },\n}\n"
+ "pub struct ConvertArgs {\n pub file: PathBuf,\n}\n")
+ tree = CliSubcommandCheck._command_tree(crate, "Commands")
+ if tree != {"tree": {"array": {}, "layout": {}}, "convert": {}, "browse": {}}:
+ failures.append(f"_command_tree mis-resolved the nested command tree (got {tree!r})")
+ vp = CliSubcommandCheck._validate_path
+ # a fabricated SECOND-level subcommand is flagged; a real nested path + its args are valid
+ if vp(tree, ["tree", "frobnicate"]) != "tree frobnicate":
+ failures.append("_validate_path did not flag a fabricated nested subcommand")
+ if vp(tree, ["tree", "layout", "f.vortex"]) is not None:
+ failures.append("_validate_path flagged a valid nested path + arg")
+ # top-level fabrications (incl. case / underscore / dotted), real commands + args, flags, sentence dot
+ for bad_tokens, want in [(["frobnicate"], "frobnicate"), (["Convert"], "Convert"),
+ (["convert_file"], "convert_file"), (["convert.old"], "convert.old")]:
+ if vp(tree, bad_tokens) != want:
+ failures.append(f"_validate_path did not flag fabricated `{want}`")
+ for ok_tokens in (["convert", "a.parquet"], ["browse", "f.vortex"], ["convert."], ["--help"],
+ ["tree", "--verbose"]):
+ if vp(tree, ok_tokens) is not None:
+ failures.append(f"_validate_path flagged a valid invocation {ok_tokens!r}")
+ # a shell operator / separator stops the walk (so chained args are not mis-read as subcommands)
+ if vp(tree, ["convert", "f", "&&", "vx", "frobnicate"]) is not None:
+ failures.append("_validate_path walked past a shell operator into the next command")
+ # a REQUIRED nested subcommand omitted in favor of a path/arg is flagged (`vx tree ./file.vortex`
+ # would fail when copy-pasted); a leaf command's path arg, and a flag after `tree`, are still valid
+ if vp(tree, ["tree", "./file.vortex"]) != "tree ./file.vortex":
+ failures.append("_validate_path did not flag a missing REQUIRED nested subcommand (path arg)")
+ if vp(tree, ["tree", "2024.vortex"]) != "tree 2024.vortex":
+ failures.append("_validate_path did not flag a missing REQUIRED nested subcommand (numeric arg)")
+ if vp(tree, ["convert", "/tmp/a.parquet"]) is not None or vp(tree, ["tree", "--help"]) is not None:
+ failures.append("_validate_path flagged a leaf's path arg or a flag after a required-children cmd")
+ # rustfmt-wrapped MULTI-LINE tuple variant still exposes its nested subcommand type
+ crate_ml = (
+ "enum Commands {\n Tree(\n super::t::TreeArgs,\n ),\n}\n"
+ "struct TreeArgs {\n #[clap(subcommand)]\n pub mode: TreeMode,\n}\n"
+ "enum TreeMode {\n Array { f: PathBuf },\n Layout { f: PathBuf },\n}\n")
+ if CliSubcommandCheck._command_tree(crate_ml, "Commands") != {"tree": {"array": {}, "layout": {}}}:
+ failures.append("_command_tree missed a nested subcommand in a multi-line tuple variant")
+ # plural clap aliases (`visible_aliases = ["ls", "dir"]`) are honored
+ plural = CliSubcommandCheck._subcommand_names(
+ 'enum Commands {\n #[command(visible_aliases = ["ls", "dir"])]\n'
+ " Browse { f: PathBuf },\n}", "Commands")
+ if plural != {"browse", "ls", "dir"}:
+ failures.append(f"_subcommand_names ignored plural clap aliases (got {plural!r})")
+ # the command prefix is SOURCED from the crate's `[[bin]] name` (tomllib), picking the [[bin]] name
+ # over the [package] name — so a rename WOULD be picked up, not silently passed.
+ renamed = '[package]\nname = "vortex-tui"\n\n[[bin]]\nname = "renamed"\npath = "src/main.rs"\n'
+ if _toml_str_from(renamed, "synth", "bin", 0, "name") != "renamed":
+ failures.append("_toml_str_from did not select the [[bin]] name over [package] name")
+ # end-to-end on the real repo it currently resolves to the vx binary
+ if CLI_CHECKS[0].resolve_prefix(root) != "vx ":
+ failures.append(f"resolve_prefix wiring broken (got {CLI_CHECKS[0].resolve_prefix(root)!r})")
+ # nested subcommand detection tolerates EXTRA clap meta (`subcommand, required = true`)
+ crate_req = (
+ "enum Commands {\n Tree(super::t::TreeArgs),\n}\n"
+ "struct TreeArgs {\n #[command(subcommand, required = true)]\n pub mode: TreeMode,\n}\n"
+ "enum TreeMode {\n Array { f: PathBuf },\n Layout { f: PathBuf },\n}\n")
+ if CliSubcommandCheck._command_tree(crate_req, "Commands") != {"tree": {"array": {}, "layout": {}}}:
+ failures.append("_command_tree missed a nested subcommand with extra clap meta")
+ # variant metadata split across SEPARATE attributes is merged (the `name=` is honored)
+ split_names = CliSubcommandCheck._subcommand_names(
+ 'enum Commands {\n #[command(about = "x")]\n #[command(name = "ls")]\n'
+ " Browse { file: PathBuf },\n}", "Commands")
+ if split_names != {"ls"}:
+ failures.append(f"_subcommand_names ignored a split-attribute name= override (got {split_names!r})")
+ # OPTIONAL nested subcommand → lenient leaf (ACCEPTED TRADEOFF): `vx inspect ` is NOT
+ # flagged, because we cannot tell a fabricated optional-subcommand from the parent's own positional
+ # argument without risking false-positives on real `vx inspect `. Asserted as intentional.
+ if vp({"inspect": {}}, ["inspect", "bogus", "file.vortex"]) is not None:
+ failures.append("_validate_path flagged an optional-subcommand parent (should be lenient leaf)")
+ # end-to-end: per-line scan validates EACH same-line `vx` invocation (a second after `&&` is not
+ # hidden); `uvx ruff` is not matched as `vx ruff`; a line-final `vx` cannot bind across a newline.
+ chk = CLI_CHECKS[0]
+ # build the pattern from the SOURCED prefix (resolve_prefix), matching the live check path
+ pfx = re.compile(r"(? int:
+ ap = argparse.ArgumentParser(description="Vortex docs-conformance lint")
+ ap.add_argument("--self-test", action="store_true", help="prove the checker detects drift")
+ ap.add_argument("-v", "--verbose", action="store_true", help="print every check's resolved values")
+ ns = ap.parse_args()
+ root = repo_root()
+ return self_test(root) if ns.self_test else run_checks(root, ns.verbose)
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/scripts/encoding_stability.py b/scripts/encoding_stability.py
new file mode 100644
index 00000000000..ef89312a2ba
--- /dev/null
+++ b/scripts/encoding_stability.py
@@ -0,0 +1,504 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright the Vortex contributors
+"""Derive the STABLE Vortex encoding set from the repository, code-authoritatively.
+
+Stability rule (the single predicate): an array encoding is *in-spec* (STABLE) iff it is registered
+WITHOUT the `unstable_encodings` Cargo feature. The complementary set — encodings whose registration
+is gated behind `unstable_encodings` — is UNSTABLE (out-of-spec, covered by the fail-loud reject rule,
+not by a spec section).
+
+This is the prerequisite for the spec-conformance tripwire, which imports `stable_encodings()` and
+asserts each has a spec section. This module ONLY derives the set; it enforces nothing.
+
+Source of truth (both sides derived at runtime, never a hard-coded list). The must-spec set is the
+UNION of all first-party registered array encodings across THREE registration sources:
+- Source 1 — the vortex-array SESSION: the `this.register()` calls in `impl Default for
+ ArraySession` (vortex-array/src/session/mod.rs), where the CANONICAL kinds register (Null, Bool,
+ Primitive, Struct, List, ListView, FixedSizeList, VarBin, VarBinView, Decimal, Constant, Chunked,
+ Dict, Masked, Variant, Extension). These are in-scope encodings that need byte-layout specs. The
+ `this.register(...)` scan is scoped to THAT impl block only — the identically-shaped calls in the
+ sibling dtype/scalar_fn/aggregate_fn sessions register extension DTypes and scalar/aggregate fns
+ (NOT array encodings), so they are deliberately not collected.
+- Source 2 — the vortex-file DEFAULTS: the `arrays.register(...)` calls in `register_default_encodings`
+ (vortex-file/src/lib.rs) PLUS the per-crate `initialize()` functions it calls (e.g.
+ `vortex_alp::initialize` registers ALP/ALPRD; `vortex_fastlanes::initialize` registers
+ BitPacked/Delta/FoR/RLE). Following the initialize() calls is what exposes the finer per-encoding
+ granularity the spec cares about, rather than only the crate name.
+- Source 3 — PARQUET-VARIANT: `vortex.parquet.variant` (`encodings/parquet-variant`), registered via
+ `vortex_parquet_variant::initialize`. That initialize() is called from the JNI session, NOT from
+ `register_default_encodings`, so it is added as an explicit third source; it is stable (not
+ `unstable_encodings`-gated) and in-scope. Parsed from the crate's initialize() so it stays
+ code-authoritative.
+- {unstable_encodings-gated}: the `#[cfg(feature = "unstable_encodings")]` (or an `all(...)` cfg that
+ contains it) attribute on a registration, OR propagation from an initialize() call that is itself so
+ gated. The `unstable_encodings` feature is cross-checked as still-defined in vortex-file/Cargo.toml, so
+ a rename of the predicate fails loud instead of silently classifying everything STABLE.
+
+Three classes are produced:
+- STABLE — registered without `unstable_encodings` (this is the spec-covered set the tripwire consumes).
+- UNSTABLE — registered only under `unstable_encodings`.
+- PARKED — registered only inside a runtime `use_experimental_patches()` guard (the env-var-gated
+ Patched array + its ALP/BitPacked shim plugins). Treated as out-of-spec here; its env-var
+ gate is intentionally not reconciled onto the Cargo feature.
+
+Note: the base `Zstd` array is gated by `#[cfg(feature = "zstd")]` (a DEFAULT feature of the
+`vortex` crate), NOT `unstable_encodings`, so it is STABLE / in-spec — it ships by default and is a
+documented encoding. Only `ZstdBuffers` (the `unstable_encodings`-gated part of zstd) is UNSTABLE.
+
+Note: `vortex_tensor::initialize` registers extension DTypes and scalar functions, NOT an `arrays`
+encoding, so tensor contributes nothing to the array-encoding set (it is neither STABLE nor UNSTABLE
+here) even though its initialize() call is `unstable_encodings`-gated.
+
+Usage:
+ python scripts/encoding_stability.py # print the derived stable / unstable / parked sets
+ python scripts/encoding_stability.py --self-test # prove the derivation flips when gating changes
+"""
+from __future__ import annotations
+
+import argparse
+import re
+import subprocess
+import sys
+import tomllib
+from functools import lru_cache
+from pathlib import Path
+
+STABLE = "stable"
+UNSTABLE = "unstable"
+PARKED = "parked"
+
+# Highest wins when a name is seen more than once (defensive; encodings register in one place today).
+_PRECEDENCE = {STABLE: 0, PARKED: 1, UNSTABLE: 2}
+
+# `arrays.register(X)` (a local `arrays` binding in register_default_encodings) OR
+# `session.arrays().register(X)` (the per-crate initialize() form). Anchored to `arrays` so
+# `dtypes().register(...)` / `scalar_fns().register(...)` / `register_aggregate_kernel(...)` do NOT match.
+_REGISTER_RE = re.compile(r"\barrays(?:\(\))?\.register\(\s*([\w:]+)\s*\)")
+# `this.register(X)` — the receiver form used inside `impl Default for ArraySession` where the canonical
+# kinds register. WARNING: the identically-shaped `this.register(...)` calls in the sibling
+# dtype/scalar_fn/aggregate_fn sessions register NON-array items, so this regex is only ever applied to
+# the ArraySession Default impl body (see `_array_session_registrations`), never repo-wide.
+_SESSION_REGISTER_RE = re.compile(r"\bthis\.register\(\s*([\w:]+)\s*\)")
+ARRAY_SESSION_SRC = "vortex-array/src/session/mod.rs"
+# A `vortex_::initialize(session)` call — group 1 is the crate module (e.g. `alp`, `datetime_parts`).
+_INIT_RE = re.compile(r"\bvortex_(\w+)::initialize\s*\(")
+_CFG_RE = re.compile(r"#\[cfg\((.*)\)\]")
+# The runtime experimental-patches guard body (single-statement, no nested braces) — its registrations are
+# PARKED. `finditer` collects every such guard in a fn body (ALP + FastLanes each have one).
+_ENV_GUARD_RE = re.compile(r"use_experimental_patches\s*\(\s*\)\s*\{([^{}]*)\}", re.DOTALL)
+_UNSTABLE_CFG_RE = re.compile(r'feature\s*=\s*"unstable_encodings"')
+
+UNSTABLE_FEATURE = "unstable_encodings"
+
+
+def repo_root() -> Path:
+ """The git top-level, or — on any failure to obtain it — the script's parent's parent (the script
+ lives at /scripts/)."""
+ try:
+ out = subprocess.run(
+ ["git", "rev-parse", "--show-toplevel"],
+ capture_output=True, text=True, check=True,
+ ).stdout.strip()
+ return Path(out)
+ except (FileNotFoundError, subprocess.CalledProcessError):
+ return Path(__file__).resolve().parent.parent
+
+
+def _strip_rust_comments(src: str) -> str:
+ """Remove `//` line comments and NESTED `/* */` block comments from Rust source, so a decoy
+ registration hidden in a comment cannot be sourced. (Copied from the docs-conformance harness to keep
+ this module standalone-importable without a circular dependency.)"""
+ out: list[str] = []
+ i, n, depth = 0, len(src), 0
+ while i < n:
+ two = src[i:i + 2]
+ if depth == 0 and two == "//":
+ j = src.find("\n", i)
+ i = n if j < 0 else j
+ elif two == "/*":
+ depth += 1
+ i += 2
+ elif two == "*/" and depth > 0:
+ depth -= 1
+ i += 2
+ elif depth > 0:
+ i += 1
+ else:
+ out.append(src[i])
+ i += 1
+ return "".join(out)
+
+
+def _brace_body(src: str, header_re: str, label: str) -> str:
+ """The brace-matched `{ ... }` body following the FIRST match of `header_re` in `src` (comments
+ assumed stripped). Raises if the header or its closing brace is not found, so a moved/renamed source
+ fails loud."""
+ m = re.search(header_re, src)
+ if not m:
+ raise LookupError(f"could not find {label} — the registration source moved")
+ start = src.index("{", m.start())
+ depth, end = 0, None
+ for j in range(start, len(src)):
+ if src[j] == "{":
+ depth += 1
+ elif src[j] == "}":
+ depth -= 1
+ if depth == 0:
+ end = j
+ break
+ if end is None:
+ raise LookupError(f"{label} body is not closed")
+ return src[start + 1:end]
+
+
+def _fn_body(src: str, fn_name: str) -> str:
+ """The brace-matched body of `fn (...) { ... }` in `src`, tolerating an optional
+ `-> ReturnType` before the brace (comments assumed stripped). Raises if the function or its closing
+ brace is not found, so a moved/renamed source fails loud."""
+ return _brace_body(
+ src, rf"fn {re.escape(fn_name)}\s*\([^)]*\)\s*(?:->\s*[^{{;]+?)?\{{", f"`fn {fn_name}(...)`")
+
+
+@lru_cache(maxsize=None)
+def _crate_dirs(root: Path) -> dict[str, Path]:
+ """Map each workspace crate's package name to its directory (from every Cargo.toml `[package] name`),
+ so a `vortex_::initialize` call resolves to that crate's source. Excludes build output."""
+ dirs: dict[str, Path] = {}
+ for cargo in root.rglob("Cargo.toml"):
+ s = str(cargo)
+ if "/target/" in s or "/.git/" in s:
+ continue
+ m = re.search(r'^\s*name\s*=\s*"([^"]+)"', cargo.read_text(encoding="utf-8"), re.MULTILINE)
+ if m:
+ dirs[m.group(1)] = cargo.parent
+ if len(dirs) < 10:
+ raise LookupError(f"found only {len(dirs)} workspace crates; the Cargo.toml scan may be broken")
+ return dirs
+
+
+def _env_gated_idents(body: str) -> set[str]:
+ """Registered idents inside a `use_experimental_patches()` runtime guard within `body` — the PARKED
+ (env-var-gated) encodings, derived from the guard rather than hard-coded to `Patched`."""
+ ids: set[str] = set()
+ for m in _ENV_GUARD_RE.finditer(body):
+ ids |= {x.split("::")[-1] for x in _REGISTER_RE.findall(m.group(1))}
+ return ids
+
+
+def _walk_registrations(body: str, register_re: re.Pattern[str] = _REGISTER_RE):
+ """Yield `(kind, name, cfg)` for each registration statement in a fn body, in source order.
+ `kind` is 'register' (name = the registered struct ident, last path segment) or 'init' (name = the
+ `vortex_::initialize` crate module). `register_re` selects the receiver form: the default
+ `arrays`/`arrays()` form (Sources 2/3), or `_SESSION_REGISTER_RE` (`this.register`) for the
+ vortex-array session body (Source 1). `cfg` is the raw contents of the `#[cfg(...)]` attribute
+ immediately preceding the statement, or None. A `#[cfg(...)]` attribute on its own line applies to the
+ next statement; any other non-blank line clears a pending attribute."""
+ pending: str | None = None
+ for raw in body.split("\n"):
+ s = raw.strip()
+ if not s:
+ continue
+ cfgm = _CFG_RE.search(s)
+ has_stmt = register_re.search(s) or _INIT_RE.search(s)
+ if cfgm and not has_stmt:
+ pending = cfgm.group(1)
+ continue
+ matched = False
+ for m in register_re.finditer(s):
+ yield "register", m.group(1).split("::")[-1], pending
+ matched = True
+ for m in _INIT_RE.finditer(s):
+ yield "init", m.group(1), pending
+ matched = True
+ pending = None if matched or not cfgm else pending
+
+
+def _cfg_is_unstable(cfg: str | None) -> bool:
+ """Whether a `#[cfg(...)]` contents string gates on `unstable_encodings` (bare or within `all(...)`)."""
+ return cfg is not None and _UNSTABLE_CFG_RE.search(cfg) is not None
+
+
+def _classify(is_unstable: bool, is_env_gated: bool) -> str:
+ """UNSTABLE wins over PARKED wins over STABLE. A non-`unstable_encodings` feature gate (e.g. `zstd`)
+ does NOT make an encoding unstable — that is the deliberate rule (see the module docstring)."""
+ if is_unstable:
+ return UNSTABLE
+ if is_env_gated:
+ return PARKED
+ return STABLE
+
+
+def _merge(out: dict[str, str], name: str, cls: str) -> None:
+ if name not in out or _PRECEDENCE[cls] > _PRECEDENCE[out[name]]:
+ out[name] = cls
+
+
+def stability_from_body(
+ body: str, *, parent_unstable: bool = False, register_re: re.Pattern[str] = _REGISTER_RE
+) -> dict[str, str]:
+ """Classify the DIRECT `register(...)` calls in a single fn body (no crate recursion) — the pure core
+ the self-test drives on synthetic input to prove a gating flip moves an encoding between sets.
+ `parent_unstable` models an `unstable_encodings`-gated initialize() call enclosing the body.
+ `register_re` selects the receiver form (default `arrays`; `_SESSION_REGISTER_RE` for `this.register`)."""
+ env = _env_gated_idents(body)
+ out: dict[str, str] = {}
+ for kind, name, cfg in _walk_registrations(body, register_re):
+ if kind != "register":
+ continue
+ _merge(out, name, _classify(parent_unstable or _cfg_is_unstable(cfg), name in env))
+ return out
+
+
+def _array_session_registrations(root: Path) -> dict[str, str]:
+ """Classify the canonical array kinds registered by `impl Default for ArraySession` in vortex-array
+ (Source 1 — the `this.register()` calls). These are in-scope encodings that need byte-layout
+ specs. Scoped to that impl block so the identically-shaped `this.register(...)` calls in the sibling
+ dtype/scalar_fn/aggregate_fn sessions (extension DTypes, scalar/aggregate fns — NOT array encodings)
+ are not miscollected. Fails loud if implausibly few kinds are found (the parse likely broke)."""
+ src = _strip_rust_comments((root / ARRAY_SESSION_SRC).read_text(encoding="utf-8"))
+ body = _brace_body(src, r"impl Default for ArraySession\s*\{", "`impl Default for ArraySession`")
+ out = stability_from_body(body, register_re=_SESSION_REGISTER_RE)
+ if len(out) < 8:
+ raise LookupError(
+ f"derived only {len(out)} canonical kinds from `impl Default for ArraySession` in "
+ f"{ARRAY_SESSION_SRC}; the session-registration parse likely broke")
+ return out
+
+
+def _initialize_body(root: Path, crate_mod: str) -> str:
+ """The body of `pub fn initialize(session: ...)` for the crate behind a `vortex_::initialize`
+ call. `crate_mod` (e.g. `datetime_parts`) maps to package `vortex-datetime-parts`."""
+ pkg = "vortex-" + crate_mod.replace("_", "-")
+ d = _crate_dirs(root).get(pkg)
+ if d is None:
+ raise LookupError(f"initialize() references crate {pkg!r}, not found in the workspace")
+ for p in sorted((d / "src").rglob("*.rs")):
+ t = _strip_rust_comments(p.read_text(encoding="utf-8"))
+ if re.search(r"\bpub fn initialize\s*\(", t):
+ return _fn_body(t, "initialize")
+ raise LookupError(f"no `pub fn initialize` found in {pkg}")
+
+
+def _collect(root: Path, body: str, parent_unstable: bool, visited: frozenset[str]) -> dict[str, str]:
+ """Classify every array encoding reachable from `body`: its direct `arrays.register(...)` calls, plus
+ (recursively, cycle-guarded) the encodings registered by each `vortex_::initialize` it calls.
+ `parent_unstable` propagates an `unstable_encodings` gate down into a called initialize()."""
+ env = _env_gated_idents(body)
+ out: dict[str, str] = {}
+ for kind, name, cfg in _walk_registrations(body):
+ gate_unstable = parent_unstable or _cfg_is_unstable(cfg)
+ if kind == "register":
+ _merge(out, name, _classify(gate_unstable, name in env))
+ else: # 'init' — descend into the crate's initialize()
+ pkg = "vortex-" + name.replace("_", "-")
+ if pkg in visited:
+ continue
+ child = _collect(root, _initialize_body(root, name), gate_unstable, visited | {pkg})
+ for cn, cc in child.items():
+ _merge(out, cn, cc)
+ return out
+
+
+def _assert_unstable_feature_defined(root: Path) -> None:
+ """Fail loud if vortex-file/Cargo.toml no longer defines the `unstable_encodings` feature — the cfg
+ gate this module keys on. Ties the two sides together: a rename would move both, and a mismatch (only
+ the feature or only the cfg renamed) is caught rather than silently classifying everything STABLE."""
+ data = tomllib.loads((root / "vortex-file/Cargo.toml").read_text(encoding="utf-8"))
+ if UNSTABLE_FEATURE not in data.get("features", {}):
+ raise LookupError(
+ f"vortex-file/Cargo.toml no longer defines the `{UNSTABLE_FEATURE}` feature; the stability "
+ "predicate moved — update encoding_stability.py")
+
+
+@lru_cache(maxsize=None)
+def classify_encodings(root: Path) -> dict[str, str]:
+ """The derived classification: array-encoding name -> STABLE / UNSTABLE / PARKED, sourced from the
+ UNION of the three registration sources (see the module docstring): the vortex-array session
+ canonical kinds, the vortex-file `register_default_encodings` (+ the crate initialize() fns it
+ calls), and parquet-variant. Fails loud if any registration source, an initialize() crate, or the
+ `unstable_encodings` feature moved."""
+ _assert_unstable_feature_defined(root)
+ result: dict[str, str] = {}
+
+ # Source 1 — vortex-array session: the canonical kinds (`this.register(...)`), always in the default
+ # session (VortexSession::default wires `.with::()`); all ungated -> STABLE.
+ for name, cls in _array_session_registrations(root).items():
+ _merge(result, name, cls)
+
+ # Source 2 — vortex-file: register_default_encodings + the crate initialize() fns it calls, with
+ # per-registration unstable_encodings / zstd-feature / env-patch gating classified as today.
+ src = _strip_rust_comments((root / "vortex-file/src/lib.rs").read_text(encoding="utf-8"))
+ body = _fn_body(src, "register_default_encodings")
+ for name, cls in _collect(root, body, parent_unstable=False,
+ visited=frozenset({"vortex-file"})).items():
+ _merge(result, name, cls)
+
+ # Source 3 — parquet-variant: parsed from its own initialize() (called from the JNI session, not
+ # register_default_encodings). Stable, in-scope.
+ for name, cls in _collect(root, _initialize_body(root, "parquet_variant"), parent_unstable=False,
+ visited=frozenset({"vortex-parquet-variant"})).items():
+ _merge(result, name, cls)
+
+ if "ParquetVariant" not in result:
+ raise LookupError(
+ "expected `ParquetVariant` from vortex_parquet_variant::initialize; the parquet-variant "
+ "registration source moved — update encoding_stability.py")
+ if len(result) < 24:
+ raise LookupError(
+ f"derived only {len(result)} encodings across the array session + file defaults + "
+ "parquet-variant; the parse likely broke")
+ return dict(result)
+
+
+def stable_encodings(root: Path) -> set[str]:
+ """The spec-covered stable encoding set = registered encodings NOT gated behind `unstable_encodings`
+ (and not env-gated). This is the predicate the tripwire consumes."""
+ return {n for n, c in classify_encodings(root).items() if c == STABLE}
+
+
+def unstable_encodings(root: Path) -> set[str]:
+ """The excluded set: encodings registered only under the `unstable_encodings` Cargo feature."""
+ return {n for n, c in classify_encodings(root).items() if c == UNSTABLE}
+
+
+def parked_encodings(root: Path) -> set[str]:
+ """Encodings registered only inside a `use_experimental_patches()` env-var guard — out-of-spec by
+ fiat (Patched + its ALP/BitPacked shim plugins); their env-gate is not reconciled onto the feature."""
+ return {n for n, c in classify_encodings(root).items() if c == PARKED}
+
+
+def feature_gated_stable(root: Path) -> set[str]:
+ """STABLE encodings that are nonetheless behind SOME (non-`unstable_encodings`) cfg feature gate —
+ today just `Zstd` (behind the default `zstd` feature). Surfaced as the known divergence from the
+ original 'exclude zstd' expectation. Derived, not hard-coded (see the module docstring)."""
+ src = _strip_rust_comments((root / "vortex-file/src/lib.rs").read_text(encoding="utf-8"))
+ body = _fn_body(src, "register_default_encodings")
+ stable = stable_encodings(root)
+ gated: set[str] = set()
+ for kind, name, cfg in _walk_registrations(body):
+ if kind == "register" and cfg is not None and not _cfg_is_unstable(cfg) and name in stable:
+ gated.add(name)
+ return gated
+
+
+def self_test_failures(root: Path) -> list[str]:
+ """Negative + positive checks proving the derivation's logic on SYNTHETIC input, then confirming the
+ live derivation. Returned as a list of failure strings (empty == green), in the docs-harness style."""
+ f: list[str] = []
+
+ # --- drift: flipping the unstable_encodings gate moves an encoding between the sets --------------
+ off = stability_from_body("arrays.register(Widget);\n arrays.register(Keeper);")
+ if off.get("Widget") != STABLE or off.get("Keeper") != STABLE:
+ f.append("baseline: an ungated encoding must be STABLE")
+ on = stability_from_body('#[cfg(feature = "unstable_encodings")]\n arrays.register(Widget);\n'
+ " arrays.register(Keeper);")
+ if on.get("Widget") != UNSTABLE:
+ f.append("drift: gating Widget behind unstable_encodings did not move it to UNSTABLE")
+ if on.get("Keeper") != STABLE:
+ f.append("drift: gating one encoding wrongly reclassified an unrelated one")
+ if "Widget" not in {n for n, c in off.items() if c == STABLE}:
+ f.append("drift: Widget should be in the STABLE set when ungated")
+ if "Widget" in {n for n, c in on.items() if c == STABLE}:
+ f.append("drift: Widget still in the STABLE set after being gated behind unstable_encodings")
+
+ # --- a non-unstable feature gate (the zstd-base case) stays STABLE; a combined cfg is UNSTABLE ---
+ if stability_from_body('#[cfg(feature = "zstd")]\n arrays.register(Zstd);').get("Zstd") != STABLE:
+ f.append("a non-unstable feature gate (zstd) must remain STABLE per the rule")
+ combined = stability_from_body('#[cfg(all(feature = "zstd", feature = "unstable_encodings"))]\n'
+ " arrays.register(ZstdBuffers);")
+ if combined.get("ZstdBuffers") != UNSTABLE:
+ f.append("a combined cfg containing unstable_encodings must be UNSTABLE")
+
+ # --- env-gated (Patched-style) shim is PARKED; its else-branch encoding stays STABLE ------------
+ env = stability_from_body("if use_experimental_patches() {\n arrays.register(Shim);\n"
+ " } else {\n arrays.register(Real);\n }")
+ if env.get("Shim") != PARKED or env.get("Real") != STABLE:
+ f.append("env-gated shim should be PARKED and its else-branch encoding STABLE")
+
+ # --- an unstable_encodings-gated initialize() propagates UNSTABLE to its registrations ----------
+ if stability_from_body("arrays.register(Inner);", parent_unstable=True).get("Inner") != UNSTABLE:
+ f.append("an encoding under an unstable_encodings-gated initialize() must be UNSTABLE")
+
+ # --- Source 1: the vortex-array session receiver form (`this.register(...)`) is parsed ----------
+ sess = stability_from_body("this.register(Canon);\n #[cfg(feature = \"unstable_encodings\")]\n"
+ " this.register(Exp);", register_re=_SESSION_REGISTER_RE)
+ if sess.get("Canon") != STABLE:
+ f.append("session source: an ungated `this.register(...)` canonical kind must be STABLE")
+ if sess.get("Exp") != UNSTABLE:
+ f.append("session source: a `this.register(...)` gated behind unstable_encodings must be UNSTABLE")
+ # scoping guarantee: the default `arrays` receiver must NOT pick up a `this.register(...)` call, so
+ # the sibling dtype/scalar_fn/aggregate_fn sessions cannot leak into the array-encoding set.
+ if stability_from_body("this.register(Canon);").get("Canon") is not None:
+ f.append("scoping: the default `arrays` receiver wrongly matched a `this.register(...)` call")
+
+ # --- live derivation over the real repo ---------------------------------------------------------
+ try:
+ cls = classify_encodings(root)
+ except (LookupError, OSError) as e:
+ f.append(f"live derivation raised {type(e).__name__}: {e}")
+ return f
+ stable = {n for n, c in cls.items() if c == STABLE}
+ unstable = {n for n, c in cls.items() if c == UNSTABLE}
+ parked = {n for n, c in cls.items() if c == PARKED}
+ for name in ("ByteBool", "Dict", "FSST", "Pco", "ZigZag", "ALP", "ALPRD", "FoR", "Delta", "RLE",
+ "BitPacked", "RunEnd", "Sparse", "DateTimeParts", "DecimalByteParts", "Sequence",
+ # Source 1 — canonical kinds from the vortex-array session
+ "Null", "Bool", "Primitive", "Decimal", "VarBin", "VarBinView", "Struct", "List",
+ "ListView", "FixedSizeList", "Constant", "Chunked", "Masked", "Variant", "Extension",
+ # Source 3 — parquet-variant
+ "ParquetVariant"):
+ if name not in stable:
+ f.append(f"live: expected {name} in the STABLE set (got {sorted(stable)})")
+ if unstable != {"OnPair", "ZstdBuffers"}:
+ f.append(f"live: expected UNSTABLE == {{OnPair, ZstdBuffers}}, got {sorted(unstable)}")
+ if "Patched" not in parked:
+ f.append(f"live: expected Patched in the PARKED set (got {sorted(parked)})")
+ if stable & unstable or stable & parked or unstable & parked:
+ f.append("live: stable/unstable/parked sets are not disjoint")
+ if feature_gated_stable(root) != {"Zstd"}:
+ f.append(f"live: expected the known divergence feature_gated_stable == {{Zstd}}, "
+ f"got {sorted(feature_gated_stable(root))}")
+ return f
+
+
+def _print_sets(root: Path) -> int:
+ cls = classify_encodings(root)
+ stable = sorted(n for n, c in cls.items() if c == STABLE)
+ unstable = sorted(n for n, c in cls.items() if c == UNSTABLE)
+ parked = sorted(n for n, c in cls.items() if c == PARKED)
+ print(f"STABLE encodings ({len(stable)}, spec-covered): {', '.join(stable)}")
+ print(f"UNSTABLE encodings ({len(unstable)}, excluded via unstable_encodings): {', '.join(unstable)}")
+ print(f"PARKED encodings ({len(parked)}, env-gated / out-of-spec): {', '.join(parked)}")
+ print("\nSources unioned: (1) vortex-array session canonical kinds, (2) vortex-file "
+ "register_default_encodings + crate initialize() fns, (3) parquet-variant.")
+ diverge = sorted(feature_gated_stable(root))
+ if diverge:
+ print(f"\nNOTE: {', '.join(diverge)} classified STABLE by the unstable_encodings rule "
+ "but gated behind a non-unstable feature (`zstd`, a default) and is in-spec; only ZstdBuffers "
+ "(the unstable_encodings-gated part) is excluded (see module docstring).")
+ print("NOTE: vortex_tensor::initialize registers extension DTypes/scalar-fns, not an array encoding, "
+ "so tensor is absent from all three sets by construction.")
+ return 0
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser(description="Derive the stable Vortex encoding set from the repo")
+ ap.add_argument("--self-test", action="store_true", help="prove the derivation flips on a gating change")
+ ns = ap.parse_args()
+ root = repo_root()
+ if ns.self_test:
+ failures = self_test_failures(root)
+ for msg in failures:
+ print(f" {msg}", file=sys.stderr)
+ if failures:
+ print(f"\nSELF-TEST FAILED: {len(failures)} check(s).", file=sys.stderr)
+ return 1
+ print("SELF-TEST OK — gating flips move an encoding between the stable/unstable sets, and the live "
+ "derivation matches the expected classification.")
+ return 0
+ return _print_sets(root)
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/vortex-flatbuffers/flatbuffers/vortex-array/array.fbs b/vortex-flatbuffers/flatbuffers/vortex-array/array.fbs
index c3c3ddfd34a..f2869c16b19 100644
--- a/vortex-flatbuffers/flatbuffers/vortex-array/array.fbs
+++ b/vortex-flatbuffers/flatbuffers/vortex-array/array.fbs
@@ -1,8 +1,9 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright the Vortex contributors
-/// An Array describes the hierarchy of an array as well as the locations of the data buffers that appear
-/// immediately after the message in the byte stream.
+/// An Array describes the hierarchy of an array as well as the locations of the data buffers. In the
+/// byte stream the data buffers appear first; this Array message is written as a suffix after them,
+/// followed by a little-endian u32 length of the message.
table Array {
/// The array's hierarchical definition.
root: ArrayNode;
diff --git a/vortex-flatbuffers/flatbuffers/vortex-file/footer.fbs b/vortex-flatbuffers/flatbuffers/vortex-file/footer.fbs
index 9d6e4be1d1c..18bd0270d6c 100644
--- a/vortex-flatbuffers/flatbuffers/vortex-file/footer.fbs
+++ b/vortex-flatbuffers/flatbuffers/vortex-file/footer.fbs
@@ -6,7 +6,7 @@ include "vortex-layout/layout.fbs";
// [postscript]
/// The `Postscript` is guaranteed by the file format to never exceed
-/// 65528 bytes (i.e., u16::MAX - 8 bytes) in length, and is immediately
+/// 65527 bytes (i.e., u16::MAX - 8 bytes) in length, and is immediately
/// followed by an 8-byte `EndOfFile` struct.
///
/// An initial read of a Vortex file defaults to at least 64KB (u16::MAX bytes) and therefore
diff --git a/vortex-flatbuffers/flatbuffers/vortex-serde/message.fbs b/vortex-flatbuffers/flatbuffers/vortex-serde/message.fbs
index f159023dc30..30b5b48b7bd 100644
--- a/vortex-flatbuffers/flatbuffers/vortex-serde/message.fbs
+++ b/vortex-flatbuffers/flatbuffers/vortex-serde/message.fbs
@@ -8,7 +8,8 @@ enum MessageVersion: uint8 {
V0 = 0,
}
-/// Indicates the message body contains a flatbuffer Array message, followed by array buffers.
+/// Indicates the message body is a serialized array: the data buffers first, followed by a flatbuffer
+/// Array message and a trailing little-endian u32 length of that message.
table ArrayMessage {
/// The row count of the array.
row_count: uint32;
diff --git a/vortex-flatbuffers/src/generated/array.rs b/vortex-flatbuffers/src/generated/array.rs
index 6d903a56aa5..cad4d2761b0 100644
--- a/vortex-flatbuffers/src/generated/array.rs
+++ b/vortex-flatbuffers/src/generated/array.rs
@@ -371,8 +371,9 @@ impl<'a> Buffer {
pub enum ArrayOffset {}
#[derive(Copy, Clone, PartialEq)]
-/// An Array describes the hierarchy of an array as well as the locations of the data buffers that appear
-/// immediately after the message in the byte stream.
+/// An Array describes the hierarchy of an array as well as the locations of the data buffers. In the
+/// byte stream the data buffers appear first; this Array message is written as a suffix after them,
+/// followed by a little-endian u32 length of the message.
pub struct Array<'a> {
pub _tab: ::flatbuffers::Table<'a>,
}
diff --git a/vortex-flatbuffers/src/generated/footer.rs b/vortex-flatbuffers/src/generated/footer.rs
index 842a151edc7..680518a2e64 100644
--- a/vortex-flatbuffers/src/generated/footer.rs
+++ b/vortex-flatbuffers/src/generated/footer.rs
@@ -328,7 +328,7 @@ pub enum PostscriptOffset {}
#[derive(Copy, Clone, PartialEq)]
/// The `Postscript` is guaranteed by the file format to never exceed
-/// 65528 bytes (i.e., u16::MAX - 8 bytes) in length, and is immediately
+/// 65527 bytes (i.e., u16::MAX - 8 bytes) in length, and is immediately
/// followed by an 8-byte `EndOfFile` struct.
///
/// An initial read of a Vortex file defaults to at least 64KB (u16::MAX bytes) and therefore
diff --git a/vortex-flatbuffers/src/generated/message.rs b/vortex-flatbuffers/src/generated/message.rs
index b3e4be76ef7..8f53dd27f96 100644
--- a/vortex-flatbuffers/src/generated/message.rs
+++ b/vortex-flatbuffers/src/generated/message.rs
@@ -182,7 +182,8 @@ pub struct MessageHeaderUnionTableOffset {}
pub enum ArrayMessageOffset {}
#[derive(Copy, Clone, PartialEq)]
-/// Indicates the message body contains a flatbuffer Array message, followed by array buffers.
+/// Indicates the message body is a serialized array: the data buffers first, followed by a flatbuffer
+/// Array message and a trailing little-endian u32 length of that message.
pub struct ArrayMessage<'a> {
pub _tab: ::flatbuffers::Table<'a>,
}
diff --git a/vortex-tui/src/lib.rs b/vortex-tui/src/lib.rs
index 98a3343f24d..94e548ca1bd 100644
--- a/vortex-tui/src/lib.rs
+++ b/vortex-tui/src/lib.rs
@@ -70,7 +70,7 @@ mod native_cli {
enum Commands {
/// Print tree views of a Vortex file (layout tree or array tree)
Tree(super::tree::TreeArgs),
- /// Convert a Parquet file to a Vortex file. Chunking occurs on Parquet RowGroup boundaries.
+ /// Convert a Parquet file to a Vortex file. Chunking occurs in 8192-row batches.
Convert(#[command(flatten)] super::convert::ConvertArgs),
/// Interactively browse the Vortex file.
Browse { file: PathBuf },