From 61e3b219cbed114737c826ad30dd60c47cd3b6fc Mon Sep 17 00:00:00 2001 From: mprammer Date: Mon, 13 Jul 2026 11:35:55 -0400 Subject: [PATCH 1/2] Add file-level user metadata Adds optional file-level metadata to the Vortex file format: string-keyed, opaque byte segments referenced from the postscript. Keys and locators live in the postscript (read at open); values load only on `include_metadata`, resolved per-locator through the segment source (from the initial footer read when covered, else a targeted read) and copied out. A default open materializes nothing. `Footer` carries only locators, so a cached footer is identical regardless of the opener's flag and one file's metadata can't surface for another via the multi-file cache. `DType`, its FlatBuffers/protobuf serialization, and the scan path are untouched; the wire change is a single additive `Postscript` field (file version stays 1). `Alignment::from_exponent` on untrusted postscript input now returns an error instead of panicking (the TUI inspector included). Revives Nicholas Gates' file-metadata-segments work (#7954), rebased onto develop and reshaped so metadata never widens the footer read or pins its backing buffer. cargo build + fmt + clippy clean; vortex-file 100/100, vortex-buffer 822/822. Co-Authored-By: Nicholas Gates Co-Authored-By: Claude Signed-off-by: mprammer --- docs/specs/file-format.md | 7 + vortex-buffer/src/alignment.rs | 28 +- vortex-file/src/file.rs | 27 ++ vortex-file/src/footer/deserializer.rs | 58 +++- vortex-file/src/footer/mod.rs | 47 ++- vortex-file/src/footer/postscript.rs | 323 +++++++++++++++++- vortex-file/src/footer/segment.rs | 2 +- vortex-file/src/footer/serializer.rs | 103 +++++- vortex-file/src/lib.rs | 12 +- vortex-file/src/multi/mod.rs | 198 +++++++++++ vortex-file/src/open.rs | 213 +++++++++++- vortex-file/src/tests.rs | 176 ++++++++++ vortex-file/src/writer.rs | 97 +++++- .../flatbuffers/vortex-file/footer.fbs | 11 + vortex-flatbuffers/src/generated/footer.rs | 137 ++++++++ vortex-tui/src/inspect.rs | 34 +- 16 files changed, 1410 insertions(+), 63 deletions(-) diff --git a/docs/specs/file-format.md b/docs/specs/file-format.md index 425294f2d99..8ef94fa54aa 100644 --- a/docs/specs/file-format.md +++ b/docs/specs/file-format.md @@ -52,6 +52,13 @@ The postscript contains the locations of: 2. a `layout` segment containing the root `Layout` 3. a `statistics` segment containing file-level per-field statistics (e.g., minima and maxima of each field/column, for whole-file pruning) 4. a `footer` segment containing a dictionary-encoded _segment map_, and other shared configuration such as compression and encryption schemes +5. up to 16 user-defined `metadata` segments, each identified by a unique, non-empty UTF-8 key of at most 32 bytes + +The postscript carries a locator (offset, length, and alignment) for each metadata segment that is +present; a file written without user metadata (and any file predating this feature) carries none. +Readers do not load the opaque metadata values by default. Opt-in metadata reads resolve each +locator separately, allowing values outside the initial file-tail read to be fetched without reading +the intervening file contents. :::{literalinclude} ../../vortex-flatbuffers/flatbuffers/vortex-file/footer.fbs :start-after: [postscript] diff --git a/vortex-buffer/src/alignment.rs b/vortex-buffer/src/alignment.rs index ad5c51761bd..c46f622ec8b 100644 --- a/vortex-buffer/src/alignment.rs +++ b/vortex-buffer/src/alignment.rs @@ -111,13 +111,32 @@ impl Alignment { .vortex_expect("alignment is a power of 2 within usize, so its exponent fits in u8") } + /// Try to create an alignment from its log2 exponent. + /// + /// Returns an error when the exponent cannot be represented by a `usize` alignment. + #[inline] + pub fn try_from_exponent(exponent: u8) -> Result { + let alignment = 1usize.checked_shl(u32::from(exponent)).ok_or_else(|| { + vortex_err!( + "Alignment exponent {exponent} must be less than {}", + usize::BITS + ) + })?; + Ok(Self::new(alignment)) + } + /// Create from the log2 exponent of the alignment. /// /// ## Panics /// - /// Panics if `1 << exponent` overflows `usize`. + /// Panics if `1 << exponent` overflows `usize`. Use [`Self::try_from_exponent`] when parsing + /// untrusted input. #[inline] pub const fn from_exponent(exponent: u8) -> Self { + assert!( + (exponent as u32) < usize::BITS, + "Alignment exponent must fit in usize" + ); Self::new(1 << exponent) } } @@ -214,6 +233,13 @@ mod test { assert_eq!(Alignment::from_exponent(10), alignment); } + #[test] + fn invalid_alignment_exponent() { + let error = Alignment::try_from_exponent(usize::BITS as u8) + .expect_err("an exponent as wide as usize must be rejected"); + assert!(error.to_string().contains("must be less than")); + } + #[test] fn is_aligned_to() { assert!(Alignment::new(1).is_aligned_to(Alignment::new(1))); diff --git a/vortex-file/src/file.rs b/vortex-file/src/file.rs index c9a71f85c55..df7a8e35891 100644 --- a/vortex-file/src/file.rs +++ b/vortex-file/src/file.rs @@ -15,6 +15,7 @@ use vortex_array::ArrayRef; use vortex_array::dtype::DType; use vortex_array::dtype::FieldMask; use vortex_array::expr::Expression; +use vortex_buffer::ByteBuffer; use vortex_error::VortexResult; use vortex_layout::LayoutReader; use vortex_layout::scan::layout::LayoutReaderDataSource; @@ -23,6 +24,7 @@ use vortex_layout::scan::split_by::SplitBy; use vortex_layout::segments::SegmentSource; use vortex_scan::DataSourceRef; use vortex_session::VortexSession; +use vortex_utils::aliases::hash_map::HashMap; use crate::FileStatistics; use crate::footer::Footer; @@ -42,6 +44,8 @@ pub struct VortexFile { segment_source: Arc, /// The Vortex session used to open this file. session: VortexSession, + /// User-defined metadata values resolved for this file open. + metadata: Arc>, /// None id LayoutReader caching is turned off layout_reader_cache: Option>>, } @@ -78,10 +82,16 @@ impl VortexFile { footer, segment_source, session, + metadata: Arc::new(HashMap::new()), layout_reader_cache: None, } } + pub(crate) fn with_metadata(mut self, metadata: Arc>) -> Self { + self.metadata = metadata; + self + } + /// Enable layout reader caching. /// /// Repeated calls to [`layout_reader`](Self::layout_reader), [`scan`](Self::scan), and @@ -91,6 +101,7 @@ impl VortexFile { footer: self.footer, segment_source: self.segment_source, session: self.session, + metadata: self.metadata, layout_reader_cache: Some(OnceLock::new()), } } @@ -117,6 +128,22 @@ impl VortexFile { self.footer.statistics() } + /// Returns the user-defined metadata segments loaded for this file. + /// + /// Metadata is only loaded when requested during open. Iteration order is unspecified. + pub fn metadata_segments(&self) -> impl Iterator { + self.metadata + .iter() + .map(|(key, metadata)| (key.as_str(), metadata)) + } + + /// Returns the loaded user-defined metadata segment for the given key. + /// + /// Returns `None` when the key is absent or metadata was not loaded. + pub fn metadata_segment(&self, key: &str) -> Option<&ByteBuffer> { + self.metadata.get(key) + } + /// Create a new segment source for reading from the file. /// /// This may spawn a background I/O driver that will exit when the returned segment source diff --git a/vortex-file/src/footer/deserializer.rs b/vortex-file/src/footer/deserializer.rs index a0f12e85c8a..bfb1455b3e3 100644 --- a/vortex-file/src/footer/deserializer.rs +++ b/vortex-file/src/footer/deserializer.rs @@ -1,6 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use std::sync::Arc; + use flatbuffers::root; use vortex_array::dtype::DType; use vortex_buffer::ByteBuffer; @@ -18,6 +20,7 @@ use crate::Footer; use crate::MAGIC_BYTES; use crate::VERSION; use crate::footer::FileStatistics; +use crate::footer::SegmentSpec; use crate::footer::postscript::Postscript; use crate::footer::postscript::PostscriptSegment; @@ -165,14 +168,50 @@ impl FooterDeserializer { ) }) .transpose()?; + let metadata = postscript + .metadata + .iter() + .map(|metadata| { + let segment = SegmentSpec { + offset: metadata.segment.offset, + length: metadata.segment.length, + alignment: metadata.segment.alignment, + }; + let end = segment + .offset + .checked_add(u64::from(segment.length)) + .ok_or_else(|| { + vortex_err!("Metadata segment {} range overflowed u64", metadata.key) + })?; + if end > file_size { + vortex_bail!( + "Metadata segment {} range {}..{} exceeds file size {}", + metadata.key, + segment.offset, + end, + file_size + ); + } + let offset = usize::try_from(segment.offset)?; + if !segment.alignment.is_offset_aligned(offset) { + vortex_bail!( + "Metadata segment {} offset {} is not aligned to {}", + metadata.key, + segment.offset, + segment.alignment + ); + } + Ok((metadata.key.clone(), segment)) + }) + .collect::>>()?; Ok(DeserializeStep::Done(self.parse_footer( initial_offset, &self.buffer, - &postscript.footer, - &postscript.layout, + postscript, dtype, file_stats, + metadata, )?)) } @@ -257,22 +296,31 @@ impl FooterDeserializer { &self, initial_offset: u64, initial_read: &[u8], - footer_segment: &PostscriptSegment, - layout_segment: &PostscriptSegment, + postscript: &Postscript, dtype: DType, file_stats: Option, + metadata: Arc<[(String, SegmentSpec)]>, ) -> VortexResult