Add file-level user metadata

docs/specs/file-format.md

@@ -52,6 +52,13 @@ The postscript contains the locations of:
 2. a `layout` segment containing the root `Layout`
 3. a `statistics` segment containing file-level per-field statistics (e.g., minima and maxima of each field/column, for whole-file pruning)
 4. a `footer` segment containing a dictionary-encoded _segment map_, and other shared configuration such as compression and encryption schemes
+5. an optional `metadata` segment holding user-defined file-level metadata (a `FileMetadata` flatbuffer of opaque byte values keyed by unique, non-empty UTF-8 strings)
+
+The postscript carries a single locator (offset, length, and alignment) for the metadata segment
+when the file has one; a file written without user metadata (and any file predating this feature)
+carries none. The segment may live anywhere in the file. Readers do not load the metadata by
+default; an opt-in read resolves the one locator, fetching the segment without reading the
+intervening file contents when it lies outside the initial file-tail read.
 
 :::{literalinclude} ../../vortex-flatbuffers/flatbuffers/vortex-file/footer.fbs
 :start-after: [postscript]

vortex-buffer/src/alignment.rs

@@ -111,13 +111,32 @@ impl Alignment {
             .vortex_expect("alignment is a power of 2 within usize, so its exponent fits in u8")
     }
 
+    /// Try to create an alignment from its log2 exponent.
+    ///
+    /// Returns an error when the exponent cannot be represented by a `usize` alignment.
+    #[inline]
+    pub fn try_from_exponent(exponent: u8) -> Result<Self, VortexError> {
+        let alignment = 1usize.checked_shl(u32::from(exponent)).ok_or_else(|| {
+            vortex_err!(
+                "Alignment exponent {exponent} must be less than {}",
+                usize::BITS
+            )
+        })?;
+        Ok(Self::new(alignment))
+    }
+
     /// Create from the log2 exponent of the alignment.
     ///
     /// ## Panics
     ///
-    /// Panics if `1 << exponent` overflows `usize`.
+    /// Panics if `1 << exponent` overflows `usize`. Use [`Self::try_from_exponent`] when parsing
+    /// untrusted input.
     #[inline]
     pub const fn from_exponent(exponent: u8) -> Self {
+        assert!(
+            (exponent as u32) < usize::BITS,
+            "Alignment exponent must fit in usize"
+        );
         Self::new(1 << exponent)
     }
 }
@@ -214,6 +233,13 @@ mod test {
         assert_eq!(Alignment::from_exponent(10), alignment);
     }
 
+    #[test]
+    fn invalid_alignment_exponent() {
+        let error = Alignment::try_from_exponent(u8::try_from(usize::BITS).unwrap())
+            .expect_err("an exponent as wide as usize must be rejected");
+        assert!(error.to_string().contains("must be less than"));
+    }
+
     #[test]
     fn is_aligned_to() {
         assert!(Alignment::new(1).is_aligned_to(Alignment::new(1)));

vortex-file/src/file.rs

@@ -15,6 +15,7 @@ use vortex_array::ArrayRef;
 use vortex_array::dtype::DType;
 use vortex_array::dtype::FieldMask;
 use vortex_array::expr::Expression;
+use vortex_buffer::ByteBuffer;
 use vortex_error::VortexResult;
 use vortex_layout::LayoutReader;
 use vortex_layout::scan::layout::LayoutReaderDataSource;
@@ -23,6 +24,7 @@ use vortex_layout::scan::split_by::SplitBy;
 use vortex_layout::segments::SegmentSource;
 use vortex_scan::DataSourceRef;
 use vortex_session::VortexSession;
+use vortex_utils::aliases::hash_map::HashMap;
 
 use crate::FileStatistics;
 use crate::footer::Footer;
@@ -42,6 +44,8 @@ pub struct VortexFile {
     segment_source: Arc<dyn SegmentSource>,
     /// The Vortex session used to open this file.
     session: VortexSession,
+    /// User-defined metadata values resolved for this file open.
+    metadata: Arc<HashMap<String, ByteBuffer>>,
     /// None id LayoutReader caching is turned off
     layout_reader_cache: Option<OnceLock<Arc<dyn LayoutReader>>>,
 }
@@ -78,10 +82,16 @@ impl VortexFile {
             footer,
             segment_source,
             session,
+            metadata: Arc::new(HashMap::new()),
             layout_reader_cache: None,
         }
     }
 
+    pub(crate) fn with_metadata(mut self, metadata: Arc<HashMap<String, ByteBuffer>>) -> Self {
+        self.metadata = metadata;
+        self
+    }
+
     /// Enable layout reader caching.
     ///
     /// Repeated calls to [`layout_reader`](Self::layout_reader), [`scan`](Self::scan), and
@@ -91,6 +101,7 @@ impl VortexFile {
             footer: self.footer,
             segment_source: self.segment_source,
             session: self.session,
+            metadata: self.metadata,
             layout_reader_cache: Some(OnceLock::new()),
         }
     }
@@ -117,6 +128,22 @@ impl VortexFile {
         self.footer.statistics()
     }
 
+    /// Returns the user-defined metadata segments loaded for this file.
+    ///
+    /// Metadata is only loaded when requested during open. Iteration order is unspecified.
+    pub fn metadata_segments(&self) -> impl Iterator<Item = (&str, &ByteBuffer)> {
+        self.metadata
+            .iter()
+            .map(|(key, metadata)| (key.as_str(), metadata))
+    }
+
+    /// Returns the loaded user-defined metadata segment for the given key.
+    ///
+    /// Returns `None` when the key is absent or metadata was not loaded.
+    pub fn metadata_segment(&self, key: &str) -> Option<&ByteBuffer> {
+        self.metadata.get(key)
+    }
+
     /// Create a new segment source for reading from the file.
     ///
     /// This may spawn a background I/O driver that will exit when the returned segment source

vortex-file/src/footer/deserializer.rs

@@ -18,6 +18,7 @@ use crate::Footer;
 use crate::MAGIC_BYTES;
 use crate::VERSION;
 use crate::footer::FileStatistics;
+use crate::footer::SegmentSpec;
 use crate::footer::postscript::Postscript;
 use crate::footer::postscript::PostscriptSegment;
 
@@ -165,14 +166,47 @@ impl FooterDeserializer {
                 )
             })
             .transpose()?;
+        // Validate the optional metadata locator here; values are resolved later, on demand. It
+        // never widens the required initial read.
+        let metadata_segment = match &postscript.metadata {
+            None => None,
+            Some(segment) => {
+                let spec = SegmentSpec {
+                    offset: segment.offset,
+                    length: segment.length,
+                    alignment: segment.alignment,
+                };
+                let end = spec
+                    .offset
+                    .checked_add(u64::from(spec.length))
+                    .ok_or_else(|| vortex_err!("Metadata segment range overflowed u64"))?;
+                if end > file_size {
+                    vortex_bail!(
+                        "Metadata segment range {}..{} exceeds file size {}",
+                        spec.offset,
+                        end,
+                        file_size
+                    );
+                }
+                let offset = usize::try_from(spec.offset)?;
+                if !spec.alignment.is_offset_aligned(offset) {
+                    vortex_bail!(
+                        "Metadata segment offset {} is not aligned to {}",
+                        spec.offset,
+                        spec.alignment
+                    );
+                }
+                Some(spec)
+            }
+        };
 
         Ok(DeserializeStep::Done(self.parse_footer(
             initial_offset,
             &self.buffer,
-            &postscript.footer,
-            &postscript.layout,
+            postscript,
             dtype,
             file_stats,
+            metadata_segment,
         )?))
     }
 
@@ -257,22 +291,31 @@ impl FooterDeserializer {
         &self,
         initial_offset: u64,
         initial_read: &[u8],
-        footer_segment: &PostscriptSegment,
-        layout_segment: &PostscriptSegment,
+        postscript: &Postscript,
         dtype: DType,
         file_stats: Option<FileStatistics>,
+        metadata_segment: Option<SegmentSpec>,
     ) -> VortexResult<Footer> {
+        let footer_segment = &postscript.footer;
         let footer_offset = usize::try_from(footer_segment.offset - initial_offset)?;
         let footer_bytes = FlatBuffer::copy_from(
             &initial_read[footer_offset..footer_offset + (footer_segment.length as usize)],
         );
 
+        let layout_segment = &postscript.layout;
         let layout_offset = usize::try_from(layout_segment.offset - initial_offset)?;
         let layout_bytes = FlatBuffer::copy_from(
             &initial_read[layout_offset..layout_offset + (layout_segment.length as usize)],
         );
 
-        Footer::from_flatbuffer(footer_bytes, layout_bytes, dtype, file_stats, &self.session)
+        Footer::from_flatbuffer(
+            footer_bytes,
+            layout_bytes,
+            dtype,
+            file_stats,
+            metadata_segment,
+            &self.session,
+        )
     }
 }
 

vortex-file/src/footer/metadata.rs

@@ -0,0 +1,124 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+//! User-defined file-level metadata: opaque byte values keyed by unique, non-empty strings,
+//! serialized into a single [`fb::FileMetadata`] segment that the postscript locates by offset.
+
+use flatbuffers::FlatBufferBuilder;
+use flatbuffers::WIPOffset;
+use flatbuffers::root;
+use vortex_buffer::ByteBuffer;
+use vortex_error::VortexResult;
+use vortex_error::vortex_bail;
+use vortex_flatbuffers::FlatBufferRoot;
+use vortex_flatbuffers::WriteFlatBuffer;
+use vortex_flatbuffers::footer as fb;
+use vortex_utils::aliases::hash_map::HashMap;
+
+/// User-defined file-level metadata, serialized into a single Vortex file segment.
+pub(crate) struct FileMetadata {
+    /// Entries sorted by key so the serialized bytes are deterministic.
+    entries: Vec<(String, ByteBuffer)>,
+}
+
+impl FileMetadata {
+    /// Build [`FileMetadata`] from a keyed map of opaque values.
+    pub(crate) fn new(metadata: HashMap<String, ByteBuffer>) -> Self {
+        let mut entries: Vec<(String, ByteBuffer)> = metadata.into_iter().collect();
+        entries.sort_unstable_by(|(left, _), (right, _)| left.cmp(right));
+        Self { entries }
+    }
+
+    /// Parse a metadata segment into a keyed map, copying values out so the file bytes aren't pinned.
+    pub(crate) fn parse(bytes: &[u8]) -> VortexResult<HashMap<String, ByteBuffer>> {
+        let fb = root::<fb::FileMetadata>(bytes)?;
+        let mut map = HashMap::default();
+        if let Some(entries) = fb.entries() {
+            for entry in entries.iter() {
+                let key = entry.key();
+                if key.is_empty() {
+                    vortex_bail!("File metadata contains an empty key");
+                }
+                let value = ByteBuffer::copy_from(entry.value().bytes());
+                if map.insert(key.to_string(), value).is_some() {
+                    vortex_bail!("File metadata contains duplicate key {key}");
+                }
+            }
+        }
+        Ok(map)
+    }
+}
+
+impl FlatBufferRoot for FileMetadata {}
+
+impl WriteFlatBuffer for FileMetadata {
+    type Target<'a> = fb::FileMetadata<'a>;
+
+    fn write_flatbuffer<'fb>(
+        &self,
+        fbb: &mut FlatBufferBuilder<'fb>,
+    ) -> VortexResult<WIPOffset<Self::Target<'fb>>> {
+        let entries = self
+            .entries
+            .iter()
+            .map(|(key, value)| {
+                let key = fbb.create_string(key);
+                let value = fbb.create_vector(value.as_slice());
+                fb::MetadataEntry::create(
+                    fbb,
+                    &fb::MetadataEntryArgs {
+                        key: Some(key),
+                        value: Some(value),
+                    },
+                )
+            })
+            .collect::<Vec<_>>();
+        let entries = fbb.create_vector(entries.as_slice());
+        Ok(fb::FileMetadata::create(
+            fbb,
+            &fb::FileMetadataArgs {
+                entries: Some(entries),
+            },
+        ))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use vortex_flatbuffers::WriteFlatBufferExt;
+
+    use super::*;
+
+    fn metadata<const N: usize>(entries: [(&str, &[u8]); N]) -> FileMetadata {
+        FileMetadata::new(
+            entries
+                .into_iter()
+                .map(|(key, value)| (key.to_string(), ByteBuffer::copy_from(value)))
+                .collect(),
+        )
+    }
+
+    #[test]
+    fn roundtrip_is_deterministic() -> VortexResult<()> {
+        // Insertion order must not affect the serialized bytes (entries are sorted by key).
+        let forward =
+            metadata([("a", b"alpha"), ("b", b""), ("c", b"gamma")]).write_flatbuffer_bytes()?;
+        let reverse =
+            metadata([("c", b"gamma"), ("b", b""), ("a", b"alpha")]).write_flatbuffer_bytes()?;
+        assert_eq!(forward.as_slice(), reverse.as_slice());
+
+        let parsed = FileMetadata::parse(&forward)?;
+        assert_eq!(parsed.len(), 3);
+        assert_eq!(parsed["a"].as_slice(), b"alpha");
+        assert!(parsed["b"].is_empty());
+        Ok(())
+    }
+
+    #[test]
+    fn parse_rejects_empty_key() -> VortexResult<()> {
+        let bytes = metadata([("", b"value")]).write_flatbuffer_bytes()?;
+        let error = FileMetadata::parse(&bytes).expect_err("empty key must be rejected");
+        assert!(error.to_string().contains("empty key"));
+        Ok(())
+    }
+}