diff --git a/Cargo.lock b/Cargo.lock index c24f1fa07..351908ec9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1195,8 +1195,11 @@ version = "0.1.13" dependencies = [ "base64 0.22.1", "bigdecimal", + "flate2", + "md-5 0.10.6", "serde", "serde_json", + "sha2 0.10.9", "thiserror", "time", "uuid", diff --git a/Cargo.toml b/Cargo.toml index 36cd0c1c7..9f5a8639d 100755 --- a/Cargo.toml +++ b/Cargo.toml @@ -74,6 +74,10 @@ dashmap = "6" # Crypto & checksums crc32fast = "1" +# Pinned to the versions already resolved in Cargo.lock. +flate2 = "1.1.9" +md-5 = "0.10.6" +sha2 = "0.10.9" uuid = { version = "1", features = ["v4"] } base64 = "0.22" bigdecimal = { version = "0.4", features = ["serde"] } diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 971153844..7e04256bf 100755 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -13,5 +13,8 @@ serde_json = { workspace = true } thiserror = { workspace = true } base64 = { workspace = true } bigdecimal = { workspace = true } +flate2 = { workspace = true } +md-5 = { workspace = true } +sha2 = { workspace = true } time = { workspace = true } uuid = { workspace = true } diff --git a/crates/core/src/backup_format/data_file.rs b/crates/core/src/backup_format/data_file.rs new file mode 100644 index 000000000..4b114916b --- /dev/null +++ b/crates/core/src/backup_format/data_file.rs @@ -0,0 +1,1480 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Data file encoding and decoding. +//! +//! A data file is gzip-compressed newline-delimited JSON, one +//! `{"Item": }` object per line, the encoding Amazon +//! `DynamoDB` writes for `DYNAMODB_JSON` exports and the same line encoding +//! the engine's `ExportTableToPointInTime` handler produces (a +//! `serde_json::json!({"Item": item})` wrapper serialized compactly, then a +//! newline), so the two agree byte for byte. +//! +//! Checksums: `sha256` and `md5` are computed over the compressed bytes, the +//! object as stored, which is what `extenddb-manifest.json` records per data +//! file and what `manifest-files.json` records as `md5Checksum`. Item and +//! uncompressed byte counts are computed over the line stream. +//! +//! Memory: [`DataFileEncoder`] writes through any [`Write`] and hashes the +//! compressed bytes as they leave the compressor, so a caller streaming to a +//! file or an upload holds the gzip window and one serialized line, not the +//! file. [`DataFileDecoder`] reads from any [`Read`], hashes the compressed +//! bytes as they are consumed, and checks the digests when the stream ends. +//! A caller that must not act on any item before the digests are known (a +//! restore) runs the bytes through [`CompressedHasher`] first, keeping a copy +//! on local disk, and decodes from that copy; [`DataFileDecoder::verified`] +//! does both steps for a slice already in memory. + +use std::io::{BufRead, BufReader, Read, Write}; + +use base64::Engine; +use base64::engine::general_purpose::STANDARD as BASE64; +use flate2::Compression; +use flate2::read::MultiGzDecoder; +use flate2::write::GzEncoder; +use md5::Md5; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +use super::FormatError; +use crate::types::Item; + +/// Maximum length of a single decompressed data-file line, in bytes. +/// +/// One line holds one item wrapped as `{"Item": }` plus a +/// trailing newline. A `DynamoDB` item is capped at 400 KB. The `DYNAMODB_JSON` +/// wire form inflates that: every binary (`B`, `BS`) value is base64 (about +/// 4/3), attribute names and the `{"S":...}` / `{"N":...}` type tags repeat per +/// value, and the `{"Item":...}` wrapper adds a fixed frame. Four megabytes is +/// an order of magnitude above the worst realistic expansion of a 400 KB item, +/// so it never rejects a legitimate line, while it bounds the memory a single +/// `next_item` call can allocate from a malicious or corrupt gzip stream (a +/// gzip member can expand about 1030 to 1, so an unbounded read of a small +/// object can otherwise allocate gigabytes). A line longer than the cap +/// (newline included) yields [`FormatError::LineTooLong`] before the whole +/// line is materialized; a line of exactly the cap is accepted. +/// +/// A data file has no blank lines: every line is one item. A blank line is +/// a [`FormatError::Json`] like any other unparseable line. +pub const MAX_LINE_BYTES: usize = 4 * 1024 * 1024; + +/// Lowercase hex encoding of a byte slice. +pub(super) fn hex_encode(bytes: &[u8]) -> String { + use std::fmt::Write as _; + let mut out = String::with_capacity(bytes.len() * 2); + for byte in bytes { + let _ = write!(out, "{byte:02x}"); + } + out +} + +/// Strip a single trailing `\n` and an optional preceding `\r` from a line. +fn strip_trailing_newline(line: &[u8]) -> &[u8] { + let line = line.strip_suffix(b"\n").unwrap_or(line); + line.strip_suffix(b"\r").unwrap_or(line) +} + +/// The single-key wrapper each data file line holds. +#[derive(Deserialize)] +struct ItemLine { + #[serde(rename = "Item")] + item: Item, +} + +/// Borrowing form of [`ItemLine`] for the writer, so an item is serialized in +/// place rather than cloned into a `serde_json::Value`. +#[derive(Serialize)] +struct ItemLineRef<'a> { + #[serde(rename = "Item")] + item: &'a Item, +} + +/// Checksums and counts produced while encoding a data file. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DataFileChecksums { + /// Number of items encoded. + pub item_count: u64, + /// Lowercase hex sha256 of the compressed bytes. + pub sha256_hex: String, + /// Base64 MD5 of the compressed bytes, the `md5Checksum` encoding + /// `manifest-files.json` uses. + pub md5_base64: String, + /// Size of the compressed output. + pub compressed_bytes: u64, + /// Total size of the uncompressed lines, newlines included. + pub uncompressed_bytes: u64, +} + +/// What a decoder should verify. Only the members that are `Some` are +/// checked, so a caller holding just the `md5Checksum` from a +/// `manifest-files.json` line can still verify what it has. +#[derive(Debug, Clone, Default)] +pub struct DataFileExpectations { + /// Expected lowercase hex sha256 of the compressed bytes. + pub sha256_hex: Option, + /// Expected base64 MD5 of the compressed bytes. + pub md5_base64: Option, + /// Expected item count. + pub item_count: Option, + /// Expected uncompressed byte count. + pub uncompressed_bytes: Option, +} + +impl From<&DataFileChecksums> for DataFileExpectations { + fn from(checksums: &DataFileChecksums) -> Self { + Self { + sha256_hex: Some(checksums.sha256_hex.clone()), + md5_base64: Some(checksums.md5_base64.clone()), + item_count: Some(checksums.item_count), + uncompressed_bytes: Some(checksums.uncompressed_bytes), + } + } +} + +/// Digest of a data file's compressed bytes: what the manifests record and +/// what a reader verifies before trusting the file's contents. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CompressedDigest { + /// Lowercase hex sha256 of the compressed bytes. + pub sha256_hex: String, + /// Base64 MD5 of the compressed bytes, the `md5Checksum` encoding + /// `manifest-files.json` uses. + pub md5_base64: String, + /// Number of compressed bytes. + pub bytes: u64, +} + +/// Incremental hasher over compressed bytes. Feed it the bytes of a data +/// file as they arrive (from a download, a file read, or the compressor) and +/// call [`CompressedHasher::finish`] for the digest. +#[derive(Default)] +pub struct CompressedHasher { + sha256: Sha256, + md5: Md5, + bytes: u64, +} + +impl CompressedHasher { + /// A hasher with nothing fed yet. + #[must_use] + pub fn new() -> Self { + Self::default() + } + + /// Feed bytes. + pub fn update(&mut self, bytes: &[u8]) { + self.sha256.update(bytes); + self.md5.update(bytes); + self.bytes += bytes.len() as u64; + } + + /// Finish and return the digest. + #[must_use] + pub fn finish(self) -> CompressedDigest { + CompressedDigest { + sha256_hex: hex_encode(&self.sha256.finalize()), + md5_base64: BASE64.encode(self.md5.finalize()), + bytes: self.bytes, + } + } + + /// Digest of a slice, in one call. + #[must_use] + pub fn digest(bytes: &[u8]) -> CompressedDigest { + let mut hasher = Self::new(); + hasher.update(bytes); + hasher.finish() + } +} + +impl DataFileExpectations { + /// Check the digest fields that are set against a computed digest. + /// + /// # Errors + /// + /// Returns [`FormatError::ChecksumMismatch`] naming the first digest that + /// differs. + pub fn verify_digest(&self, digest: &CompressedDigest) -> Result<(), FormatError> { + if let Some(expected) = &self.sha256_hex + && expected != &digest.sha256_hex + { + return Err(FormatError::ChecksumMismatch { + subject: "data file sha256", + expected: expected.clone(), + computed: digest.sha256_hex.clone(), + }); + } + if let Some(expected) = &self.md5_base64 + && expected != &digest.md5_base64 + { + return Err(FormatError::ChecksumMismatch { + subject: "data file md5", + expected: expected.clone(), + computed: digest.md5_base64.clone(), + }); + } + Ok(()) + } +} + +/// A [`Write`] that hashes everything written through it before passing it +/// on. The compressor writes into this, so the digest covers the bytes as +/// stored. +/// +/// `inner` is taken out by [`DataFileEncoder::into_inner`], and by the +/// encoder's `Drop` when the encoder is poisoned; the compressor's own `Drop` +/// then writes into nothing and its `BrokenPipe` is ignored, which is the +/// documented flate2 behaviour for a dropped encoder. +struct HashingWriter { + inner: Option, + hasher: CompressedHasher, +} + +impl Write for HashingWriter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + let Some(inner) = self.inner.as_mut() else { + return Err(std::io::Error::new( + std::io::ErrorKind::BrokenPipe, + "sink already taken", + )); + }; + let written = inner.write(buf)?; + // `Write::write` may return `Ok(0)` for a sink that accepts no more + // bytes. flate2's header and trailer writers loop until their bytes + // are accepted, so passing `Ok(0)` through would spin forever; std's + // `write_all` turns it into `WriteZero` for the same reason. + if written == 0 && !buf.is_empty() { + return Err(std::io::Error::new( + std::io::ErrorKind::WriteZero, + "sink accepted no bytes", + )); + } + self.hasher.update(&buf[..written]); + Ok(written) + } + + fn flush(&mut self) -> std::io::Result<()> { + match self.inner.as_mut() { + Some(inner) => inner.flush(), + None => Ok(()), + } + } +} + +/// A [`Read`] that hashes everything read through it. The decompressor reads +/// from this, so the digest covers exactly the compressed bytes consumed. +struct HashingReader { + inner: R, + hasher: CompressedHasher, +} + +impl Read for HashingReader { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + let read = self.inner.read(buf)?; + self.hasher.update(&buf[..read]); + Ok(read) + } +} + +/// Streaming encoder: serializes items as `{"Item": ...}` lines into a gzip +/// stream written to `W`, hashing the compressed output as it goes. +/// +/// The gzip level is fixed at [`GZIP_LEVEL`] so two runs of the same build +/// encoding the same items produce the same bytes and checksums (compression +/// is not part of the format contract; the output also depends on the flate2 +/// backend and version, which `Cargo.lock` pins). +/// +/// A write error poisons the encoder: part of a line may have entered the +/// compressor, so the stream is damaged and no later write or `finish` can +/// produce a valid file. Every later call returns [`FormatError::Encode`]; +/// the caller discards the file and starts it again from its first item. +/// Dropping a poisoned encoder writes nothing more into the sink: the sink +/// is taken out first, so the compressor's own `Drop` has nowhere to flush +/// its buffered data and trailer. A caller whose sink needs cleanup (an +/// upload to abort) takes it back with [`Self::into_inner`] before dropping. +pub struct DataFileEncoder { + encoder: GzEncoder>, + item_count: u64, + uncompressed_bytes: u64, + state: EncoderState, +} + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum EncoderState { + /// Accepting items. + Open, + /// `try_finish` succeeded; the trailer is written and no item may follow. + Finished, + /// A write or finish failed; the stream is damaged. + Poisoned, +} + +/// gzip compression level the encoder uses: flate2's default, pinned so two +/// runs of one build produce identical bytes for identical items. Measured +/// once on a 16-core Xeon with 1M synthetic 318-byte `DynamoDB` JSON items +/// (318 MB, localhost, single thread) using the `gzip` CLI (zlib), not the +/// `miniz_oxide` backend flate2 uses here, so the absolute figures transfer +/// only approximately: level 6 ran at about 50 MB/s with a 21% output ratio; +/// level 1 was 1.6x faster at 25%; level 9 was 4x slower for 20%. Ratios +/// depend on the data; the level choice depends only on the ratios between +/// levels. +pub const GZIP_LEVEL: u32 = 6; + +const POISONED: &str = "encoder poisoned by an earlier write error; discard the file"; +const FINISHED: &str = "encoder already finished; no item may follow the trailer"; + +impl DataFileEncoder { + /// Start a data file written to `sink`. + #[must_use] + pub fn new(sink: W) -> Self { + Self { + encoder: GzEncoder::new( + HashingWriter { + inner: Some(sink), + hasher: CompressedHasher::new(), + }, + Compression::new(GZIP_LEVEL), + ), + item_count: 0, + uncompressed_bytes: 0, + state: EncoderState::Open, + } + } + + fn check_open(&self) -> Result<(), FormatError> { + match self.state { + EncoderState::Open => Ok(()), + EncoderState::Finished => Err(FormatError::Encode(FINISHED.to_owned())), + EncoderState::Poisoned => Err(FormatError::Encode(POISONED.to_owned())), + } + } + + /// Append one item as a line. + /// + /// # Errors + /// + /// Returns [`FormatError::Json`] when the item fails to serialize (nothing + /// has been written; the encoder stays usable) and [`FormatError::Encode`] + /// when the compressor or sink fails, after which the encoder is poisoned, + /// or when the encoder is already finished or poisoned. + pub fn write_item(&mut self, item: &Item) -> Result<(), FormatError> { + self.check_open()?; + // The exact line encoding the engine's export handler produces. + let mut line = serde_json::to_vec(&ItemLineRef { item })?; + line.push(b'\n'); + if let Err(e) = self.encoder.write_all(&line) { + self.state = EncoderState::Poisoned; + return Err(FormatError::Encode(e.to_string())); + } + self.item_count += 1; + self.uncompressed_bytes += line.len() as u64; + Ok(()) + } + + /// Whether a write or finish error has made this file unusable. False + /// after a successful [`Self::try_finish`]; see [`Self::is_finished`]. + #[must_use] + pub fn is_poisoned(&self) -> bool { + self.state == EncoderState::Poisoned + } + + /// Whether [`Self::try_finish`] has succeeded, so the file is complete + /// and accepts no more items. + #[must_use] + pub fn is_finished(&self) -> bool { + self.state == EncoderState::Finished + } + + /// Number of items written so far. + #[must_use] + pub fn item_count(&self) -> u64 { + self.item_count + } + + /// Uncompressed bytes written so far, newlines included. A writer + /// targeting a per-file size compares this against its limit. + #[must_use] + pub fn uncompressed_bytes(&self) -> u64 { + self.uncompressed_bytes + } + + /// Finish the gzip stream and flush the sink, keeping the encoder so the + /// sink can be recovered with [`Self::into_inner`] on failure (an upload + /// handle that must be aborted, for example). Returns the checksums a + /// manifest records. After a successful `try_finish` the encoder accepts + /// no more items. + /// + /// # Errors + /// + /// Returns [`FormatError::Encode`] when the compressor or sink fails, or + /// when the encoder is already finished or poisoned. The encoder is + /// poisoned after a failure. + pub fn try_finish(&mut self) -> Result { + self.check_open()?; + let result = self + .encoder + .try_finish() + .and_then(|()| self.encoder.get_mut().flush()); + if let Err(e) = result { + self.state = EncoderState::Poisoned; + return Err(FormatError::Encode(e.to_string())); + } + // No item may follow the trailer. + self.state = EncoderState::Finished; + let digest = std::mem::take(&mut self.encoder.get_mut().hasher).finish(); + Ok(DataFileChecksums { + item_count: self.item_count, + sha256_hex: digest.sha256_hex, + md5_base64: digest.md5_base64, + compressed_bytes: digest.bytes, + uncompressed_bytes: self.uncompressed_bytes, + }) + } + + /// Give back the sink, finished or not. After a failed `write_item` or + /// `try_finish` the sink holds a partial file the caller must discard. + /// + /// # Panics + /// + /// Never: the sink is present until this call consumes the encoder. + #[must_use] + pub fn into_inner(mut self) -> W { + self.encoder + .get_mut() + .inner + .take() + .expect("the sink is taken exactly once, here") + } + + /// [`Self::try_finish`] then [`Self::into_inner`], for callers whose sink + /// needs no cleanup on failure (a `Vec`, a file removed by path). + /// + /// # Errors + /// + /// Returns [`FormatError::Encode`] when the compressor or sink fails; the + /// sink is dropped without being written to again. + pub fn finish(mut self) -> Result<(W, DataFileChecksums), FormatError> { + let checksums = self.try_finish()?; + Ok((self.into_inner(), checksums)) + } +} + +impl Drop for DataFileEncoder { + /// A poisoned encoder gives the compressor nothing to write into, so its + /// `Drop` cannot push the buffered data and trailer of a damaged stream + /// into a sink the caller has already decided to abandon. An open or + /// finished encoder is left to flate2's default behaviour. + fn drop(&mut self) { + if self.state == EncoderState::Poisoned { + drop(self.encoder.get_mut().inner.take()); + } + } +} + +/// Encode items into an in-memory gzip data file, returning the compressed +/// bytes and the checksums a manifest records for them. Convenience over +/// [`DataFileEncoder`] for callers that hold a whole file anyway (tests, small +/// files); a backup job streams through the encoder instead. +/// +/// # Errors +/// +/// Returns [`FormatError::Json`] when an item fails to serialize and +/// [`FormatError::Encode`] when compression fails. +pub fn encode_data_file(items: I) -> Result<(Vec, DataFileChecksums), FormatError> +where + I: IntoIterator, +{ + let mut encoder = DataFileEncoder::new(Vec::new()); + for item in items { + encoder.write_item(&item)?; + } + encoder.finish() +} + +/// Streaming decoder for a data file read from any [`Read`]. +/// +/// Items stream back one line at a time. The compressed bytes are hashed as +/// they are consumed and the digests, item count and uncompressed byte count +/// are checked when the stream ends, so the last `next_item` call returns the +/// verification result. Until then every item is unverified: a caller that +/// needs the digests checked before it acts on any item uses +/// [`CompressedHasher`] over a local copy first, or [`Self::verified`] when +/// the file is already in memory. +/// +/// Items are checked for JSON shape only. An `N` value that is not a valid +/// `DynamoDB` number (`"abc"`, `"1E999"`, `"1_000"`) decodes as written; a +/// restore runs each item through the service's validation layer before +/// writing it, the same path `PutItem` takes. +/// +/// Any error ends the stream: every later `next_item` returns +/// [`FormatError::Decode`] naming the earlier failure, never `Ok(None)`, so a +/// consumer that drops errors cannot mistake a corrupt file for a short one. +/// The [`Iterator`] impl is fused on top of that: it yields the first error +/// once and then `None`, so error-dropping iterator idioms terminate. +pub struct DataFileDecoder { + reader: BufReader>>, + expectations: DataFileExpectations, + items_decoded: u64, + bytes_decoded: u64, + finished: bool, + /// Set by the first error; replayed by every later call. + failed: Option<&'static str>, +} + +impl<'a> DataFileDecoder<&'a [u8]> { + /// Verify the compressed-byte checksums against an in-memory file, then + /// open the stream over it. The digests are checked here, before any item + /// is returned; the end-of-stream checks still run. + /// + /// # Errors + /// + /// Returns [`FormatError::ChecksumMismatch`] when a checksum in + /// `expectations` does not match `compressed`. + pub fn verified( + compressed: &'a [u8], + expectations: DataFileExpectations, + ) -> Result { + expectations.verify_digest(&CompressedHasher::digest(compressed))?; + Ok(Self::new(compressed, expectations)) + } +} + +impl DataFileDecoder { + /// Open a stream over `compressed`. Nothing is verified until the stream + /// ends; see the type docs. + pub fn new(compressed: R, expectations: DataFileExpectations) -> Self { + Self { + reader: BufReader::new(MultiGzDecoder::new(HashingReader { + inner: compressed, + hasher: CompressedHasher::new(), + })), + expectations, + items_decoded: 0, + bytes_decoded: 0, + finished: false, + failed: None, + } + } + + /// Decode the next item, or run the end-of-stream checks and return + /// `Ok(None)`. + /// + /// The line is read through a bounded loop that refuses to materialize + /// more than [`MAX_LINE_BYTES`], so a malicious or corrupt gzip member + /// cannot drive unbounded memory. When + /// [`DataFileExpectations::uncompressed_bytes`] is set it is also enforced + /// incrementally: the running total is checked against the expectation as + /// each line is consumed, so an oversized stream is rejected before the + /// whole of it is read rather than only at clean end of stream. + /// + /// # Errors + /// + /// Returns [`FormatError::Decode`] when decompression fails, + /// [`FormatError::LineTooLong`] when a single line reaches the cap, + /// [`FormatError::Json`] when a line does not parse, and at end of stream + /// [`FormatError::ChecksumMismatch`], [`FormatError::ItemCountMismatch`] + /// or [`FormatError::ByteCountMismatch`] when the consumed bytes differ + /// from the expectations (the byte count is also enforced mid-stream once + /// it is exceeded). + pub fn next_item(&mut self) -> Result, FormatError> { + if let Some(reason) = self.failed { + return Err(FormatError::Decode(format!( + "decoder failed earlier: {reason}" + ))); + } + if self.finished { + return Ok(None); + } + let result = self.next_item_inner(); + if result.is_err() { + self.finished = true; + self.failed = Some(match &result { + Err(FormatError::ChecksumMismatch { .. }) => "checksum mismatch", + Err(FormatError::ItemCountMismatch { .. }) => "item count mismatch", + Err(FormatError::ByteCountMismatch { .. }) => "byte count mismatch", + Err(FormatError::LineTooLong { .. }) => "line too long", + Err(FormatError::Json(_)) => "unparseable line", + _ => "decode error", + }); + } + result + } + + fn next_item_inner(&mut self) -> Result, FormatError> { + let mut line: Vec = Vec::new(); + let read = self.read_capped_line(&mut line)?; + if read == 0 { + self.finished = true; + return self.finish_checks().map(|()| None); + } + self.bytes_decoded += read as u64; + // Enforce the recorded uncompressed size incrementally: once the + // running total passes the expectation the stream is oversized, so + // reject it here rather than continuing to read. + if let Some(expected) = self.expectations.uncompressed_bytes + && self.bytes_decoded > expected + { + return Err(FormatError::ByteCountMismatch { + expected, + actual: self.bytes_decoded, + }); + } + let text = std::str::from_utf8(strip_trailing_newline(&line)) + .map_err(|e| FormatError::Decode(e.to_string()))?; + let parsed: ItemLine = serde_json::from_str(text)?; + self.items_decoded += 1; + Ok(Some(parsed.item)) + } + + /// End-of-stream verification: digests first, then counts. By the time + /// the decompressor reports end of stream it has consumed every byte of + /// the source (`MultiGzDecoder` parses any trailing bytes as a further + /// member and fails on garbage), so the digest covers the whole object. + fn finish_checks(&mut self) -> Result<(), FormatError> { + let hashing = self.reader.get_mut().get_mut(); + let digest = std::mem::take(&mut hashing.hasher).finish(); + self.expectations.verify_digest(&digest)?; + if let Some(expected) = self.expectations.item_count + && expected != self.items_decoded + { + return Err(FormatError::ItemCountMismatch { + expected, + actual: self.items_decoded, + }); + } + if let Some(expected) = self.expectations.uncompressed_bytes + && expected != self.bytes_decoded + { + return Err(FormatError::ByteCountMismatch { + expected, + actual: self.bytes_decoded, + }); + } + Ok(()) + } + + /// Read one newline-terminated line into `line`, appending at most + /// [`MAX_LINE_BYTES`] bytes (newline included) before erroring. + /// + /// Returns the number of bytes read, `0` at end of stream. The read walks + /// the `BufReader`'s own buffer with `fill_buf` / `consume`, so peak + /// allocation is the growing line (capped) plus the reader's fixed + /// buffer, never the whole compressed member expanded at once. + fn read_capped_line(&mut self, line: &mut Vec) -> Result { + loop { + let available = self + .reader + .fill_buf() + .map_err(|e| FormatError::Decode(e.to_string()))?; + if available.is_empty() { + return Ok(line.len()); + } + if let Some(nl) = available.iter().position(|&b| b == b'\n') { + let want = nl + 1; + if line.len() + want > MAX_LINE_BYTES { + return Err(FormatError::LineTooLong { + cap: MAX_LINE_BYTES, + }); + } + line.extend_from_slice(&available[..want]); + self.reader.consume(want); + return Ok(line.len()); + } + let take = available.len(); + if line.len() + take > MAX_LINE_BYTES { + return Err(FormatError::LineTooLong { + cap: MAX_LINE_BYTES, + }); + } + line.extend_from_slice(available); + self.reader.consume(take); + } + } +} + +impl Iterator for DataFileDecoder { + type Item = Result; + + /// Fused: once an error has been yielded, `None`. `next_item` keeps + /// returning the error for callers that use it directly. + fn next(&mut self) -> Option { + if self.failed.is_some() { + return None; + } + self.next_item().transpose() + } +} + +impl std::iter::FusedIterator for DataFileDecoder {} + +/// Decode a whole in-memory data file into a `Vec`, verifying the digests +/// before the first item and the counts at the end. For tests and files known +/// to be small; a restore streams through [`DataFileDecoder`] instead and +/// never holds a file's items in memory. +/// +/// # Errors +/// +/// Propagates every error [`DataFileDecoder`] can produce. +pub fn decode_data_file( + compressed: &[u8], + expectations: DataFileExpectations, +) -> Result, FormatError> { + let mut decoder = DataFileDecoder::verified(compressed, expectations)?; + let mut items = Vec::new(); + while let Some(item) = decoder.next_item()? { + items.push(item); + } + Ok(items) +} + +#[cfg(test)] +mod tests { + use std::collections::{BTreeMap, BTreeSet}; + + use super::*; + use crate::types::AttributeValue; + + fn item(entries: Vec<(&str, AttributeValue)>) -> Item { + entries + .into_iter() + .map(|(k, v)| (k.to_owned(), v)) + .collect() + } + + fn string_set(values: &[&str]) -> BTreeSet { + values.iter().map(|s| (*s).to_owned()).collect() + } + + fn byte_set(values: &[&[u8]]) -> BTreeSet> { + values.iter().map(|b| b.to_vec()).collect() + } + + /// A map nested to the given depth, ending in a string leaf. + fn nested_map(depth: usize) -> AttributeValue { + let mut value = AttributeValue::S("leaf".to_owned()); + for level in (0..depth).rev() { + let mut map = BTreeMap::new(); + map.insert(format!("level{level}"), value); + value = AttributeValue::M(map); + } + value + } + + /// Items covering every `DynamoDB` attribute type, with numbers at the + /// 38-significant-digit maximum and in negative form. Number strings are + /// written in the normalized form the `N` deserializer produces, so a + /// round trip compares equal. + fn all_types_items() -> Vec { + vec![ + item(vec![ + ("pk", AttributeValue::S("item-1".to_owned())), + ( + "n_38_digits", + AttributeValue::N("99999999999999999999999999999999999999".to_owned()), + ), + ( + "n_negative", + AttributeValue::N("-12345678901234567890.123456789".to_owned()), + ), + ( + "n_large_magnitude", + AttributeValue::N(format!("1{}", "0".repeat(100))), + ), + ("n_small", AttributeValue::N("0.001".to_owned())), + ("b", AttributeValue::B(vec![0xde, 0xad, 0xbe, 0xef])), + ("bool_true", AttributeValue::Bool(true)), + ("null", AttributeValue::Null), + ]), + item(vec![ + ("pk", AttributeValue::S("item-2".to_owned())), + ("ss", AttributeValue::SS(string_set(&["a", "b", "c"]))), + ( + "ns", + AttributeValue::NS( + ["-1", "0.5", "42"] + .iter() + .map(|s| (*s).to_owned()) + .collect(), + ), + ), + ( + "bs", + AttributeValue::BS(byte_set(&[b"one".as_slice(), b"two".as_slice()])), + ), + ( + "l", + AttributeValue::L(vec![ + AttributeValue::N("1".to_owned()), + AttributeValue::S("x".to_owned()), + AttributeValue::Bool(false), + AttributeValue::Null, + AttributeValue::L(vec![]), + ]), + ), + ("m_depth_32", nested_map(32)), + ]), + ] + } + + #[test] + fn encode_decode_round_trip_all_types() { + let items = all_types_items(); + let (compressed, checksums) = encode_data_file(items.clone()).unwrap(); + assert_eq!(checksums.item_count, 2); + assert_eq!(checksums.compressed_bytes, compressed.len() as u64); + assert!(checksums.uncompressed_bytes > checksums.compressed_bytes / 4); + + let decoded = decode_data_file(&compressed, (&checksums).into()).unwrap(); + assert_eq!(decoded, items); + } + + #[test] + fn exponent_form_numbers_decode_to_normalized_values() { + // Exponent and shorthand forms are valid on the wire; the N + // deserializer normalizes them, so a file holding them decodes to the + // plain forms. + let line = concat!( + r#"{"Item":{"e_neg":{"N":"-2.5e-2"},"e_pos":{"N":"1.5E3"},"#, + r#""pk":{"S":"exp"}}}"#, + "\n", + ); + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(line.as_bytes()).unwrap(); + let compressed = encoder.finish().unwrap(); + + let decoded = decode_data_file(&compressed, DataFileExpectations::default()).unwrap(); + assert_eq!(decoded.len(), 1); + assert_eq!( + decoded[0].get("e_pos"), + Some(&AttributeValue::N("1500".to_owned())) + ); + assert_eq!( + decoded[0].get("e_neg"), + Some(&AttributeValue::N("-0.025".to_owned())) + ); + } + + #[test] + fn line_encoding_matches_export_handler_shape() { + // One item, then decompress and compare against the exact string the + // engine's export handler would write for it. + let one = item(vec![ + ("a", AttributeValue::N("1".to_owned())), + ("pk", AttributeValue::S("x".to_owned())), + ]); + let (compressed, _) = encode_data_file(vec![one.clone()]).unwrap(); + + let mut reader = BufReader::new(MultiGzDecoder::new(compressed.as_slice())); + let mut text = String::new(); + std::io::Read::read_to_string(&mut reader, &mut text).unwrap(); + + let wrapper = serde_json::json!({ "Item": one }); + let expected = format!("{}\n", serde_json::to_string(&wrapper).unwrap()); + assert_eq!(text, expected); + assert_eq!( + text, + "{\"Item\":{\"a\":{\"N\":\"1\"},\"pk\":{\"S\":\"x\"}}}\n" + ); + } + + #[test] + fn empty_file_round_trip() { + let (compressed, checksums) = encode_data_file(Vec::::new()).unwrap(); + assert_eq!(checksums.item_count, 0); + assert_eq!(checksums.uncompressed_bytes, 0); + let decoded = decode_data_file(&compressed, (&checksums).into()).unwrap(); + assert!(decoded.is_empty()); + } + + #[test] + fn tampered_compressed_bytes_fail_sha256() { + let (mut compressed, checksums) = encode_data_file(all_types_items()).unwrap(); + let last = compressed.len() - 1; + compressed[last] ^= 0xff; + let err = decode_data_file(&compressed, (&checksums).into()).unwrap_err(); + assert!(matches!( + err, + FormatError::ChecksumMismatch { + subject: "data file sha256", + .. + } + )); + } + + #[test] + fn tampered_compressed_bytes_fail_md5_when_only_md5_given() { + let (mut compressed, checksums) = encode_data_file(all_types_items()).unwrap(); + compressed[0] ^= 0x01; + let expectations = DataFileExpectations { + md5_base64: Some(checksums.md5_base64), + ..Default::default() + }; + let err = decode_data_file(&compressed, expectations).unwrap_err(); + assert!(matches!( + err, + FormatError::ChecksumMismatch { + subject: "data file md5", + .. + } + )); + } + + #[test] + fn wrong_item_count_is_detected() { + let (compressed, checksums) = encode_data_file(all_types_items()).unwrap(); + let expectations = DataFileExpectations { + item_count: Some(checksums.item_count + 1), + ..Default::default() + }; + let err = decode_data_file(&compressed, expectations).unwrap_err(); + assert!(matches!( + err, + FormatError::ItemCountMismatch { + expected: 3, + actual: 2 + } + )); + } + + #[test] + fn wrong_uncompressed_byte_count_is_detected() { + let (compressed, checksums) = encode_data_file(all_types_items()).unwrap(); + let expectations = DataFileExpectations { + uncompressed_bytes: Some(checksums.uncompressed_bytes + 1), + ..Default::default() + }; + let err = decode_data_file(&compressed, expectations).unwrap_err(); + assert!(matches!(err, FormatError::ByteCountMismatch { .. })); + } + + #[test] + fn decoder_streams_via_iterator() { + let items = all_types_items(); + let (compressed, checksums) = encode_data_file(items.clone()).unwrap(); + let decoder = DataFileDecoder::verified(&compressed, (&checksums).into()).unwrap(); + let streamed: Result, FormatError> = decoder.collect(); + assert_eq!(streamed.unwrap(), items); + } + + #[test] + fn truncated_gzip_stream_is_an_error() { + let (compressed, _) = encode_data_file(all_types_items()).unwrap(); + let truncated = &compressed[..compressed.len() / 2]; + let err = decode_data_file(truncated, DataFileExpectations::default()).unwrap_err(); + assert!(matches!(err, FormatError::Decode(_))); + } + + #[test] + fn malformed_line_is_an_error() { + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(b"{\"NotItem\":{}}\n").unwrap(); + let compressed = encoder.finish().unwrap(); + let err = decode_data_file(&compressed, DataFileExpectations::default()).unwrap_err(); + assert!(matches!(err, FormatError::Json(_))); + } + + #[test] + fn md5_checksum_encoding_matches_manifest_files_shape() { + // 16 MD5 bytes encode to 24 base64 characters with padding, the + // shape manifest-files.json carries. + let (_, checksums) = encode_data_file(all_types_items()).unwrap(); + assert_eq!(checksums.md5_base64.len(), 24); + assert!(checksums.md5_base64.ends_with("==")); + assert_eq!(checksums.sha256_hex.len(), 64); + } + + /// Read the process high-water resident set size in kibibytes from + /// `/proc/self/status` (`VmHWM`). + #[cfg(target_os = "linux")] + fn peak_rss_kib() -> u64 { + let status = std::fs::read_to_string("/proc/self/status").unwrap(); + for line in status.lines() { + if let Some(rest) = line.strip_prefix("VmHWM:") { + let kib = rest + .split_whitespace() + .next() + .and_then(|n| n.parse::().ok()) + .unwrap(); + return kib; + } + } + panic!("VmHWM not found in /proc/self/status"); + } + + #[test] + fn single_oversized_line_hits_the_cap_without_materializing_it() { + // A gzip whose one line is far larger than the cap: highly + // compressible repeated bytes with no newline until the very end. + // Uncompressed this is 64 MiB; compressed it is tiny. A decoder that + // read the whole line into memory would allocate the full 64 MiB. + let uncompressed_len = 64 * 1024 * 1024; + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + let block = vec![b'a'; 1024 * 1024]; + let mut written = 0usize; + while written < uncompressed_len { + encoder.write_all(&block).unwrap(); + written += block.len(); + } + // No trailing newline: the whole thing is one line. + let compressed = encoder.finish().unwrap(); + assert!( + compressed.len() < 512 * 1024, + "fixture should be small compressed, was {}", + compressed.len() + ); + + #[cfg(target_os = "linux")] + let before = peak_rss_kib(); + + let err = decode_data_file(&compressed, DataFileExpectations::default()).unwrap_err(); + assert!( + matches!(err, FormatError::LineTooLong { cap } if cap == MAX_LINE_BYTES), + "expected LineTooLong, got {err:?}" + ); + + // On Linux, prove the decoder never materialized the 64 MiB line: + // peak RSS growth across the call must stay well under it. The cap is + // 4 MiB plus the reader buffer, so a generous ceiling of 32 MiB + // growth still fails loudly if the whole line were buffered. + #[cfg(target_os = "linux")] + { + let after = peak_rss_kib(); + let growth_kib = after.saturating_sub(before); + println!( + "bomb test peak RSS: before={before} KiB, after={after} KiB, growth={growth_kib} KiB" + ); + assert!( + growth_kib < 32 * 1024, + "peak RSS grew {growth_kib} KiB decoding a 64 MiB bomb; \ + the line was not bounded" + ); + } + } + + #[test] + fn oversized_stream_is_rejected_before_full_read_when_byte_count_is_known() { + // Many small lines whose running total will exceed a deliberately + // low expectation; the decoder must stop mid-stream, not at EOF. + let items: Vec = (0..1000) + .map(|i| item(vec![("pk", AttributeValue::S(format!("item-{i}")))])) + .collect(); + let (compressed, _) = encode_data_file(items).unwrap(); + let expectations = DataFileExpectations { + uncompressed_bytes: Some(64), + ..Default::default() + }; + let mut decoder = DataFileDecoder::verified(&compressed, expectations).unwrap(); + let mut err = None; + // Pull items until the incremental byte-count check fires. + loop { + match decoder.next_item() { + Ok(Some(_)) => {} + Ok(None) => break, + Err(e) => { + err = Some(e); + break; + } + } + } + assert!( + matches!( + err, + Some(FormatError::ByteCountMismatch { expected: 64, .. }) + ), + "expected a mid-stream ByteCountMismatch, got {err:?}" + ); + } + + #[test] + fn multi_member_gzip_decodes_all_members() { + // Two independent gzip members concatenated: MultiGzDecoder must read + // both, GzDecoder would stop after the first. + let first = all_types_items(); + let second = vec![ + item(vec![("pk", AttributeValue::S("m2-a".to_owned()))]), + item(vec![("pk", AttributeValue::S("m2-b".to_owned()))]), + ]; + let (mut member_one, _) = encode_data_file(first.clone()).unwrap(); + let (member_two, _) = encode_data_file(second.clone()).unwrap(); + member_one.extend_from_slice(&member_two); + + let decoded = decode_data_file(&member_one, DataFileExpectations::default()).unwrap(); + let mut expected = first; + expected.extend(second); + assert_eq!(decoded, expected); + } + + /// The cap is exclusive: a line of exactly `MAX_LINE_BYTES` (newline + /// included) is read and then fails to parse as an item; one byte more is + /// `LineTooLong` before the line is materialized. + #[test] + fn line_cap_boundary() { + let build = |len: usize| { + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + let mut line = vec![b'x'; len - 1]; + line.push(b'\n'); + encoder.write_all(&line).unwrap(); + encoder.finish().unwrap() + }; + let at_cap = build(MAX_LINE_BYTES); + let err = decode_data_file(&at_cap, DataFileExpectations::default()).unwrap_err(); + assert!( + matches!(err, FormatError::Json(_)), + "at the cap the line is read: {err}" + ); + let over = build(MAX_LINE_BYTES + 1); + let err = decode_data_file(&over, DataFileExpectations::default()).unwrap_err(); + assert!(matches!(err, FormatError::LineTooLong { .. }), "{err}"); + } + + /// A sink that only exposes `Write`, like a file, and hands back at most + /// 7 bytes per call so the hashing writer's short-write handling is + /// exercised. + struct CountingSink(Vec, usize); + impl Write for CountingSink { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + let n = buf.len().min(7); + self.0.extend_from_slice(&buf[..n]); + self.1 += 1; + Ok(n) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + #[test] + fn streaming_encoder_matches_in_memory_encoder() { + let items = all_types_items(); + let (expected_bytes, expected) = encode_data_file(items.clone()).unwrap(); + let mut encoder = DataFileEncoder::new(CountingSink(Vec::new(), 0)); + for item in &items { + encoder.write_item(item).unwrap(); + } + assert_eq!(encoder.item_count(), items.len() as u64); + assert_eq!(encoder.uncompressed_bytes(), expected.uncompressed_bytes); + let (sink, checksums) = encoder.finish().unwrap(); + assert_eq!(sink.0, expected_bytes); + assert_eq!(checksums, expected); + assert!(sink.1 > 1, "short writes were exercised"); + } + + #[test] + fn encoder_output_is_deterministic() { + let (a, ca) = encode_data_file(all_types_items()).unwrap(); + let (b, cb) = encode_data_file(all_types_items()).unwrap(); + assert_eq!(a, b); + assert_eq!(ca, cb); + } + + #[test] + fn compressed_hasher_incremental_equals_one_shot() { + let (compressed, checksums) = encode_data_file(all_types_items()).unwrap(); + let mut hasher = CompressedHasher::new(); + for chunk in compressed.chunks(13) { + hasher.update(chunk); + } + let digest = hasher.finish(); + assert_eq!(digest, CompressedHasher::digest(&compressed)); + assert_eq!(digest.sha256_hex, checksums.sha256_hex); + assert_eq!(digest.md5_base64, checksums.md5_base64); + assert_eq!(digest.bytes, checksums.compressed_bytes); + DataFileExpectations::from(&checksums) + .verify_digest(&digest) + .unwrap(); + } + + #[test] + fn streaming_decoder_verifies_at_end_of_stream() { + let (mut compressed, checksums) = encode_data_file(all_types_items()).unwrap(); + // Flip a byte inside the gzip payload (past the 10-byte header, before + // the 8-byte trailer): the stream still decompresses something, but + // the digest over the stored bytes changes. + let n = compressed.len(); + compressed[n / 2] ^= 0x01; + let mut decoder = DataFileDecoder::new( + std::io::Cursor::new(compressed), + DataFileExpectations::from(&checksums), + ); + // Items are unverified until the end: the decoder either yields + // items, fails to decompress, or fails the digest check at EOF. It + // must not report clean completion. + let mut saw_error = false; + loop { + match decoder.next_item() { + Ok(Some(_)) => {} + Ok(None) => break, + Err(_) => { + saw_error = true; + break; + } + } + } + assert!(saw_error, "a tampered stream must not complete cleanly"); + } + + /// Bytes after the gzip member are parsed as a further member by + /// `MultiGzDecoder`, so garbage is a decode error (not a digest mismatch: + /// the digest check never runs because the stream did not end cleanly). + #[test] + fn trailing_bytes_after_gzip_member_are_a_decode_error() { + for garbage in [ + &b"x"[..], + &b"garbage after the gzip member"[..], + &[0u8; 8][..], + ] { + let (mut compressed, checksums) = encode_data_file(all_types_items()).unwrap(); + compressed.extend_from_slice(garbage); + let mut decoder = DataFileDecoder::new( + std::io::Cursor::new(compressed), + DataFileExpectations::from(&checksums), + ); + let mut n = 0; + let err = loop { + match decoder.next_item() { + Ok(Some(_)) => n += 1, + Ok(None) => panic!("trailing bytes must not decode cleanly"), + Err(e) => break e, + } + }; + assert!(matches!(err, FormatError::Decode(_)), "{err}"); + assert_eq!(n, all_types_items().len(), "the valid member decoded first"); + // The error is sticky. + assert!(matches!(decoder.next_item(), Err(FormatError::Decode(_)))); + } + } + + #[test] + fn decoder_errors_are_sticky() { + let (mut compressed, checksums) = encode_data_file(all_types_items()).unwrap(); + let n = compressed.len(); + compressed[n / 2] ^= 0x01; + let mut decoder = DataFileDecoder::new( + std::io::Cursor::new(compressed), + DataFileExpectations::from(&checksums), + ); + while let Ok(Some(_)) = decoder.next_item() {} + for _ in 0..3 { + let err = decoder + .next_item() + .expect_err("an error must not turn into Ok(None)"); + assert!(matches!(err, FormatError::Decode(_)), "{err}"); + assert!(err.to_string().contains("failed earlier"), "{err}"); + } + } + + /// The iterator yields the first error once and then ends, so a consumer + /// that drops errors terminates instead of spinning on the sticky error; + /// a consumer that collects into a `Result` still gets the error. + #[test] + fn iterator_is_fused_after_an_error() { + let tampered = || { + let (mut compressed, checksums) = encode_data_file(all_types_items()).unwrap(); + let n = compressed.len(); + compressed[n - 1] ^= 0x01; + DataFileDecoder::new( + std::io::Cursor::new(compressed), + DataFileExpectations::from(&checksums), + ) + }; + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let count = tampered().filter_map(Result::ok).count(); + let _ = tx.send(count); + }); + let count = rx + .recv_timeout(std::time::Duration::from_secs(10)) + .expect("an error-dropping consumer must terminate"); + assert!(count <= all_types_items().len()); + + let collected: Result, FormatError> = tampered().collect(); + assert!(collected.is_err(), "collect still surfaces the error"); + + let mut decoder = tampered(); + let mut errors = 0; + for r in decoder.by_ref() { + if r.is_err() { + errors += 1; + } + } + assert_eq!(errors, 1, "the error is yielded exactly once"); + assert!(decoder.next().is_none()); + assert!(decoder.next_item().is_err(), "next_item stays sticky"); + } + + /// Flaky sink: fails the `fail_on`th write, counts every write attempt. + struct Flaky { + buf: Vec, + calls: usize, + fail_on: usize, + } + impl Write for Flaky { + fn write(&mut self, b: &[u8]) -> std::io::Result { + self.calls += 1; + if self.calls == self.fail_on { + return Err(std::io::Error::other("boom")); + } + self.buf.extend_from_slice(b); + Ok(b.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + /// Incompressible payloads so the compressor flushes to the sink during + /// `write_item` rather than only at finish. + fn incompressible_items() -> Vec { + (0..4u64) + .map(|i| { + let mut x: u64 = 0x9e37_79b9_7f4a_7c15 ^ i; + let blob: String = (0..300_000) + .map(|_| { + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + char::from(b'a' + u8::try_from(x % 26).unwrap()) + }) + .collect(); + item(vec![("pk", AttributeValue::S(blob))]) + }) + .collect() + } + + #[test] + fn encoder_is_poisoned_after_a_sink_error() { + let big = incompressible_items(); + let mut encoder = DataFileEncoder::new(Flaky { + buf: Vec::new(), + calls: 0, + fail_on: 2, + }); + let mut first_err = None; + for it in &big { + if let Err(e) = encoder.write_item(it) { + first_err = Some(e); + break; + } + } + assert!( + matches!(first_err, Some(FormatError::Encode(_))), + "{first_err:?}" + ); + assert!(encoder.is_poisoned()); + assert!(!encoder.is_finished()); + // A retry of the item and a finish are both refused. + let err = encoder.write_item(&big[0]).unwrap_err(); + assert!(err.to_string().contains("poisoned"), "{err}"); + let err = encoder.try_finish().unwrap_err(); + assert!(err.to_string().contains("poisoned"), "{err}"); + // The sink can still be recovered for cleanup. + let sink = encoder.into_inner(); + assert!(sink.calls >= 2); + } + + /// Dropping a poisoned encoder must not push the compressor's buffered + /// data and trailer into the sink: the caller has decided to abandon the + /// file, and for an upload sink that would be tens of kilobytes of + /// multipart data after the abort decision. + #[test] + fn dropping_a_poisoned_encoder_writes_nothing_more() { + use std::sync::{Arc, Mutex}; + struct Shared(Arc>); + impl Write for Shared { + fn write(&mut self, b: &[u8]) -> std::io::Result { + self.0.lock().unwrap().write(b) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let sink = Arc::new(Mutex::new(Flaky { + buf: Vec::new(), + calls: 0, + fail_on: 2, + })); + let mut encoder = DataFileEncoder::new(Shared(Arc::clone(&sink))); + for it in &incompressible_items() { + if encoder.write_item(it).is_err() { + break; + } + } + assert!(encoder.is_poisoned()); + let calls_before = sink.lock().unwrap().calls; + drop(encoder); + assert_eq!( + sink.lock().unwrap().calls, + calls_before, + "drop of a poisoned encoder wrote into the sink" + ); + } + + #[test] + fn sink_accepting_no_bytes_is_an_error_not_a_hang() { + struct Zero; + impl Write for Zero { + fn write(&mut self, _b: &[u8]) -> std::io::Result { + Ok(0) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let mut encoder = DataFileEncoder::new(Zero); + let r = encoder.write_item(&all_types_items()[0]); + let _ = tx.send(r.map_err(|e| e.to_string())); + }); + let outcome = rx + .recv_timeout(std::time::Duration::from_secs(10)) + .expect("write_item returned within the deadline"); + let err = outcome.expect_err("Ok(0) sink is an error"); + assert!(err.contains("accepted no bytes"), "{err}"); + } + + #[test] + fn try_finish_keeps_the_sink_and_refuses_further_items() { + let mut encoder = DataFileEncoder::new(Vec::new()); + for it in all_types_items() { + encoder.write_item(&it).unwrap(); + } + let checksums = encoder.try_finish().unwrap(); + assert!(encoder.is_finished()); + assert!(!encoder.is_poisoned(), "a clean finish is not a failure"); + let err = encoder.write_item(&all_types_items()[0]).unwrap_err(); + assert!(err.to_string().contains("finished"), "{err}"); + let bytes = encoder.into_inner(); + assert_eq!(bytes.len() as u64, checksums.compressed_bytes); + let (expected_bytes, expected) = encode_data_file(all_types_items()).unwrap(); + assert_eq!(bytes, expected_bytes); + assert_eq!(checksums, expected); + } + + #[test] + fn streaming_decoder_clean_stream_completes_with_counts_checked() { + let (compressed, checksums) = encode_data_file(all_types_items()).unwrap(); + let mut decoder = DataFileDecoder::new( + std::io::Cursor::new(compressed), + DataFileExpectations::from(&checksums), + ); + let mut n = 0; + while let Some(_item) = decoder.next_item().unwrap() { + n += 1; + } + assert_eq!(n, all_types_items().len()); + // A second call after completion stays at the end. + assert!(decoder.next_item().unwrap().is_none()); + } + + #[test] + fn verified_decoder_rejects_before_first_item() { + let (mut compressed, checksums) = encode_data_file(all_types_items()).unwrap(); + let n = compressed.len(); + compressed[n / 2] ^= 0x01; + let err = DataFileDecoder::verified(&compressed, DataFileExpectations::from(&checksums)) + .err() + .expect("digest checked before any item"); + assert!(matches!(err, FormatError::ChecksumMismatch { .. }), "{err}"); + } +} diff --git a/crates/core/src/backup_format/ddb_export.rs b/crates/core/src/backup_format/ddb_export.rs new file mode 100644 index 000000000..c3bc6ee25 --- /dev/null +++ b/crates/core/src/backup_format/ddb_export.rs @@ -0,0 +1,339 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The two manifests Amazon `DynamoDB` writes for `ExportTableToPointInTime` +//! with `DYNAMODB_JSON` output: `manifest-summary.json` (one JSON document) +//! and `manifest-files.json` (JSON lines, one object per data file). +//! +//! Field sets and encodings follow the service's documented output format: +//! . +//! Notably: `version` is the string `"2020-06-30"` for full exports, +//! timestamps are ISO 8601 strings, `s3SseKmsKeyId` is serialized as an +//! explicit `null` when absent, `exportType` appears only on incremental +//! exports, and each files-manifest line carries `itemCount`, `md5Checksum` +//! (base64 of the data file's MD5), `etag`, and `dataFileS3Key`. +//! +//! `dataFileS3Key` is the object's full key in the bucket, not a path relative +//! to the manifest: the service writes `AWSDynamoDB//data/`. +//! An `ExtendDB` writer records the full store key the same way, +//! `///data/`. `ImportTable` +//! does not read this manifest (it lists the prefix it is given), so the +//! field is informational; `etag` is optional here because a filesystem store +//! has none. + +use serde::{Deserialize, Serialize}; + +use super::FormatError; + +/// The `version` value the service writes for full exports. +pub const EXPORT_SUMMARY_VERSION: &str = "2020-06-30"; + +/// The `outputFormat` value for `DynamoDB` JSON exports. +pub const DYNAMODB_JSON_OUTPUT_FORMAT: &str = "DYNAMODB_JSON"; + +/// The `manifest-summary.json` document. +/// +/// Unknown fields are tolerated so a newer service revision (for example the +/// incremental-export `exportFromTime`/`exportToTime`/`outputView` members) +/// still parses. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ExportSummaryManifest { + /// Manifest schema version, `"2020-06-30"` for full exports. + pub version: String, + /// ARN of the export. + pub export_arn: String, + /// When the export started, ISO 8601. + pub start_time: String, + /// When the export finished, ISO 8601. + pub end_time: String, + /// ARN of the exported table. + pub table_arn: String, + /// Unique id of the exported table. + pub table_id: String, + /// Point in time the export represents, ISO 8601. + pub export_time: String, + /// Destination bucket. + pub s3_bucket: String, + /// Destination prefix. + pub s3_prefix: String, + /// Server-side encryption algorithm of the written objects. + pub s3_sse_algorithm: String, + /// KMS key id when SSE-KMS was used; the service writes an explicit + /// `null` otherwise, so this member is never skipped. + pub s3_sse_kms_key_id: Option, + /// Key of the `manifest-files.json` object. + pub manifest_files_s3_key: String, + /// Billed size of the export. + pub billed_size_bytes: i64, + /// Total item count across all data files. + pub item_count: i64, + /// Output format, `"DYNAMODB_JSON"` here. + pub output_format: String, + /// Present only for incremental exports, where the service writes + /// `"INCREMENTAL_EXPORT"`; absent for full exports. + #[serde(skip_serializing_if = "Option::is_none", default)] + pub export_type: Option, +} + +impl ExportSummaryManifest { + /// Serialize to the compact JSON document the file holds. + /// + /// # Errors + /// + /// Returns [`FormatError::Json`] when serialization fails. + pub fn to_json(&self) -> Result { + Ok(serde_json::to_string(self)?) + } + + /// Parse a `manifest-summary.json` document. + /// + /// # Errors + /// + /// Returns [`FormatError::Json`] when the document does not parse. + pub fn from_json(s: &str) -> Result { + Ok(serde_json::from_str(s)?) + } +} + +/// One line of `manifest-files.json`: a single exported data file. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ExportFileEntry { + /// Number of items in the data file. + pub item_count: u64, + /// Base64 MD5 of the data file's bytes. + pub md5_checksum: String, + /// `ETag` of the data file object as the store reported it, without + /// quotes. The service always writes one; an `ExtendDB` writer omits the + /// member when the store (filesystem) has no `ETag`. + #[serde(skip_serializing_if = "Option::is_none", default)] + pub etag: Option, + /// Key of the data file object. + pub data_file_s3_key: String, +} + +/// Serialize a files manifest: one compact JSON object per line, each line +/// terminated with a newline, matching the JSON lines format the service +/// writes. +/// +/// # Errors +/// +/// Returns [`FormatError::Json`] when serialization fails. +pub fn write_files_manifest(entries: &[ExportFileEntry]) -> Result { + let mut out = String::new(); + for entry in entries { + out.push_str(&serde_json::to_string(entry)?); + out.push('\n'); + } + Ok(out) +} + +/// Parse a `manifest-files.json` document (JSON lines). Blank lines are +/// tolerated, including the absence of a trailing newline. +/// +/// # Errors +/// +/// Returns [`FormatError::Json`] when any non-blank line does not parse. +pub fn parse_files_manifest(s: &str) -> Result, FormatError> { + let mut entries = Vec::new(); + for line in s.lines() { + if line.trim().is_empty() { + continue; + } + entries.push(serde_json::from_str(line)?); + } + Ok(entries) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sample_summary() -> ExportSummaryManifest { + ExportSummaryManifest { + version: EXPORT_SUMMARY_VERSION.to_owned(), + export_arn: "arn:aws:dynamodb:us-east-1:123456789012:table/ProductCatalog/export/01234567890123-a1b2c3d4".to_owned(), + start_time: "2020-11-04T07:28:34.028Z".to_owned(), + end_time: "2020-11-04T07:33:43.897Z".to_owned(), + table_arn: "arn:aws:dynamodb:us-east-1:123456789012:table/ProductCatalog".to_owned(), + table_id: "12345a12-abcd-123a-ab12-1234abc12345".to_owned(), + export_time: "2020-11-04T07:28:34.028Z".to_owned(), + s3_bucket: "ddb-productcatalog-export".to_owned(), + s3_prefix: "2020-Nov".to_owned(), + s3_sse_algorithm: "AES256".to_owned(), + s3_sse_kms_key_id: None, + manifest_files_s3_key: "AWSDynamoDB/01693685827463-2d8752fd/manifest-files.json".to_owned(), + billed_size_bytes: 0, + item_count: 8, + output_format: DYNAMODB_JSON_OUTPUT_FORMAT.to_owned(), + export_type: None, + } + } + + #[test] + fn summary_serde_round_trip() { + let summary = sample_summary(); + let json = summary.to_json().unwrap(); + let parsed = ExportSummaryManifest::from_json(&json).unwrap(); + assert_eq!(parsed, summary); + } + + #[test] + fn summary_uses_documented_member_names() { + let json = sample_summary().to_json().unwrap(); + for member in [ + "\"version\"", + "\"exportArn\"", + "\"startTime\"", + "\"endTime\"", + "\"tableArn\"", + "\"tableId\"", + "\"exportTime\"", + "\"s3Bucket\"", + "\"s3Prefix\"", + "\"s3SseAlgorithm\"", + "\"s3SseKmsKeyId\"", + "\"manifestFilesS3Key\"", + "\"billedSizeBytes\"", + "\"itemCount\"", + "\"outputFormat\"", + ] { + assert!(json.contains(member), "missing member {member} in {json}"); + } + } + + #[test] + fn summary_writes_explicit_null_kms_key() { + let json = sample_summary().to_json().unwrap(); + assert!(json.contains("\"s3SseKmsKeyId\":null")); + } + + #[test] + fn summary_omits_export_type_for_full_exports() { + let json = sample_summary().to_json().unwrap(); + assert!(!json.contains("exportType")); + } + + #[test] + fn summary_carries_export_type_for_incremental_exports() { + let mut summary = sample_summary(); + summary.export_type = Some("INCREMENTAL_EXPORT".to_owned()); + let json = summary.to_json().unwrap(); + assert!(json.contains("\"exportType\":\"INCREMENTAL_EXPORT\"")); + let parsed = ExportSummaryManifest::from_json(&json).unwrap(); + assert_eq!(parsed, summary); + } + + #[test] + fn summary_parses_documented_example() { + // The full-export example from the documented output format. + let doc = r#"{ + "version": "2020-06-30", + "exportArn": "arn:aws:dynamodb:us-east-1:123456789012:table/ProductCatalog/export/01234567890123-a1b2c3d4", + "startTime": "2020-11-04T07:28:34.028Z", + "endTime": "2020-11-04T07:33:43.897Z", + "tableArn": "arn:aws:dynamodb:us-east-1:123456789012:table/ProductCatalog", + "tableId": "12345a12-abcd-123a-ab12-1234abc12345", + "exportTime": "2020-11-04T07:28:34.028Z", + "s3Bucket": "ddb-productcatalog-export", + "s3Prefix": "2020-Nov", + "s3SseAlgorithm": "AES256", + "s3SseKmsKeyId": null, + "manifestFilesS3Key": "AWSDynamoDB/01693685827463-2d8752fd/manifest-files.json", + "billedSizeBytes": 0, + "itemCount": 8, + "outputFormat": "DYNAMODB_JSON" +}"#; + let parsed = ExportSummaryManifest::from_json(doc).unwrap(); + assert_eq!(parsed, sample_summary()); + } + + #[test] + fn summary_tolerates_incremental_only_members() { + // Incremental exports add members full-export readers do not model. + let doc = r#"{ + "version": "2023-08-01", + "exportArn": "arn:aws:dynamodb:us-east-1:123456789012:table/t/export/x", + "startTime": "2023-09-19T04:20:18.000Z", + "endTime": "2023-09-19T04:40:24.780Z", + "tableArn": "arn:aws:dynamodb:us-east-1:123456789012:table/t", + "tableId": "b116b490-6460-4d4a-9a6b-5d360abf4fb3", + "exportFromTime": "2023-09-18T17:00:00.000Z", + "exportToTime": "2023-09-19T04:00:00.000Z", + "exportTime": "2023-09-19T04:00:00.000Z", + "s3Bucket": "b", + "s3Prefix": "p", + "s3SseAlgorithm": "AES256", + "s3SseKmsKeyId": null, + "manifestFilesS3Key": "p/AWSDynamoDB/x/manifest-files.json", + "billedSizeBytes": 20901239349, + "itemCount": 169928274, + "outputFormat": "DYNAMODB_JSON", + "outputView": "NEW_AND_OLD_IMAGES", + "exportType": "INCREMENTAL_EXPORT" +}"#; + let parsed = ExportSummaryManifest::from_json(doc).unwrap(); + assert_eq!(parsed.export_type.as_deref(), Some("INCREMENTAL_EXPORT")); + assert_eq!(parsed.item_count, 169_928_274); + } + + #[test] + fn file_entry_serde_round_trip() { + let entry = ExportFileEntry { + item_count: 8, + md5_checksum: "sQMSpEILNgoQmarvDFonGQ==".to_owned(), + etag: Some("af83d6f217c19b8b0fff8023d8ca4716-1".to_owned()), + data_file_s3_key: "AWSDynamoDB/01693685827463-2d8752fd/data/asdl123dasas.json.gz" + .to_owned(), + }; + let json = serde_json::to_string(&entry).unwrap(); + let parsed: ExportFileEntry = serde_json::from_str(&json).unwrap(); + assert_eq!(parsed, entry); + } + + #[test] + fn files_manifest_write_parse_round_trip() { + let entries = vec![ + ExportFileEntry { + item_count: 8, + md5_checksum: "sQMSpEILNgoQmarvDFonGQ==".to_owned(), + etag: Some("af83d6f217c19b8b0fff8023d8ca4716-1".to_owned()), + data_file_s3_key: "data/000001.json.gz".to_owned(), + }, + ExportFileEntry { + item_count: 0, + md5_checksum: "1B2M2Y8AsgTpgAmY7PhCfg==".to_owned(), + etag: Some("d41d8cd98f00b204e9800998ecf8427e".to_owned()), + data_file_s3_key: "data/000002.json.gz".to_owned(), + }, + ]; + let text = write_files_manifest(&entries).unwrap(); + assert_eq!(text.lines().count(), 2); + assert!(text.ends_with('\n')); + let parsed = parse_files_manifest(&text).unwrap(); + assert_eq!(parsed, entries); + } + + #[test] + fn files_manifest_parses_documented_example_line() { + // The files-manifest example from the documented output format, + // reflowed onto one line as the service writes it. + let line = r#"{"itemCount":8,"md5Checksum":"sQMSpEILNgoQmarvDFonGQ==","etag":"af83d6f217c19b8b0fff8023d8ca4716-1","dataFileS3Key":"AWSDynamoDB/01693685827463-2d8752fd/data/asdl123dasas.json.gz"}"#; + let parsed = parse_files_manifest(line).unwrap(); + assert_eq!(parsed.len(), 1); + assert_eq!(parsed[0].item_count, 8); + assert_eq!(parsed[0].md5_checksum, "sQMSpEILNgoQmarvDFonGQ=="); + // Writing back reproduces the line byte for byte, plus the newline. + let written = write_files_manifest(&parsed).unwrap(); + assert_eq!(written, format!("{line}\n")); + } + + #[test] + fn files_manifest_tolerates_blank_lines() { + let text = + "\n{\"itemCount\":1,\"md5Checksum\":\"m\",\"etag\":\"e\",\"dataFileS3Key\":\"k\"}\n\n"; + let parsed = parse_files_manifest(text).unwrap(); + assert_eq!(parsed.len(), 1); + } +} diff --git a/crates/core/src/backup_format/error.rs b/crates/core/src/backup_format/error.rs new file mode 100644 index 000000000..07c8e7123 --- /dev/null +++ b/crates/core/src/backup_format/error.rs @@ -0,0 +1,102 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Error type for backup format encoding, decoding, and verification. + +use thiserror::Error; + +/// Errors produced while reading, writing, or verifying the backup format. +#[derive(Debug, Error)] +pub enum FormatError { + /// The manifest declares a format version this build does not understand. + #[error( + "unsupported backup format version {found}; this build supports format version {supported}" + )] + UnsupportedFormatVersion { + /// Version found in the manifest. + found: u32, + /// Version this build supports. + supported: u32, + }, + + /// A checksum did not match the expected value. + #[error("{subject} checksum mismatch: expected {expected}, computed {computed}")] + ChecksumMismatch { + /// What was checksummed, for the error message. + subject: &'static str, + /// The recorded checksum. + expected: String, + /// The checksum computed from the bytes at hand. + computed: String, + }, + + /// The number of items decoded differs from the number recorded. + #[error("item count mismatch: expected {expected}, decoded {actual}")] + ItemCountMismatch { + /// The recorded item count. + expected: u64, + /// The number of items actually decoded. + actual: u64, + }, + + /// The uncompressed byte count differs from the number recorded. + #[error("uncompressed byte count mismatch: expected {expected}, decoded {actual}")] + ByteCountMismatch { + /// The recorded uncompressed byte count. + expected: u64, + /// The number of bytes actually decoded. + actual: u64, + }, + + /// A backup ARN did not have the expected shape. + #[error("invalid backup ARN: {0}")] + InvalidArn(String), + + /// JSON serialization or deserialization failed. + #[error("JSON error: {0}")] + Json(#[from] serde_json::Error), + + /// A single decompressed data-file line exceeded the maximum allowed + /// length, so decoding stopped before materializing it in full. + #[error( + "data file line exceeded the {cap}-byte maximum length; \ + refusing to decompress further to bound memory" + )] + LineTooLong { + /// The maximum line length, in bytes. + cap: usize, + }, + + /// A `DataFileEntry.key` did not have the required + /// `data/.json.gz` shape. + #[error("invalid data file key {key:?}: {reason}")] + InvalidDataFileKey { + /// The offending key. + key: String, + /// Why the key was rejected. + reason: &'static str, + }, + + /// Writing to the gzip encoder failed. + #[error("data file encode error: {0}")] + Encode(String), + + /// Gzip decompression failed or the stream was malformed. + #[error("data file decode error: {0}")] + Decode(String), + + /// The key schema has no order the format can define (multi-part keys, + /// or a key attribute without a definition). + #[error("key schema has no format-defined order: {0}")] + UnorderableKeySchema(String), + + /// An item's key attributes do not match the key schema. + #[error("invalid key: {0}")] + InvalidKey(String), + + /// A manifest's data file list is inconsistent: duplicate keys, + /// overlapping or unsorted key ranges, or a size total that does not + /// match. + #[error("invalid manifest: {0}")] + InvalidManifest(String), +} diff --git a/crates/core/src/backup_format/golden_tests.rs b/crates/core/src/backup_format/golden_tests.rs new file mode 100644 index 000000000..6e7b95d37 --- /dev/null +++ b/crates/core/src/backup_format/golden_tests.rs @@ -0,0 +1,374 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Golden-file tests against the checked-in `DynamoDB` export under +//! `testdata/ddb-export/` and the `ExtendDB` manifest under +//! `testdata/extenddb-manifest/`. +//! +//! The export fixture is constructed, not captured from the live service: it +//! follows the documented S3 export output format (member names, the +//! `2020-06-30` summary version, timestamps, the explicit `null` for +//! `s3SseKmsKeyId`) and was generated with this crate's own encoder, so what +//! it proves is that the reader accepts a document of that shape and that the +//! writer's output does not drift, not that it matches bytes the service +//! wrote. The README in that directory records the details. + +use std::path::PathBuf; + +use super::{ + DataFileExpectations, ExportSummaryManifest, decode_data_file, parse_files_manifest, + write_files_manifest, +}; +use crate::types::AttributeValue; + +fn fixture_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("testdata/ddb-export") +} + +fn read_fixture(name: &str) -> Vec { + let path = fixture_dir().join(name); + std::fs::read(&path).unwrap_or_else(|e| panic!("cannot read {}: {e}", path.display())) +} + +#[test] +fn golden_summary_manifest_parses() { + let raw = String::from_utf8(read_fixture("manifest-summary.json")).unwrap(); + let summary = ExportSummaryManifest::from_json(&raw).unwrap(); + assert_eq!(summary.version, "2020-06-30"); + assert_eq!( + summary.export_arn, + "arn:aws:dynamodb:us-east-1:123456789012:table/ProductCatalog/export/01693685827463-2d8752fd" + ); + assert_eq!(summary.table_id, "12345a12-abcd-123a-ab12-1234abc12345"); + assert_eq!(summary.s3_sse_algorithm, "AES256"); + assert_eq!(summary.s3_sse_kms_key_id, None); + assert_eq!(summary.item_count, 3); + assert_eq!(summary.output_format, "DYNAMODB_JSON"); + assert_eq!(summary.export_type, None); + + // The serializer reproduces the document byte for byte. + assert_eq!(summary.to_json().unwrap(), raw); +} + +#[test] +fn golden_files_manifest_parses_and_reserializes_byte_for_byte() { + let raw = String::from_utf8(read_fixture("manifest-files.json")).unwrap(); + let entries = parse_files_manifest(&raw).unwrap(); + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].item_count, 3); + assert_eq!(entries[0].md5_checksum, "1jUj2wzOC+iuxJN+LKy6Aw=="); + assert_eq!( + entries[0].data_file_s3_key, + "AWSDynamoDB/01693685827463-2d8752fd/data/ka2sswm5ha4uejfqcmnjcbg6ru.json.gz" + ); + + assert_eq!(write_files_manifest(&entries).unwrap(), raw); +} + +#[test] +fn golden_data_file_decodes_with_manifest_checksum() { + let files_raw = String::from_utf8(read_fixture("manifest-files.json")).unwrap(); + let entry = &parse_files_manifest(&files_raw).unwrap()[0]; + + let compressed = read_fixture("data/ka2sswm5ha4uejfqcmnjcbg6ru.json.gz"); + let expectations = DataFileExpectations { + md5_base64: Some(entry.md5_checksum.clone()), + item_count: Some(entry.item_count), + ..Default::default() + }; + let items = decode_data_file(&compressed, expectations).unwrap(); + assert_eq!(items.len(), 3); + + assert_eq!( + items[0].get("Id"), + Some(&AttributeValue::N("103".to_owned())) + ); + assert_eq!( + items[0].get("Title"), + Some(&AttributeValue::S("Book 103 Title".to_owned())) + ); + assert_eq!( + items[0].get("Authors"), + Some(&AttributeValue::SS( + ["Author1", "Author2"] + .iter() + .map(|s| (*s).to_owned()) + .collect() + )) + ); + assert_eq!( + items[0].get("InPublication"), + Some(&AttributeValue::Bool(false)) + ); + assert_eq!( + items[1].get("Binary"), + Some(&AttributeValue::B(vec![0xde, 0xad, 0xbe, 0xef])) + ); + match items[2].get("Meta") { + Some(AttributeValue::M(map)) => { + assert_eq!(map.get("a"), Some(&AttributeValue::Null)); + assert_eq!( + map.get("list"), + Some(&AttributeValue::L(vec![ + AttributeValue::N("1".to_owned()), + AttributeValue::S("x".to_owned()), + ])) + ); + } + other => panic!("unexpected Meta value: {other:?}"), + } +} + +#[test] +fn golden_data_file_reencodes_to_identical_lines() { + // Decoding the fixture and re-encoding its items must reproduce the same + // uncompressed lines: the fixture's line encoding and the encoder's agree + // byte for byte. (The gzip container itself is not compared, since + // compressor output is implementation-defined.) + use std::io::Read; + + let compressed = read_fixture("data/ka2sswm5ha4uejfqcmnjcbg6ru.json.gz"); + let items = decode_data_file(&compressed, DataFileExpectations::default()).unwrap(); + + let (reencoded, checksums) = super::encode_data_file(items).unwrap(); + assert_eq!(checksums.item_count, 3); + + let mut fixture_lines = String::new(); + flate2::read::GzDecoder::new(compressed.as_slice()) + .read_to_string(&mut fixture_lines) + .unwrap(); + let mut reencoded_lines = String::new(); + flate2::read::GzDecoder::new(reencoded.as_slice()) + .read_to_string(&mut reencoded_lines) + .unwrap(); + assert_eq!(reencoded_lines, fixture_lines); + assert_eq!(checksums.uncompressed_bytes, fixture_lines.len() as u64); +} + +/// The frozen `extenddb-manifest.json` for format version 1. The nested +/// members reuse the `DynamoDB` wire types, so a change to one of those types +/// (a rename, a `skip_serializing_if` toggle, a new required member) changes +/// the on-disk format without a version bump and without any other test +/// noticing; this fixture makes such a change fail here. Regenerate it only +/// with a format version bump. +mod extenddb_manifest_golden { + use super::super::{ExtenddbManifest, manifest_checksum, verify_manifest_checksum}; + + const MANIFEST: &[u8] = include_bytes!("../../testdata/extenddb-manifest/v1.json"); + const SIDECAR: &str = include_str!("../../testdata/extenddb-manifest/v1.sha256"); + + #[test] + fn golden_manifest_parses_and_reserializes_byte_for_byte() { + let manifest = ExtenddbManifest::from_json_bytes(MANIFEST).expect("golden parses"); + let again = manifest.to_json_bytes().unwrap(); + assert_eq!( + std::str::from_utf8(&again).unwrap(), + std::str::from_utf8(MANIFEST).unwrap(), + "the on-disk shape of format version 1 changed; bump the version" + ); + } + + #[test] + fn golden_sidecar_verifies() { + verify_manifest_checksum(MANIFEST, SIDECAR).unwrap(); + assert_eq!(manifest_checksum(MANIFEST), SIDECAR); + } + + #[test] + fn golden_manifest_covers_every_optional_member() { + // The fixture sets every top-level optional member of the source + // table so a serialization change to any of them is caught. Nested + // optionals inside the index and vector shapes (an LSI's + // `NonKeyAttributes`, the vector index's `SearchSchema` and + // `Projection`) are absent in the fixture; the byte-for-byte test + // still catches a `skip_serializing_if` change on them but not a + // rename. + let manifest = ExtenddbManifest::from_json_bytes(MANIFEST).unwrap(); + let t = &manifest.source_table; + assert!(t.provisioned_throughput.is_some()); + assert!(t.on_demand_throughput.is_some()); + assert!(t.table_class.is_some()); + assert!(t.sse_description.is_some()); + assert!(!t.global_secondary_indexes.is_empty()); + assert!(!t.local_secondary_indexes.is_empty()); + assert!(t.vector_indexes.as_ref().is_some_and(|v| !v.is_empty())); + assert!(manifest.data_files.iter().all(|f| f.key_range.is_some())); + } +} + +#[cfg(test)] +mod generate_golden_manifest { + #![allow(clippy::too_many_lines)] + //! Run with `cargo test -p extenddb-core write_golden_manifest -- + //! --ignored` to (re)generate the fixture. Only when the format version + //! is bumped. + use super::super::{ + DataFileEntry, ExtenddbManifest, ItemOrder, KeyRange, SnapshotDescriptor, + SourceTableSchema, manifest_checksum, + }; + use crate::types::{ + AttributeDefinition, AttributeValue, BillingMode, DistanceFunction, GsiInput, Item, + KeySchemaElement, KeyType, LsiInput, OnDemandThroughput, Projection, ProjectionType, + ProvisionedThroughput, ScalarAttributeType, SseDescription, SseType, VectorAttribute, + VectorIndexSpecification, + }; + + fn key(a: &str, t: &str) -> Item { + [ + ("Artist".to_owned(), AttributeValue::S(a.to_owned())), + ("SongTitle".to_owned(), AttributeValue::S(t.to_owned())), + ] + .into_iter() + .collect() + } + + fn golden() -> ExtenddbManifest { + ExtenddbManifest { + format_version: 1, + backup_arn: + "arn:aws:dynamodb:us-east-1:123456789012:table/Music/backup/01757500000123-0badcafe" + .to_owned(), + backup_name: "music-nightly".to_owned(), + backup_creation_date_time: 1_757_500_000.123, + backup_size_bytes: 61_728 + 100, + source_backend: "postgres".to_owned(), + source_extenddb_version: "0.1.13".to_owned(), + snapshot: SnapshotDescriptor { + kind: "postgres_repeatable_read".to_owned(), + marker: "1234:1234:".to_owned(), + }, + item_order: ItemOrder::KeyOrdered, + source_table: SourceTableSchema { + table_name: "Music".to_owned(), + table_id: "12345a12-abcd-123a-ab12-1234abc12345".to_owned(), + table_arn: "arn:aws:dynamodb:us-east-1:123456789012:table/Music".to_owned(), + table_creation_date_time: 1_757_400_000.0, + key_schema: vec![ + KeySchemaElement { + attribute_name: "Artist".to_owned(), + key_type: KeyType::Hash, + }, + KeySchemaElement { + attribute_name: "SongTitle".to_owned(), + key_type: KeyType::Range, + }, + ], + attribute_definitions: vec![ + AttributeDefinition { + attribute_name: "Artist".to_owned(), + attribute_type: ScalarAttributeType::S, + }, + AttributeDefinition { + attribute_name: "SongTitle".to_owned(), + attribute_type: ScalarAttributeType::S, + }, + AttributeDefinition { + attribute_name: "Genre".to_owned(), + attribute_type: ScalarAttributeType::S, + }, + ], + billing_mode: BillingMode::Provisioned, + provisioned_throughput: Some(ProvisionedThroughput { + read_capacity_units: 5, + write_capacity_units: 5, + }), + on_demand_throughput: Some(OnDemandThroughput { + max_read_request_units: Some(100), + max_write_request_units: Some(50), + }), + table_class: Some("STANDARD".to_owned()), + sse_description: Some(SseDescription { + status: "ENABLED".to_owned(), + sse_type: Some(SseType::KMS), + kms_master_key_arn: Some( + "arn:aws:kms:us-east-1:123456789012:key/abcd".to_owned(), + ), + }), + global_secondary_indexes: vec![GsiInput { + index_name: "GenreIndex".to_owned(), + key_schema: vec![KeySchemaElement { + attribute_name: "Genre".to_owned(), + key_type: KeyType::Hash, + }], + projection: Projection { + projection_type: ProjectionType::Include, + non_key_attributes: Some(vec!["Year".to_owned()]), + }, + provisioned_throughput: Some(ProvisionedThroughput { + read_capacity_units: 1, + write_capacity_units: 1, + }), + }], + local_secondary_indexes: vec![LsiInput { + index_name: "TitleIndex".to_owned(), + key_schema: vec![ + KeySchemaElement { + attribute_name: "Artist".to_owned(), + key_type: KeyType::Hash, + }, + KeySchemaElement { + attribute_name: "SongTitle".to_owned(), + key_type: KeyType::Range, + }, + ], + projection: Projection { + projection_type: ProjectionType::KeysOnly, + non_key_attributes: None, + }, + }], + vector_indexes: Some(vec![VectorIndexSpecification { + index_name: "embeddings".to_owned(), + dimensions: 1024, + distance_function: DistanceFunction::Cosine, + vector_attribute: VectorAttribute { + attribute_name: "vec".to_owned(), + }, + search_schema: None, + projection: None, + }]), + item_count: 100_000, + table_size_bytes: 123_456, + }, + data_files: vec![ + DataFileEntry { + key: "data/000001.json.gz".to_owned(), + item_count: 50_000, + sha256: "0".repeat(64), + md5: "1B2M2Y8AsgTpgAmY7PhCfg==".to_owned(), + bytes: 61_728, + uncompressed_bytes: 300_000, + key_range: Some(KeyRange { + first: key("A", "a"), + last: key("M", "z"), + }), + }, + DataFileEntry { + key: "data/000002.json.gz".to_owned(), + item_count: 50_000, + sha256: "1".repeat(64), + md5: "1B2M2Y8AsgTpgAmY7PhCfg==".to_owned(), + bytes: 100, + uncompressed_bytes: 300_000, + key_range: Some(KeyRange { + first: key("N", "a"), + last: key("Z", "z"), + }), + }, + ], + } + } + + /// Regenerates the fixture from `golden()`. Ignored by default so an + /// ordinary `cargo test` never rewrites checked-in files; run it with + /// `cargo test -p extenddb-core write_golden_manifest -- --ignored` after + /// an intentional format change, then review the diff. + #[test] + #[ignore = "rewrites the checked-in fixture; run explicitly"] + fn write_golden_manifest() { + let dir = + std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("testdata/extenddb-manifest"); + let bytes = golden().to_json_bytes().unwrap(); + std::fs::write(dir.join("v1.json"), &bytes).unwrap(); + std::fs::write(dir.join("v1.sha256"), manifest_checksum(&bytes)).unwrap(); + } +} diff --git a/crates/core/src/backup_format/key_order.rs b/crates/core/src/backup_format/key_order.rs new file mode 100644 index 000000000..8f3043506 --- /dev/null +++ b/crates/core/src/backup_format/key_order.rs @@ -0,0 +1,463 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The one definition of base-table key order for the backup format. +//! +//! A `key_ordered` backup promises its items ascend in this order within each +//! data file and from one file to the next: +//! +//! - Hash key: the text encoding [`hash_key_text`] produces (`S` as is, `N` +//! as its normalized decimal string, `B` as standard base64), compared +//! bytewise. A number hash key therefore sorts as text (`"10"` before +//! `"9"`); this is how the storage layer keys rows and the format inherits +//! it rather than defining a second order. +//! - Range key: `S` bytewise, `N` by numeric value, `B` bytewise. +//! +//! The SQL backends can produce this order from their key columns, but only +//! with the right collation on the query, whatever the column declares: +//! +//! - SQLite: `ORDER BY pk, sk_s` (or `sk_b`) with the default `BINARY` +//! collation is memcmp; `ORDER BY pk, sk_n` is numeric because `sk_n` holds +//! `encode_orderable_number`'s order-preserving text encoding. The +//! `storage-sqlite` test `number_key_order_matches_backup_format` binds +//! that encoding to [`KeyOrder::compare`]. +//! - Postgres: `ORDER BY pk COLLATE "C", sk_s COLLATE "C"`; `sk_n` is +//! `NUMERIC` and `sk_b` is `bytea`, both already in the format's order. +//! Tables created before the key columns were declared `COLLATE "C"` keep +//! the database collation (`en_US.utf8` in the default setups, where +//! `a < B` and punctuation is weak), so a writer that omits the collation +//! on `pk` or `sk_s` emits a file that is not in this order. +//! +//! Backends that cannot produce this order write `unordered`. The restore +//! path binds [`KeyOrder::compare`] for its overlap and monotonicity checks, +//! and the storage layer's `pk_to_text` binds [`hash_key_text`], so for the +//! hash key the writer, the reader, and the row key agree by construction. +//! (Two other copies of the hash-key text mapping remain in the Postgres and +//! SQLite `tx_helpers` for stream shard assignment; they are not on the +//! backup path.) Range-key agreement rests on each backend's column order +//! matching the rules above, which the tests named here check. + +use std::borrow::Cow; +use std::cmp::Ordering; +use std::str::FromStr; + +use base64::Engine; +use base64::engine::general_purpose::STANDARD as BASE64; +use bigdecimal::BigDecimal; + +use super::FormatError; +use crate::types::{ + AttributeDefinition, AttributeValue, Item, KeySchemaElement, KeyType, ScalarAttributeType, +}; +use crate::validation::number::validate_and_normalize_number; + +/// Text encoding of a hash key value: the row key the SQL backends store. +/// `None` for a value that cannot be a hash key. +#[must_use] +pub fn hash_key_text(value: &AttributeValue) -> Option> { + match value { + AttributeValue::S(s) => Some(Cow::Borrowed(s)), + AttributeValue::N(n) => Some(Cow::Borrowed(n)), + AttributeValue::B(b) => Some(Cow::Owned(BASE64.encode(b))), + _ => None, + } +} + +/// A table's key shape, resolved once from its key schema and attribute +/// definitions, able to compare and validate items' keys. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct KeyOrder { + hash: (String, ScalarAttributeType), + range: Option<(String, ScalarAttributeType)>, +} + +impl KeyOrder { + /// Resolve the key shape. Only hash-only and single-range-key tables have + /// a format-defined order; a multi-part range key returns + /// [`FormatError::UnorderableKeySchema`], and a writer for such a table + /// writes `unordered`. + /// + /// # Errors + /// + /// [`FormatError::UnorderableKeySchema`] when the schema has no hash key, + /// more than one hash or range key, a key attribute without a definition, + /// or an attribute defined more than once. + pub fn resolve( + key_schema: &[KeySchemaElement], + attribute_definitions: &[AttributeDefinition], + ) -> Result { + let type_of = |name: &str| { + let mut matching = attribute_definitions + .iter() + .filter(|d| d.attribute_name == name) + .map(|d| d.attribute_type); + let ty = matching.next().ok_or_else(|| { + FormatError::UnorderableKeySchema(format!( + "key attribute {name:?} has no attribute definition" + )) + })?; + if matching.next().is_some() { + return Err(FormatError::UnorderableKeySchema(format!( + "key attribute {name:?} has more than one attribute definition" + ))); + } + Ok(ty) + }; + let mut hash = None; + let mut range = None; + for element in key_schema { + let slot = match element.key_type { + KeyType::Hash => &mut hash, + KeyType::Range => &mut range, + }; + if slot.is_some() { + return Err(FormatError::UnorderableKeySchema( + "multi-part keys have no format-defined order".to_owned(), + )); + } + *slot = Some(( + element.attribute_name.clone(), + type_of(&element.attribute_name)?, + )); + } + let hash = hash.ok_or_else(|| { + FormatError::UnorderableKeySchema("key schema has no hash key".to_owned()) + })?; + Ok(Self { hash, range }) + } + + /// Names of the key attributes, hash first. + pub fn attribute_names(&self) -> impl Iterator { + std::iter::once(self.hash.0.as_str()).chain(self.range.iter().map(|(n, _)| n.as_str())) + } + + /// Check that `key` holds exactly the key attributes with the declared + /// types and nothing else: the shape [`super::KeyRange`] carries. An `N` + /// value must be a valid `DynamoDB` number in its normalized form (the + /// form the deserializer and the row key produce), so an endpoint such as + /// `1E200`, `1_000` or 39 significant digits is refused here rather than + /// parsed leniently by [`Self::compare`]. + /// + /// # Errors + /// + /// [`FormatError::InvalidKey`] naming what is wrong. + pub fn validate_key_item(&self, key: &Item) -> Result<(), FormatError> { + let expected: usize = 1 + usize::from(self.range.is_some()); + if key.len() != expected { + return Err(FormatError::InvalidKey(format!( + "key has {} attributes, the key schema has {expected}", + key.len() + ))); + } + for (name, ty) in std::iter::once(&self.hash).chain(self.range.iter()) { + let value = key.get(name).ok_or_else(|| { + FormatError::InvalidKey(format!("key attribute {name:?} missing")) + })?; + let actual = match value { + AttributeValue::S(_) => ScalarAttributeType::S, + AttributeValue::N(n) => { + let normalized = validate_and_normalize_number(n).map_err(|e| { + FormatError::InvalidKey(format!( + "key attribute {name:?} value {n:?} is not a valid number: {e}" + )) + })?; + if normalized != *n { + return Err(FormatError::InvalidKey(format!( + "key attribute {name:?} value {n:?} is not in normalized form \ + ({normalized:?})" + ))); + } + ScalarAttributeType::N + } + AttributeValue::B(_) => ScalarAttributeType::B, + _ => { + return Err(FormatError::InvalidKey(format!( + "key attribute {name:?} is not a scalar S, N or B" + ))); + } + }; + if actual != *ty { + return Err(FormatError::InvalidKey(format!( + "key attribute {name:?} is {actual:?}, the schema declares {ty:?}" + ))); + } + } + Ok(()) + } + + /// Compare two items by their keys. Both must pass + /// [`Self::validate_key_item`] on their key attributes (extra attributes + /// are ignored here, so whole items can be compared). + /// + /// # Errors + /// + /// [`FormatError::InvalidKey`] when a key attribute is missing, has the + /// wrong type, or (for `N`) does not parse as a number. The last case is + /// defensive: a value that passed [`Self::validate_key_item`] always + /// parses. + pub fn compare(&self, a: &Item, b: &Item) -> Result { + let ha = hash_text(a, &self.hash)?; + let hb = hash_text(b, &self.hash)?; + match ha.as_bytes().cmp(hb.as_bytes()) { + Ordering::Equal => {} + other => return Ok(other), + } + let Some((name, ty)) = &self.range else { + return Ok(Ordering::Equal); + }; + let ra = a + .get(name) + .ok_or_else(|| FormatError::InvalidKey(format!("range key {name:?} missing")))?; + let rb = b + .get(name) + .ok_or_else(|| FormatError::InvalidKey(format!("range key {name:?} missing")))?; + match (ty, ra, rb) { + (ScalarAttributeType::S, AttributeValue::S(x), AttributeValue::S(y)) => { + Ok(x.as_bytes().cmp(y.as_bytes())) + } + (ScalarAttributeType::B, AttributeValue::B(x), AttributeValue::B(y)) => Ok(x.cmp(y)), + (ScalarAttributeType::N, AttributeValue::N(x), AttributeValue::N(y)) => { + let parse = |s: &str| { + BigDecimal::from_str(s).map_err(|e| { + FormatError::InvalidKey(format!("range key {name:?} value {s:?}: {e}")) + }) + }; + Ok(parse(x)?.cmp(&parse(y)?)) + } + _ => Err(FormatError::InvalidKey(format!( + "range key {name:?} does not match its declared type {ty:?}" + ))), + } + } +} + +fn hash_text<'a>( + item: &'a Item, + (name, ty): &(String, ScalarAttributeType), +) -> Result, FormatError> { + let value = item + .get(name) + .ok_or_else(|| FormatError::InvalidKey(format!("hash key {name:?} missing")))?; + let actual = match value { + AttributeValue::S(_) => ScalarAttributeType::S, + AttributeValue::N(_) => ScalarAttributeType::N, + AttributeValue::B(_) => ScalarAttributeType::B, + _ => { + return Err(FormatError::InvalidKey(format!( + "hash key {name:?} is not a scalar S, N or B" + ))); + } + }; + if actual != *ty { + return Err(FormatError::InvalidKey(format!( + "hash key {name:?} is {actual:?}, the schema declares {ty:?}" + ))); + } + Ok(hash_key_text(value).expect("scalar checked above")) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn schema(range: Option) -> KeyOrder { + let mut ks = vec![KeySchemaElement { + attribute_name: "pk".to_owned(), + key_type: KeyType::Hash, + }]; + let mut defs = vec![AttributeDefinition { + attribute_name: "pk".to_owned(), + attribute_type: ScalarAttributeType::S, + }]; + if let Some(ty) = range { + ks.push(KeySchemaElement { + attribute_name: "sk".to_owned(), + key_type: KeyType::Range, + }); + defs.push(AttributeDefinition { + attribute_name: "sk".to_owned(), + attribute_type: ty, + }); + } + KeyOrder::resolve(&ks, &defs).unwrap() + } + + fn item(pk: &str, sk: Option) -> Item { + let mut m = Item::new(); + m.insert("pk".to_owned(), AttributeValue::S(pk.to_owned())); + if let Some(sk) = sk { + m.insert("sk".to_owned(), sk); + } + m + } + + #[test] + fn hash_key_text_matches_storage_encoding() { + assert_eq!(hash_key_text(&AttributeValue::S("a".into())).unwrap(), "a"); + assert_eq!( + hash_key_text(&AttributeValue::N("10".into())).unwrap(), + "10" + ); + assert_eq!( + hash_key_text(&AttributeValue::B(vec![0xde, 0xad])).unwrap(), + "3q0=" + ); + assert!(hash_key_text(&AttributeValue::Bool(true)).is_none()); + } + + #[test] + fn number_hash_keys_sort_as_text() { + let ks = vec![KeySchemaElement { + attribute_name: "pk".to_owned(), + key_type: KeyType::Hash, + }]; + let defs = vec![AttributeDefinition { + attribute_name: "pk".to_owned(), + attribute_type: ScalarAttributeType::N, + }]; + let order = KeyOrder::resolve(&ks, &defs).unwrap(); + let n = |s: &str| { + let mut m = Item::new(); + m.insert("pk".to_owned(), AttributeValue::N(s.to_owned())); + m + }; + assert_eq!(order.compare(&n("10"), &n("9")).unwrap(), Ordering::Less); + } + + #[test] + fn number_range_keys_sort_numerically() { + let order = schema(Some(ScalarAttributeType::N)); + let a = item("p", Some(AttributeValue::N("9".into()))); + let b = item("p", Some(AttributeValue::N("10".into()))); + let c = item("p", Some(AttributeValue::N("-1.5".into()))); + assert_eq!(order.compare(&a, &b).unwrap(), Ordering::Less); + assert_eq!(order.compare(&c, &a).unwrap(), Ordering::Less); + assert_eq!(order.compare(&a, &a).unwrap(), Ordering::Equal); + } + + #[test] + fn string_and_binary_range_keys_sort_bytewise() { + let order = schema(Some(ScalarAttributeType::S)); + let a = item("p", Some(AttributeValue::S("B".into()))); + let b = item("p", Some(AttributeValue::S("a".into()))); + assert_eq!(order.compare(&a, &b).unwrap(), Ordering::Less); + let order = schema(Some(ScalarAttributeType::B)); + let a = item("p", Some(AttributeValue::B(vec![1, 2]))); + let b = item("p", Some(AttributeValue::B(vec![1, 2, 0]))); + assert_eq!(order.compare(&a, &b).unwrap(), Ordering::Less); + } + + #[test] + fn hash_key_decides_before_range_key() { + let order = schema(Some(ScalarAttributeType::N)); + let a = item("a", Some(AttributeValue::N("99".into()))); + let b = item("b", Some(AttributeValue::N("1".into()))); + assert_eq!(order.compare(&a, &b).unwrap(), Ordering::Less); + } + + #[test] + fn validate_key_item_rejects_wrong_shape() { + let order = schema(Some(ScalarAttributeType::N)); + order + .validate_key_item(&item("p", Some(AttributeValue::N("1".into())))) + .unwrap(); + assert!(order.validate_key_item(&item("p", None)).is_err()); + assert!( + order + .validate_key_item(&item("p", Some(AttributeValue::S("1".into())))) + .is_err() + ); + let mut extra = item("p", Some(AttributeValue::N("1".into()))); + extra.insert("v".to_owned(), AttributeValue::Null); + assert!(order.validate_key_item(&extra).is_err()); + } + + /// `N` endpoints must be valid `DynamoDB` numbers in normalized form; + /// `BigDecimal::from_str` alone would accept all of these. + #[test] + fn validate_key_item_rejects_non_dynamodb_numbers() { + let order = schema(Some(ScalarAttributeType::N)); + for bad in [ + "1E200", + "1_000", + "123456789012345678901234567890123456789", + "1.0", + "+1", + ".5", + "1.", + "-0", + "abc", + ] { + let err = order + .validate_key_item(&item("p", Some(AttributeValue::N(bad.into())))) + .err() + .unwrap_or_else(|| panic!("{bad:?} must be rejected")); + assert!(matches!(err, FormatError::InvalidKey(_)), "{bad:?}: {err}"); + } + for good in ["0", "1", "-1.5", "1E+125", "1E-130", "10"] { + let normalized = validate_and_normalize_number(good).unwrap(); + order + .validate_key_item(&item("p", Some(AttributeValue::N(normalized.clone())))) + .unwrap_or_else(|e| panic!("{normalized:?} (from {good:?}) rejected: {e}")); + } + } + + #[test] + fn duplicate_attribute_definitions_are_unorderable() { + let ks = vec![KeySchemaElement { + attribute_name: "pk".to_owned(), + key_type: KeyType::Hash, + }]; + let defs = vec![ + AttributeDefinition { + attribute_name: "pk".to_owned(), + attribute_type: ScalarAttributeType::S, + }, + AttributeDefinition { + attribute_name: "pk".to_owned(), + attribute_type: ScalarAttributeType::N, + }, + ]; + let err = KeyOrder::resolve(&ks, &defs).unwrap_err(); + assert!(matches!(err, FormatError::UnorderableKeySchema(_)), "{err}"); + assert!(err.to_string().contains("more than one"), "{err}"); + } + + #[test] + fn compare_rejects_type_mismatch_and_bad_numbers() { + let order = schema(Some(ScalarAttributeType::N)); + let a = item("p", Some(AttributeValue::N("1".into()))); + let bad = item("p", Some(AttributeValue::S("1".into()))); + assert!(order.compare(&a, &bad).is_err()); + let nan = item("p", Some(AttributeValue::N("abc".into()))); + assert!(order.compare(&a, &nan).is_err()); + } + + #[test] + fn multi_part_keys_are_unorderable() { + let ks = vec![ + KeySchemaElement { + attribute_name: "a".to_owned(), + key_type: KeyType::Hash, + }, + KeySchemaElement { + attribute_name: "b".to_owned(), + key_type: KeyType::Hash, + }, + ]; + let defs = vec![ + AttributeDefinition { + attribute_name: "a".to_owned(), + attribute_type: ScalarAttributeType::S, + }, + AttributeDefinition { + attribute_name: "b".to_owned(), + attribute_type: ScalarAttributeType::S, + }, + ]; + assert!(matches!( + KeyOrder::resolve(&ks, &defs), + Err(FormatError::UnorderableKeySchema(_)) + )); + } +} diff --git a/crates/core/src/backup_format/layout.rs b/crates/core/src/backup_format/layout.rs new file mode 100644 index 000000000..3dad561fc --- /dev/null +++ b/crates/core/src/backup_format/layout.rs @@ -0,0 +1,356 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Store key layout for a backup prefix, and the backup ARN. +//! +//! A backup lives under `///` in the store +//! and holds the fixed object names defined here plus numbered data files +//! under `data/`. [`BackupArn`] parses and formats the ARN shape +//! `arn::dynamodb:::table/
/backup/` +//! and turns it into a store prefix. It is meant to become the one place that +//! shape lives; today the storage backends still format the ARN themselves +//! (in Rust and in SQL) and the engine's account check still splits it by +//! hand. Moving them onto `BackupArn` is a follow-up in the backup/restore +//! series, not part of the format freeze. +//! +//! Every component is validated on construction, whether it comes from an ARN +//! string or from the engine's own fields, so a prefix can never contain a +//! separator, a traversal component, or a control character: the store's key +//! validation is a second line, not the first. + +use super::FormatError; + +/// The `ExtendDB` manifest, written last; its presence marks a complete backup. +pub const EXTENDDB_MANIFEST_FILE: &str = "extenddb-manifest.json"; + +/// The sha256 of the manifest's bytes, written before the manifest. See +/// [`super::manifest_checksum`]. +pub const MANIFEST_CHECKSUM_FILE: &str = "extenddb-manifest.sha256"; + +/// The `DynamoDB` export summary manifest. +pub const SUMMARY_MANIFEST_FILE: &str = "manifest-summary.json"; + +/// The `DynamoDB` export files manifest. +pub const FILES_MANIFEST_FILE: &str = "manifest-files.json"; + +/// The prefix that holds only data files, so `ImportTable` against it never +/// sees a manifest. +pub const DATA_PREFIX: &str = "data/"; + +/// Longest table name `DynamoDB` accepts, kept here as a path-safety bound: +/// a longer component would exceed common filesystem name limits. The +/// engine's configurable minimum length is not enforced here; a shorter name +/// the engine accepted is a valid ARN component. +const MAX_TABLE_NAME_LEN: usize = 255; + +/// The key of the `index`-th data file within a backup prefix: +/// `data/000001.json.gz` for index 1. +#[must_use] +pub fn data_file_key(index: u32) -> String { + format!("{DATA_PREFIX}{index:06}.json.gz") +} + +/// A validated backup ARN. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BackupArn { + partition: String, + region: String, + account_id: String, + table_name: String, + backup_id: String, +} + +impl BackupArn { + /// Build an ARN from its components, validating each. + /// + /// Rules: `partition` and `region` are non-empty lowercase ASCII letters, + /// digits and `-` (`aws`, `aws-cn`, `aws-us-gov`, `us-east-1`); + /// `account_id` is non-empty ASCII letters, digits, `_` and `-`; + /// `table_name` passes the engine's character rule + /// ([`crate::validation::validate_table_name_chars`]: ASCII letters, + /// digits, `_`, `.`, `-`), is at most 255 bytes, and is not made only of + /// dots; `backup_id` is non-empty ASCII letters, digits, `_`, `.` and `-` + /// and not made only of dots. Nothing may contain `/`, `:`, or + /// whitespace, so the formatted ARN parses back to the same components + /// and the store prefix has exactly three segments. + /// + /// # Errors + /// + /// Returns [`FormatError::InvalidArn`] naming the component that failed. + pub fn new( + partition: &str, + region: &str, + account_id: &str, + table_name: &str, + backup_id: &str, + ) -> Result { + let component = |name: &str, value: &str, ok: fn(char) -> bool| { + if value.is_empty() || !value.chars().all(ok) { + return Err(FormatError::InvalidArn(format!("{name} {value:?}"))); + } + Ok(()) + }; + let lower_dash = |c: char| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-'; + let word = |c: char| c.is_ascii_alphanumeric() || c == '_' || c == '-'; + let name_chars = |c: char| c.is_ascii_alphanumeric() || c == '_' || c == '.' || c == '-'; + + component("partition", partition, lower_dash)?; + component("region", region, lower_dash)?; + component("account id", account_id, word)?; + if table_name.is_empty() || table_name.len() > MAX_TABLE_NAME_LEN { + return Err(FormatError::InvalidArn(format!( + "table name {table_name:?} length" + ))); + } + // One character rule for table names, shared with CreateTable. + crate::validation::validate_table_name_chars(table_name) + .map_err(|_| FormatError::InvalidArn(format!("table name {table_name:?}")))?; + component("backup id", backup_id, name_chars)?; + // `.` and `..` are path traversal; longer runs of dots are legal + // DynamoDB names and legal path components, but a name made only of + // dots is refused to keep the rule simple. + for (what, value) in [("table name", table_name), ("backup id", backup_id)] { + if value.bytes().all(|b| b == b'.') { + return Err(FormatError::InvalidArn(format!("{what} {value:?}"))); + } + } + Ok(Self { + partition: partition.to_owned(), + region: region.to_owned(), + account_id: account_id.to_owned(), + table_name: table_name.to_owned(), + backup_id: backup_id.to_owned(), + }) + } + + /// Parse an ARN of the shape + /// `arn::dynamodb:::table/
/backup/` + /// and validate its components with the rules of [`Self::new`]. + /// + /// # Errors + /// + /// Returns [`FormatError::InvalidArn`] when the string does not have that + /// shape or a component fails validation. + pub fn parse(arn: &str) -> Result { + let invalid = || FormatError::InvalidArn(arn.to_owned()); + + let rest = arn.strip_prefix("arn:").ok_or_else(invalid)?; + let mut fields = rest.splitn(5, ':'); + let partition = fields.next().ok_or_else(invalid)?; + let service = fields.next().ok_or_else(invalid)?; + let region = fields.next().ok_or_else(invalid)?; + let account_id = fields.next().ok_or_else(invalid)?; + let resource = fields.next().ok_or_else(invalid)?; + if service != "dynamodb" { + return Err(invalid()); + } + + let path = resource.strip_prefix("table/").ok_or_else(invalid)?; + let (table_name, backup_id) = path.split_once("/backup/").ok_or_else(invalid)?; + + // Keep the component-level reason: a mistyped ARN should say which + // part is wrong. + Self::new(partition, region, account_id, table_name, backup_id).map_err(|e| match e { + FormatError::InvalidArn(reason) => FormatError::InvalidArn(format!("{arn}: {reason}")), + other => other, + }) + } + + /// The ARN string. + #[must_use] + pub fn to_arn_string(&self) -> String { + format!( + "arn:{}:dynamodb:{}:{}:table/{}/backup/{}", + self.partition, self.region, self.account_id, self.table_name, self.backup_id + ) + } + + /// The store key prefix for this backup: + /// `///`. + #[must_use] + pub fn store_prefix(&self) -> String { + format!( + "{}/{}/{}/", + self.account_id, self.table_name, self.backup_id + ) + } + + /// Partition field (`aws`, `aws-cn`, `aws-us-gov`). + #[must_use] + pub fn partition(&self) -> &str { + &self.partition + } + + /// Region field. + #[must_use] + pub fn region(&self) -> &str { + &self.region + } + + /// Account id field. + #[must_use] + pub fn account_id(&self) -> &str { + &self.account_id + } + + /// Table name from the resource path. + #[must_use] + pub fn table_name(&self) -> &str { + &self.table_name + } + + /// Backup id from the resource path. + #[must_use] + pub fn backup_id(&self) -> &str { + &self.backup_id + } +} + +impl std::fmt::Display for BackupArn { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.to_arn_string()) + } +} + +impl std::str::FromStr for BackupArn { + type Err = FormatError; + + fn from_str(s: &str) -> Result { + Self::parse(s) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const ARN: &str = + "arn:aws:dynamodb:us-east-1:123456789012:table/Music/backup/01489602797149-73d8d5bc"; + + #[test] + fn store_prefix_shape() { + let arn = BackupArn::parse(ARN).unwrap(); + assert_eq!( + arn.store_prefix(), + "123456789012/Music/01489602797149-73d8d5bc/" + ); + } + + #[test] + fn data_file_key_is_zero_padded() { + assert_eq!(data_file_key(1), "data/000001.json.gz"); + assert_eq!(data_file_key(42), "data/000042.json.gz"); + assert_eq!(data_file_key(1_000_000), "data/1000000.json.gz"); + } + + #[test] + fn parse_valid_arn() { + let arn = BackupArn::parse(ARN).unwrap(); + assert_eq!(arn.partition(), "aws"); + assert_eq!(arn.region(), "us-east-1"); + assert_eq!(arn.account_id(), "123456789012"); + assert_eq!(arn.table_name(), "Music"); + assert_eq!(arn.backup_id(), "01489602797149-73d8d5bc"); + assert_eq!(arn.to_arn_string(), ARN); + assert_eq!(arn.to_string(), ARN); + assert_eq!(ARN.parse::().unwrap(), arn); + } + + #[test] + fn other_partitions_parse() { + for arn in [ + "arn:aws-cn:dynamodb:cn-north-1:123456789012:table/Music/backup/1-a", + "arn:aws-us-gov:dynamodb:us-gov-west-1:123456789012:table/Music/backup/1-a", + ] { + let parsed = BackupArn::parse(arn).unwrap(); + assert_eq!(parsed.to_arn_string(), arn); + } + } + + #[test] + fn new_and_parse_agree() { + let built = BackupArn::new( + "aws", + "eu-west-1", + "000000000001", + "My.Table-2", + "0123-abcd", + ) + .unwrap(); + let parsed = BackupArn::parse(&built.to_arn_string()).unwrap(); + assert_eq!(parsed, built); + assert_eq!(built.store_prefix(), "000000000001/My.Table-2/0123-abcd/"); + } + + #[test] + fn parse_rejects_malformed_arns() { + for arn in [ + "", + "arn:aws:dynamodb", + "arn:aws:dynamodb:us-east-1", + "arn:aws:dynamodb:us-east-1:123456789012", + "arn:aws:dynamodb:us-east-1:123456789012:table/Music", + "arn:aws:dynamodb:us-east-1::table/Music/backup/1", + "arn:aws:dynamodb::123456789012:table/Music/backup/1", + "arn:aws:dynamodb:us-east-1:123456789012:table//backup/1", + "arn:aws:dynamodb:us-east-1:123456789012:table/Music/backup/", + "arn:aws:dynamodb:us-east-1:123456789012:index/Music/backup/1", + "arn:aws:dynamodb:us-east-1:123456789012:table/Music/backup/a/b", + "arn:aws:s3:us-east-1:123456789012:table/Music/backup/1", + "arn::dynamodb:us-east-1:123456789012:table/Music/backup/1", + "arn:AWS:dynamodb:us-east-1:123456789012:table/Music/backup/1", + // Extra colon fields: the account must be field 4, nothing else. + "arn:aws:dynamodb:us-east-1:evil:123456789012:table/Music/backup/1", + // Traversal and separators inside components. + "arn:aws:dynamodb:us-east-1:123456789012:table/../backup/1", + "arn:aws:dynamodb:us-east-1:123456789012:table/Music/backup/..", + "arn:aws:dynamodb:us-east-1:123456789012:table/Music/backup/a\\b", + "arn:aws:dynamodb:us-east-1:12345678 9012:table/Music/backup/1", + "arn:aws:dynamodb:us-east-1:123456789012:table/Mu sic/backup/1", + "arn:aws:dynamodb:us-east-1:123456789012:table/Music/backup/1\n", + // Dot-only names are traversal components. + "arn:aws:dynamodb:us-east-1:123456789012:table/./backup/1", + "arn:aws:dynamodb:us-east-1:123456789012:table/..../backup/1", + ] { + assert!( + BackupArn::parse(arn).is_err(), + "expected rejection for {arn:?}" + ); + } + let long = "a".repeat(256); + assert!( + BackupArn::parse(&format!( + "arn:aws:dynamodb:us-east-1:123456789012:table/{long}/backup/1" + )) + .is_err() + ); + } + + #[test] + fn new_rejects_bad_components() { + assert!(BackupArn::new("aws", "us-east-1", "1234", "Music", "a/b").is_err()); + assert!(BackupArn::new("aws", "us-east-1", "12/34", "Music", "1").is_err()); + assert!(BackupArn::new("aws", "us-east-1", "1234", "Music", "...").is_err()); + assert!(BackupArn::new("aws", "us-east-1", "1234", "...", "1").is_err()); + // Short names are the engine's call (its minimum length is + // configurable); the ARN accepts them. + BackupArn::new("aws", "us-east-1", "1234", "ab", "1").unwrap(); + // The error names the failing component. + let err = BackupArn::parse("arn:aws:dynamodb:us-east-1:123456789012:table/Mu sic/backup/1") + .unwrap_err(); + assert!(err.to_string().contains("table name"), "{err}"); + assert!(BackupArn::new("aws", "us:east", "1234", "Music", "1").is_err()); + assert!(BackupArn::new("", "us-east-1", "1234", "Music", "1").is_err()); + // Dots inside names are fine; DynamoDB allows them. + BackupArn::new("aws", "us-east-1", "1234", "a.b.c", "1.2").unwrap(); + } + + #[test] + fn fixed_object_names() { + assert_eq!(EXTENDDB_MANIFEST_FILE, "extenddb-manifest.json"); + assert_eq!(MANIFEST_CHECKSUM_FILE, "extenddb-manifest.sha256"); + assert_eq!(SUMMARY_MANIFEST_FILE, "manifest-summary.json"); + assert_eq!(FILES_MANIFEST_FILE, "manifest-files.json"); + assert_eq!(DATA_PREFIX, "data/"); + } +} diff --git a/crates/core/src/backup_format/manifest.rs b/crates/core/src/backup_format/manifest.rs new file mode 100644 index 000000000..c2cc857cf --- /dev/null +++ b/crates/core/src/backup_format/manifest.rs @@ -0,0 +1,1174 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The `extenddb-manifest.json` types: the `ExtendDB` extension to the +//! `DynamoDB` export layout, and the commit record of a backup. +//! +//! The manifest is written last; its presence marks a complete backup. It +//! carries everything a restore needs that the `DynamoDB` export manifests do +//! not: the full source table definition, the snapshot descriptor, the item +//! order guarantee, and a sha256 per data file. +//! +//! Integrity of the manifest itself is a sidecar object, +//! `extenddb-manifest.sha256`, in `sha256sum` format: the lowercase hex +//! sha256 of the exact bytes of `extenddb-manifest.json`, two spaces, the +//! file name, newline. The writer computes it from the bytes it is about to +//! store and writes the sidecar before the manifest; a reader hashes the +//! bytes it fetched and compares, and an operator can run +//! `sha256sum -c extenddb-manifest.sha256`. Hashing stored bytes avoids +//! defining a canonical JSON form: an earlier revision hashed a re-serialized +//! struct with the checksum field blanked, which tied the digest to one JSON +//! library's float formatting and field order and would have failed for any +//! field an older reader did not model. +//! +//! Optional members of [`SourceTableSchema`] are always serialized, as `null` +//! when absent, for human readers and non-Rust tools; index lists are plain +//! arrays, empty when the table has none. This build's reader accepts a +//! missing member as `null` (`#[serde(default)]`), so "the writer did not +//! know this field" is signalled by `format_version`, not by absence. +//! `vector_indexes` is the one member whose `null` carries meaning, see its +//! docs. Nested members reuse the `DynamoDB` wire types, whose own optional +//! members are omitted when absent, as on the wire; the checked-in golden +//! manifest under `testdata/extenddb-manifest/` pins the exact bytes so a +//! change to one of those shared types fails a test here. +//! +//! Casing: top-level and `source_table` member names are `snake_case`; +//! `item_order` values are `snake_case`; members of the nested wire types +//! and their enum values (`PROVISIONED`, `HASH`) keep the `DynamoDB` API +//! spelling. Frozen with format version 1. + +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +use super::FormatError; +use super::data_file::hex_encode; +use super::key_order::KeyOrder; +use super::layout::{BackupArn, DATA_PREFIX, EXTENDDB_MANIFEST_FILE}; +use crate::types::{ + AttributeDefinition, BillingMode, GsiInput, Item, KeySchemaElement, LsiInput, + OnDemandThroughput, ProvisionedThroughput, SseDescription, VectorIndexSpecification, +}; + +/// The backup format version this build reads and writes. +pub const CURRENT_FORMAT_VERSION: u32 = 1; + +/// How the source backend took its consistent snapshot. +/// +/// `kind` names the mechanism (for example `postgres_repeatable_read`) and +/// `marker` records the backend-specific position, kept for diagnostics and +/// for a later change-log archive to anchor against. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct SnapshotDescriptor { + /// The snapshot mechanism, named by the backend that took it. + pub kind: String, + /// Backend-specific snapshot position. + pub marker: String, +} + +/// Order of items within and across a backup's data files. +/// +/// A restore may exploit `KeyOrdered` (SQLite inserts into a fresh table 3 to +/// 4 times faster when rows arrive in primary-key order; measured with the +/// bundled SQLite on a 1M-item fixture, ratio only) but must produce the same +/// table either way. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ItemOrder { + /// Items ascend in the order [`super::key_order::KeyOrder`] defines (hash + /// key by its text encoding, bytewise; range key `S`/`B` bytewise, `N` + /// numeric), strictly within each file and from one file to the next; + /// every [`DataFileEntry::key_range`] is set and the ranges do not + /// overlap. Only hash-only and single-range-key tables can be ordered. + KeyOrdered, + /// No ordering guarantee. + Unordered, +} + +/// The source table definition captured at backup time. +/// +/// Field types reuse the existing wire types, so the nested JSON matches what +/// the `DynamoDB` API itself carries (`PascalCase` members inside each element). +/// Timestamps are `f64` epoch seconds, the same representation +/// [`crate::types::BackupDetails`] uses. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct SourceTableSchema { + /// Name of the source table. + pub table_name: String, + /// Unique id of the source table. + pub table_id: String, + /// ARN of the source table. + pub table_arn: String, + /// When the source table was created, as epoch seconds. + pub table_creation_date_time: f64, + /// Base table key schema. + pub key_schema: Vec, + /// Attribute definitions for all key attributes. + pub attribute_definitions: Vec, + /// Billing mode at backup time. + pub billing_mode: BillingMode, + /// Provisioned throughput; `null` when the table is on demand. + #[serde(default)] + pub provisioned_throughput: Option, + /// On-demand throughput limits; `null` when none are configured. + #[serde(default)] + pub on_demand_throughput: Option, + /// Table class; `null` when the source reported none. + #[serde(default)] + pub table_class: Option, + /// Server-side encryption as `DescribeTable` reports it; `null` when the + /// source reported none. A restore maps it to an `SSESpecification`. + #[serde(default)] + pub sse_description: Option, + /// Global secondary indexes defined on the source table; empty when none. + #[serde(default)] + pub global_secondary_indexes: Vec, + /// Local secondary indexes defined on the source table; empty when none. + #[serde(default)] + pub local_secondary_indexes: Vec, + /// Vector indexes, as creation specifications. Three states: `null`, the + /// source backend has no vector support (a target with vector support + /// restores the table without vector indexes); `[]`, the source supports + /// vector search and the table had no vector indexes; a non-empty list, + /// which a target without vector support refuses to restore rather than + /// dropping them. + #[serde(default)] + pub vector_indexes: Option>, + /// Item count at backup time. + pub item_count: i64, + /// Table size in bytes at backup time. + pub table_size_bytes: i64, +} + +/// First and last base-table key in a data file, as `DynamoDB` JSON items +/// holding exactly the key attributes (enforced by +/// [`ExtenddbManifest::validate_data_files`]). Required when the backup is +/// [`ItemOrder::KeyOrdered`] and refused when it is unordered; lets a restore +/// reject overlapping files (a duplicate primary key across files) before +/// downloading any of them. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct KeyRange { + /// Key of the first item in the file. + pub first: Item, + /// Key of the last item in the file. + pub last: Item, +} + +/// One data file recorded in the manifest. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct DataFileEntry { + /// Store key of the file, relative to the backup prefix + /// (for example `data/000001.json.gz`). + pub key: String, + /// Number of items in the file. + pub item_count: u64, + /// Lowercase hex sha256 of the file's compressed bytes. + pub sha256: String, + /// Base64 MD5 of the file's compressed bytes; the same value the + /// `manifest-files.json` line for this file carries as `md5Checksum`. + pub md5: String, + /// Size of the file's compressed bytes. + pub bytes: u64, + /// Size of the file's uncompressed lines, newlines included. A decoder + /// enforces it incrementally, so a corrupt or hostile file is rejected + /// before it is fully expanded. + pub uncompressed_bytes: u64, + /// Key range covered by the file. Required for a key-ordered backup and + /// must be `null` for an unordered one; both enforced by + /// [`ExtenddbManifest::validate_data_files`]. + #[serde(default)] + pub key_range: Option, +} + +impl DataFileEntry { + /// Validate that `key` names a data file within the backup's `data/` + /// prefix and nowhere else. + /// + /// The key must be [`DATA_PREFIX`] followed by exactly one path segment + /// that matches the data-file naming pattern (`.json.gz`). This + /// rejects `..`, absolute paths, and any extra directory level, so a + /// malicious manifest cannot point the restore path at a store key outside + /// the backup. Validation lives in the format layer, the natural home for + /// the layout rule, independent of any store-level sandboxing. + /// + /// # Errors + /// + /// Returns [`FormatError::InvalidDataFileKey`] when the key is not a + /// single data-file entry under the prefix. + pub fn validate_key(&self) -> Result<(), FormatError> { + let reject = |reason: &'static str| FormatError::InvalidDataFileKey { + key: self.key.clone(), + reason, + }; + + let name = self + .key + .strip_prefix(DATA_PREFIX) + .ok_or_else(|| reject("must start with the data/ prefix"))?; + + if name.is_empty() { + return Err(reject("names no file under data/")); + } + // Exactly one segment: no further path separators, no traversal, no + // absolute component. + if name.contains('/') { + return Err(reject("must hold exactly one path segment under data/")); + } + if name.contains('\\') { + return Err(reject("must not contain a backslash")); + } + if name == "." || name == ".." || name.contains("..") { + return Err(reject("must not contain a path-traversal component")); + } + // The data-file naming pattern: one or more digits then .json.gz. + let digits = name + .strip_suffix(".json.gz") + .ok_or_else(|| reject("must end with .json.gz"))?; + if digits.is_empty() || !digits.bytes().all(|b| b.is_ascii_digit()) { + return Err(reject("file stem must be all digits")); + } + Ok(()) + } + + /// The expectations a decoder enforces for this file: every digest and + /// count the entry records. + #[must_use] + pub fn expectations(&self) -> super::data_file::DataFileExpectations { + super::data_file::DataFileExpectations { + sha256_hex: Some(self.sha256.clone()), + md5_base64: Some(self.md5.clone()), + item_count: Some(self.item_count), + uncompressed_bytes: Some(self.uncompressed_bytes), + } + } + + /// Streaming decoder over this entry's data file read from `compressed`, + /// with the entry's digests and counts as expectations. Validates the key + /// first. This is the restore path's entry point; the digests are checked + /// when the stream ends, so a caller that must know them earlier hashes a + /// local copy with [`crate::backup_format::CompressedHasher`] first. + /// + /// # Errors + /// + /// Returns [`FormatError::InvalidDataFileKey`] when the key is not a + /// single data-file entry under the prefix. + pub fn decoder( + &self, + compressed: R, + ) -> Result, FormatError> { + self.validate_key()?; + Ok(super::data_file::DataFileDecoder::new( + compressed, + self.expectations(), + )) + } + + /// Decode this entry's in-memory data file into a `Vec`, validating the + /// key first and verifying the digests before the first item. For tests + /// and files known to be small; see [`Self::decoder`] for the streaming + /// form. + /// + /// # Errors + /// + /// Returns [`FormatError::InvalidDataFileKey`] when the key is not a + /// single data-file entry under the prefix, and any error + /// [`crate::backup_format::decode_data_file`] can produce. + pub fn decode(&self, compressed: &[u8]) -> Result, FormatError> { + self.validate_key()?; + super::data_file::decode_data_file(compressed, self.expectations()) + } +} + +/// The `extenddb-manifest.json` document. +/// +/// Serde leaves unknown fields tolerated (no `deny_unknown_fields`), so a +/// newer writer can add members without breaking an older reader; the hard +/// gate is [`Self::check_supported`], which refuses a `format_version` this +/// build does not understand. Because the integrity check hashes stored bytes +/// (see the module docs), adding an optional field does not by itself require +/// a version bump; changing the meaning of an existing field or adding a +/// field a reader must understand to restore correctly does. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ExtenddbManifest { + /// Format version; see [`CURRENT_FORMAT_VERSION`]. + pub format_version: u32, + /// ARN of the backup, in the shape + /// `arn::dynamodb:::table/
/backup/`. + pub backup_arn: String, + /// Name given to the backup at creation. + pub backup_name: String, + /// When the backup was requested, as epoch seconds, the representation + /// [`crate::types::BackupDetails`] uses. The snapshot the data files hold + /// was taken at this time (see the design doc on request-time snapshots). + pub backup_creation_date_time: f64, + /// Total size of the backup's data files, in compressed bytes. + pub backup_size_bytes: i64, + /// The storage backend that produced the backup. + pub source_backend: String, + /// The `ExtendDB` version that produced the backup. + pub source_extenddb_version: String, + /// How the consistent snapshot was taken. + pub snapshot: SnapshotDescriptor, + /// Order of items in the data files. + pub item_order: ItemOrder, + /// The source table definition. + pub source_table: SourceTableSchema, + /// The data files that make up the backup, in the order a restore applies + /// them. This order is authoritative; a store listing is never used to + /// order files (zero-padding in [`super::data_file_key`] runs out at a + /// million files). + pub data_files: Vec, +} + +/// The content of the `extenddb-manifest.sha256` sidecar for a stored +/// manifest: ` extenddb-manifest.json\n`, the +/// `sha256sum` output format. +#[must_use] +pub fn manifest_checksum(manifest_bytes: &[u8]) -> String { + format!( + "{} {EXTENDDB_MANIFEST_FILE}\n", + hex_encode(&Sha256::digest(manifest_bytes)) + ) +} + +/// Compare fetched manifest bytes against the sidecar's content. The digest +/// is the first whitespace-delimited token, compared case-insensitively, so a +/// bare digest and `sha256sum` output are both accepted. +/// +/// # Errors +/// +/// Returns [`FormatError::ChecksumMismatch`] when the digests differ or the +/// sidecar is empty. +pub fn verify_manifest_checksum(manifest_bytes: &[u8], sidecar: &str) -> Result<(), FormatError> { + let expected = sidecar + .split_whitespace() + .next() + .unwrap_or("") + .to_ascii_lowercase(); + let computed = hex_encode(&Sha256::digest(manifest_bytes)); + if expected == computed { + Ok(()) + } else { + Err(FormatError::ChecksumMismatch { + subject: "manifest", + expected, + computed, + }) + } +} + +/// The members a reader needs before it can decide whether to parse the +/// rest: the version gate runs on this so a document from a later version +/// fails with [`FormatError::UnsupportedFormatVersion`] rather than with a +/// parse error about a member this build does not know. +#[derive(Deserialize)] +struct VersionOnly { + format_version: u32, +} + +impl ExtenddbManifest { + /// Serialize to the bytes stored as `extenddb-manifest.json`. The writer + /// hashes exactly these bytes for the sidecar. + /// + /// # Errors + /// + /// Returns [`FormatError::Json`] when serialization fails. + pub fn to_json_bytes(&self) -> Result, FormatError> { + Ok(serde_json::to_vec(self)?) + } + + /// Parse stored manifest bytes: the version gate first, then the full + /// document, then the identity check ([`Self::validate_identity`]) and + /// the data-file list checks ([`Self::validate_data_files`]). Callers + /// verify the sidecar checksum against the same bytes first; see + /// [`verify_manifest_checksum`]. + /// + /// # Errors + /// + /// Returns [`FormatError::UnsupportedFormatVersion`] when the version is + /// not [`CURRENT_FORMAT_VERSION`] (checked before anything else is + /// parsed), [`FormatError::Json`] when the bytes do not parse, and the + /// errors of [`Self::validate_identity`] and + /// [`Self::validate_data_files`]. + pub fn from_json_bytes(bytes: &[u8]) -> Result { + // The gate reads only `format_version`, so a document from a later + // version fails with that reason instead of a deserialization error + // about a member this build does not know. + let version: VersionOnly = serde_json::from_slice(bytes)?; + if version.format_version != CURRENT_FORMAT_VERSION { + return Err(FormatError::UnsupportedFormatVersion { + found: version.format_version, + supported: CURRENT_FORMAT_VERSION, + }); + } + let manifest: Self = serde_json::from_slice(bytes)?; + manifest.validate_identity()?; + manifest.validate_data_files()?; + Ok(manifest) + } + + /// The parsed backup ARN. + /// + /// # Errors + /// + /// Returns [`FormatError::InvalidArn`] when `backup_arn` is malformed. + pub fn backup_arn(&self) -> Result { + BackupArn::parse(&self.backup_arn) + } + + /// Check that the manifest names one table: `backup_arn` parses, its + /// table name equals `source_table.table_name`, and + /// `source_table.table_arn` is a table ARN for the same account and + /// table. `backup_arn` is the authoritative identity; the source-table + /// fields are checked against it so a restore never has two answers to + /// "which table is this". The table ARN's region is not compared: it is + /// recorded when the table is created and the backup ARN carries the + /// region configured when the backup is taken. + /// + /// # Errors + /// + /// [`FormatError::InvalidArn`] when `backup_arn` is malformed, + /// [`FormatError::InvalidManifest`] when the source-table fields disagree + /// with it. + pub fn validate_identity(&self) -> Result<(), FormatError> { + let arn = self.backup_arn()?; + if arn.table_name() != self.source_table.table_name { + return Err(FormatError::InvalidManifest(format!( + "backup_arn names table {:?} but source_table.table_name is {:?}", + arn.table_name(), + self.source_table.table_name + ))); + } + let table_arn = &self.source_table.table_arn; + let fields: Vec<&str> = table_arn.splitn(6, ':').collect(); + let matches = match fields.as_slice() { + ["arn", _partition, "dynamodb", _region, account, resource] => { + *account == arn.account_id() + && resource + .strip_prefix("table/") + .is_some_and(|name| name == arn.table_name()) + } + _ => false, + }; + if !matches { + return Err(FormatError::InvalidManifest(format!( + "source_table.table_arn {table_arn:?} is not a table ARN for account {} and \ + table {:?} named by backup_arn", + arn.account_id(), + arn.table_name() + ))); + } + Ok(()) + } + + /// The key order of the source table, when it has one. + /// + /// # Errors + /// + /// Returns [`FormatError::UnorderableKeySchema`] for a multi-part key. + pub fn key_order(&self) -> Result { + KeyOrder::resolve( + &self.source_table.key_schema, + &self.source_table.attribute_definitions, + ) + } + + /// Refuse a manifest whose format version this build does not understand. + /// + /// # Errors + /// + /// Returns [`FormatError::UnsupportedFormatVersion`] when `format_version` + /// differs from [`CURRENT_FORMAT_VERSION`]. + pub fn check_supported(&self) -> Result<(), FormatError> { + if self.format_version == CURRENT_FORMAT_VERSION { + Ok(()) + } else { + Err(FormatError::UnsupportedFormatVersion { + found: self.format_version, + supported: CURRENT_FORMAT_VERSION, + }) + } + } + + /// Validate the data-file list: every key passes the layout rule and + /// appears once; `backup_size_bytes` equals the sum of the files' sizes; + /// for a key-ordered manifest every file carries a `key_range` whose + /// `first` and `last` are key-only items of the schema's shape with + /// `first <= last`, and the ranges ascend without overlap from one file + /// to the next; for an unordered manifest no file carries a `key_range` + /// (there is no order for it to describe, and a reader must not be + /// handed ranges it cannot check). + /// + /// Callers run this when a manifest is parsed, and the decode path runs + /// the same per-entry [`DataFileEntry::validate_key`] again before it + /// resolves a key, so a key that slips past one check is caught at the + /// other. Within-file order and duplicates are checked by the restore + /// while it streams the file; this check covers what the manifest alone + /// can prove. + /// + /// # Errors + /// + /// [`FormatError::InvalidDataFileKey`] for a bad or duplicate key, a + /// missing range on a key-ordered manifest, or a range on an unordered + /// one; [`FormatError::InvalidManifest`] for a size mismatch or a range + /// violation; [`FormatError::InvalidKey`] for a malformed range endpoint; + /// [`FormatError::UnorderableKeySchema`] for a key-ordered manifest whose + /// schema has no order. + pub fn validate_data_files(&self) -> Result<(), FormatError> { + let mut seen = std::collections::BTreeSet::new(); + let mut total: u64 = 0; + for entry in &self.data_files { + entry.validate_key()?; + if !seen.insert(entry.key.as_str()) { + return Err(FormatError::InvalidDataFileKey { + key: entry.key.clone(), + reason: "listed more than once", + }); + } + if self.item_order == ItemOrder::Unordered && entry.key_range.is_some() { + return Err(FormatError::InvalidDataFileKey { + key: entry.key.clone(), + reason: "unordered manifest entry has a key_range", + }); + } + total = total.saturating_add(entry.bytes); + } + if i64::try_from(total).ok() != Some(self.backup_size_bytes) { + return Err(FormatError::InvalidManifest(format!( + "backup_size_bytes is {} but the data files total {total}", + self.backup_size_bytes + ))); + } + if self.item_order == ItemOrder::KeyOrdered { + let order = self.key_order()?; + let mut previous_last: Option<&Item> = None; + for entry in &self.data_files { + let range = + entry + .key_range + .as_ref() + .ok_or_else(|| FormatError::InvalidDataFileKey { + key: entry.key.clone(), + reason: "key_ordered manifest entry has no key_range", + })?; + order.validate_key_item(&range.first)?; + order.validate_key_item(&range.last)?; + if order.compare(&range.first, &range.last)? == std::cmp::Ordering::Greater { + return Err(FormatError::InvalidManifest(format!( + "{}: key_range first is after last", + entry.key + ))); + } + if let Some(prev) = previous_last + && order.compare(prev, &range.first)? != std::cmp::Ordering::Less + { + return Err(FormatError::InvalidManifest(format!( + "{}: key_range overlaps or does not follow the previous file", + entry.key + ))); + } + previous_last = Some(&range.last); + } + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::types::{KeyType, Projection, ProjectionType, ScalarAttributeType}; + + #[allow(clippy::too_many_lines)] + fn sample_manifest() -> ExtenddbManifest { + ExtenddbManifest { + format_version: CURRENT_FORMAT_VERSION, + backup_arn: + "arn:aws:dynamodb:us-east-1:123456789012:table/Music/backup/01489602797149-73d8d5bc" + .to_owned(), + backup_name: "music-nightly".to_owned(), + backup_creation_date_time: 1_757_500_000.123, + backup_size_bytes: 61_728, + source_backend: "postgres".to_owned(), + source_extenddb_version: "0.1.11".to_owned(), + snapshot: SnapshotDescriptor { + kind: "postgres_repeatable_read".to_owned(), + marker: "1234:1234:".to_owned(), + }, + source_table: SourceTableSchema { + table_name: "Music".to_owned(), + table_id: "12345a12-abcd-123a-ab12-1234abc12345".to_owned(), + table_arn: "arn:aws:dynamodb:us-east-1:123456789012:table/Music".to_owned(), + table_creation_date_time: 1_757_400_000.0, + key_schema: vec![ + KeySchemaElement { + attribute_name: "Artist".to_owned(), + key_type: KeyType::Hash, + }, + KeySchemaElement { + attribute_name: "SongTitle".to_owned(), + key_type: KeyType::Range, + }, + ], + attribute_definitions: vec![ + AttributeDefinition { + attribute_name: "Artist".to_owned(), + attribute_type: ScalarAttributeType::S, + }, + AttributeDefinition { + attribute_name: "SongTitle".to_owned(), + attribute_type: ScalarAttributeType::S, + }, + AttributeDefinition { + attribute_name: "Genre".to_owned(), + attribute_type: ScalarAttributeType::S, + }, + ], + billing_mode: BillingMode::Provisioned, + provisioned_throughput: Some(ProvisionedThroughput { + read_capacity_units: 5, + write_capacity_units: 5, + }), + on_demand_throughput: None, + table_class: Some("STANDARD".to_owned()), + sse_description: Some(SseDescription { + status: "ENABLED".to_owned(), + sse_type: Some(crate::types::SseType::KMS), + kms_master_key_arn: Some( + "arn:aws:kms:us-east-1:123456789012:key/abcd".to_owned(), + ), + }), + global_secondary_indexes: vec![GsiInput { + index_name: "GenreIndex".to_owned(), + key_schema: vec![KeySchemaElement { + attribute_name: "Genre".to_owned(), + key_type: KeyType::Hash, + }], + projection: Projection { + projection_type: ProjectionType::All, + non_key_attributes: None, + }, + provisioned_throughput: Some(ProvisionedThroughput { + read_capacity_units: 1, + write_capacity_units: 1, + }), + }], + local_secondary_indexes: vec![LsiInput { + index_name: "TitleIndex".to_owned(), + key_schema: vec![ + KeySchemaElement { + attribute_name: "Artist".to_owned(), + key_type: KeyType::Hash, + }, + KeySchemaElement { + attribute_name: "SongTitle".to_owned(), + key_type: KeyType::Range, + }, + ], + projection: Projection { + projection_type: ProjectionType::KeysOnly, + non_key_attributes: None, + }, + }], + vector_indexes: Some(vec![VectorIndexSpecification { + index_name: "embeddings".to_owned(), + dimensions: 1024, + distance_function: crate::types::DistanceFunction::Cosine, + vector_attribute: crate::types::VectorAttribute { + attribute_name: "vec".to_owned(), + }, + search_schema: None, + projection: None, + }]), + item_count: 100_000, + table_size_bytes: 123_456, + }, + item_order: ItemOrder::Unordered, + data_files: vec![DataFileEntry { + key: "data/000001.json.gz".to_owned(), + item_count: 50_000, + sha256: "0".repeat(64), + md5: "1B2M2Y8AsgTpgAmY7PhCfg==".to_owned(), + bytes: 61_728, + uncompressed_bytes: 300_000, + key_range: None, + }], + } + } + + #[test] + fn manifest_serde_round_trip() { + let manifest = sample_manifest(); + let bytes = manifest.to_json_bytes().unwrap(); + let parsed = ExtenddbManifest::from_json_bytes(&bytes).unwrap(); + assert_eq!(parsed, manifest); + } + + #[test] + fn snapshot_descriptor_serde_round_trip() { + let snapshot = SnapshotDescriptor { + kind: "sqlite_wal_read_txn".to_owned(), + marker: String::new(), + }; + let json = serde_json::to_string(&snapshot).unwrap(); + let parsed: SnapshotDescriptor = serde_json::from_str(&json).unwrap(); + assert_eq!(parsed, snapshot); + } + + #[test] + fn source_table_schema_serde_round_trip() { + let schema = sample_manifest().source_table; + let json = serde_json::to_string(&schema).unwrap(); + let parsed: SourceTableSchema = serde_json::from_str(&json).unwrap(); + assert_eq!(parsed, schema); + } + + #[test] + fn data_file_entry_serde_round_trip() { + let entry = DataFileEntry { + key: "data/000042.json.gz".to_owned(), + item_count: 7, + sha256: "ab".repeat(32), + md5: "cd".repeat(11), + bytes: 1234, + uncompressed_bytes: 5678, + key_range: Some(KeyRange { + first: [( + "pk".to_owned(), + crate::types::AttributeValue::S("a".to_owned()), + )] + .into_iter() + .collect(), + last: [( + "pk".to_owned(), + crate::types::AttributeValue::S("z".to_owned()), + )] + .into_iter() + .collect(), + }), + }; + let json = serde_json::to_string(&entry).unwrap(); + let parsed: DataFileEntry = serde_json::from_str(&json).unwrap(); + assert_eq!(parsed, entry); + } + + #[test] + fn absent_optional_members_serialize_as_null() { + let mut manifest = sample_manifest(); + manifest.source_table.on_demand_throughput = None; + manifest.source_table.vector_indexes = None; + manifest.source_table.table_class = None; + manifest.source_table.sse_description = None; + manifest.source_table.global_secondary_indexes.clear(); + let value = serde_json::to_value(&manifest).unwrap(); + let table = &value["source_table"]; + assert!(table["on_demand_throughput"].is_null()); + assert!(table["vector_indexes"].is_null()); + assert!(table["table_class"].is_null()); + assert!(table["sse_description"].is_null()); + assert_eq!(table["global_secondary_indexes"], serde_json::json!([])); + assert!(value["data_files"][0]["key_range"].is_null()); + } + + #[test] + fn vector_indexes_distinguish_unsupported_from_none() { + let mut manifest = sample_manifest(); + manifest.source_table.vector_indexes = Some(vec![]); + let bytes = manifest.to_json_bytes().unwrap(); + let parsed = ExtenddbManifest::from_json_bytes(&bytes).unwrap(); + assert_eq!(parsed.source_table.vector_indexes, Some(vec![])); + manifest.source_table.vector_indexes = None; + let bytes = manifest.to_json_bytes().unwrap(); + let parsed = ExtenddbManifest::from_json_bytes(&bytes).unwrap(); + assert_eq!(parsed.source_table.vector_indexes, None); + } + + fn key(artist: &str, title: &str) -> Item { + [ + ( + "Artist".to_owned(), + crate::types::AttributeValue::S(artist.to_owned()), + ), + ( + "SongTitle".to_owned(), + crate::types::AttributeValue::S(title.to_owned()), + ), + ] + .into_iter() + .collect() + } + + fn second_file(first: Item, last: Item) -> DataFileEntry { + DataFileEntry { + key: "data/000002.json.gz".to_owned(), + item_count: 1, + sha256: "1".repeat(64), + md5: "x".to_owned(), + bytes: 100, + uncompressed_bytes: 200, + key_range: Some(KeyRange { first, last }), + } + } + + #[test] + fn key_ordered_manifest_requires_valid_non_overlapping_key_ranges() { + let mut manifest = sample_manifest(); + manifest.item_order = ItemOrder::KeyOrdered; + let err = manifest.validate_data_files().unwrap_err(); + assert!( + matches!(err, FormatError::InvalidDataFileKey { .. }), + "{err}" + ); + + // Endpoints must be key-only items of the schema's shape. + manifest.data_files[0].key_range = Some(KeyRange { + first: Item::new(), + last: Item::new(), + }); + assert!(matches!( + manifest.validate_data_files(), + Err(FormatError::InvalidKey(_)) + )); + let mut with_extra = key("a", "a"); + with_extra.insert( + "Genre".to_owned(), + crate::types::AttributeValue::S("x".into()), + ); + manifest.data_files[0].key_range = Some(KeyRange { + first: with_extra, + last: key("a", "b"), + }); + assert!(matches!( + manifest.validate_data_files(), + Err(FormatError::InvalidKey(_)) + )); + + // first must not be after last. + manifest.data_files[0].key_range = Some(KeyRange { + first: key("b", "a"), + last: key("a", "a"), + }); + assert!(matches!( + manifest.validate_data_files(), + Err(FormatError::InvalidManifest(_)) + )); + + // Valid single file. + manifest.data_files[0].key_range = Some(KeyRange { + first: key("a", "a"), + last: key("m", "z"), + }); + manifest.validate_data_files().unwrap(); + assert_eq!( + serde_json::to_value(&manifest).unwrap()["item_order"], + serde_json::json!("key_ordered") + ); + + // A second file must start strictly after the first ends. + manifest + .data_files + .push(second_file(key("m", "z"), key("z", "z"))); + manifest.backup_size_bytes += 100; + assert!(matches!( + manifest.validate_data_files(), + Err(FormatError::InvalidManifest(_)) + )); + manifest.data_files[1].key_range = Some(KeyRange { + first: key("n", "a"), + last: key("z", "z"), + }); + manifest.validate_data_files().unwrap(); + // Files out of order are rejected too. + manifest.data_files.swap(0, 1); + manifest.data_files[0].key = "data/000001.json.gz".to_owned(); + manifest.data_files[1].key = "data/000002.json.gz".to_owned(); + assert!(matches!( + manifest.validate_data_files(), + Err(FormatError::InvalidManifest(_)) + )); + } + + #[test] + fn duplicate_data_file_keys_are_rejected() { + let mut manifest = sample_manifest(); + let dup = manifest.data_files[0].clone(); + manifest.data_files.push(dup); + manifest.backup_size_bytes *= 2; + let err = manifest.validate_data_files().unwrap_err(); + assert!(matches!( + err, + FormatError::InvalidDataFileKey { + reason: "listed more than once", + .. + } + )); + } + + #[test] + fn backup_size_must_equal_the_sum_of_data_files() { + let mut manifest = sample_manifest(); + manifest.backup_size_bytes += 1; + assert!(matches!( + manifest.validate_data_files(), + Err(FormatError::InvalidManifest(_)) + )); + } + + #[test] + fn version_gate_runs_before_the_full_parse() { + // A later version with a shape this build cannot parse must still be + // reported as an unsupported version, not as a JSON error. + let bytes = br#"{"format_version": 7, "something_new": {"x": 1}}"#; + let err = ExtenddbManifest::from_json_bytes(bytes).unwrap_err(); + assert!( + matches!(err, FormatError::UnsupportedFormatVersion { found: 7, .. }), + "{err}" + ); + } + + #[test] + fn malformed_backup_arn_is_rejected_on_load() { + let mut manifest = sample_manifest(); + manifest.backup_arn = + "arn:aws:dynamodb:us-east-1:evil:1234:table/Music/backup/1".to_owned(); + let err = + ExtenddbManifest::from_json_bytes(&manifest.to_json_bytes().unwrap()).unwrap_err(); + assert!(matches!(err, FormatError::InvalidArn(_)), "{err}"); + } + + /// `backup_arn` is the identity; the source-table name and ARN must agree + /// with it. + #[test] + fn source_table_must_match_backup_arn() { + let good = sample_manifest(); + good.validate_identity().unwrap(); + + let mut other_name = good.clone(); + other_name.source_table.table_name = "Other".to_owned(); + let err = + ExtenddbManifest::from_json_bytes(&other_name.to_json_bytes().unwrap()).unwrap_err(); + assert!(matches!(err, FormatError::InvalidManifest(_)), "{err}"); + assert!(err.to_string().contains("source_table.table_name"), "{err}"); + + for bad_arn in [ + "arn:aws:dynamodb:us-east-1:123456789012:table/Other", + "arn:aws:dynamodb:us-east-1:999999999999:table/Music", + "arn:aws:s3:us-east-1:123456789012:table/Music", + "arn:aws:dynamodb:us-east-1:123456789012:table/Music/index/x", + "Music", + ] { + let mut m = good.clone(); + m.source_table.table_arn = bad_arn.to_owned(); + let err = m.validate_identity().unwrap_err(); + assert!( + matches!(err, FormatError::InvalidManifest(_)), + "{bad_arn}: {err}" + ); + assert!(err.to_string().contains("table_arn"), "{err}"); + } + // The table ARN's region is recorded at table creation and is not + // compared. + let mut other_region = good; + other_region.source_table.table_arn = + "arn:aws:dynamodb:eu-west-1:123456789012:table/Music".to_owned(); + other_region.validate_identity().unwrap(); + } + + #[test] + fn unordered_manifest_must_not_carry_key_ranges() { + let mut manifest = sample_manifest(); + assert_eq!(manifest.item_order, ItemOrder::Unordered); + manifest.data_files[0].key_range = Some(KeyRange { + first: Item::new(), + last: Item::new(), + }); + let err = manifest.validate_data_files().unwrap_err(); + assert!( + matches!( + err, + FormatError::InvalidDataFileKey { + reason: "unordered manifest entry has a key_range", + .. + } + ), + "{err}" + ); + } + + #[test] + fn unknown_fields_are_tolerated() { + let manifest = sample_manifest(); + let mut value = serde_json::to_value(&manifest).unwrap(); + value + .as_object_mut() + .unwrap() + .insert("added_in_a_later_version".to_owned(), serde_json::json!(42)); + let parsed: ExtenddbManifest = serde_json::from_value(value).unwrap(); + assert_eq!(parsed, manifest); + } + + #[test] + fn check_supported_accepts_current_version() { + let manifest = sample_manifest(); + manifest.check_supported().unwrap(); + } + + #[test] + fn check_supported_rejects_version_two() { + let mut manifest = sample_manifest(); + manifest.format_version = 2; + let err = manifest.check_supported().unwrap_err(); + match err { + FormatError::UnsupportedFormatVersion { found, supported } => { + assert_eq!(found, 2); + assert_eq!(supported, CURRENT_FORMAT_VERSION); + } + other => panic!("unexpected error: {other}"), + } + } + + #[test] + fn sidecar_checksum_round_trip() { + let bytes = sample_manifest().to_json_bytes().unwrap(); + let sidecar = manifest_checksum(&bytes); + // sha256sum format: 64 hex, two spaces, the file name, newline. + assert!( + sidecar.ends_with(" extenddb-manifest.json\n"), + "{sidecar:?}" + ); + assert_eq!(sidecar.len(), 64 + 2 + EXTENDDB_MANIFEST_FILE.len() + 1); + verify_manifest_checksum(&bytes, &sidecar).unwrap(); + // A bare digest, with or without a newline, and upper-case hex are + // accepted too. + let digest = sidecar.split_whitespace().next().unwrap(); + verify_manifest_checksum(&bytes, digest).unwrap(); + verify_manifest_checksum(&bytes, &format!("{digest}\n")).unwrap(); + verify_manifest_checksum(&bytes, &digest.to_ascii_uppercase()).unwrap(); + assert!(verify_manifest_checksum(&bytes, "").is_err()); + } + + #[test] + fn sidecar_checksum_detects_tampered_bytes() { + let bytes = sample_manifest().to_json_bytes().unwrap(); + let sidecar = manifest_checksum(&bytes); + let mut tampered = bytes.clone(); + let pos = tampered.windows(5).position(|w| w == b"50000").unwrap(); + tampered[pos] = b'7'; + let err = verify_manifest_checksum(&tampered, &sidecar).unwrap_err(); + assert!(matches!( + err, + FormatError::ChecksumMismatch { + subject: "manifest", + .. + } + )); + } + + #[test] + fn sidecar_checksum_is_over_stored_bytes_not_a_canonical_form() { + // Re-serializing with different whitespace changes the digest: the + // sidecar binds the stored object, nothing else. + let manifest = sample_manifest(); + let compact = manifest.to_json_bytes().unwrap(); + let pretty = serde_json::to_vec_pretty(&manifest).unwrap(); + assert_ne!(manifest_checksum(&compact), manifest_checksum(&pretty)); + assert!(verify_manifest_checksum(&compact, &manifest_checksum(&pretty)).is_err()); + verify_manifest_checksum(&pretty, &manifest_checksum(&pretty)).unwrap(); + } + + #[test] + fn from_json_bytes_runs_version_gate_and_key_check() { + let mut manifest = sample_manifest(); + manifest.format_version = 2; + let err = + ExtenddbManifest::from_json_bytes(&manifest.to_json_bytes().unwrap()).unwrap_err(); + assert!(matches!(err, FormatError::UnsupportedFormatVersion { .. })); + let mut manifest = sample_manifest(); + manifest.data_files[0].key = "../escape.json.gz".to_owned(); + let err = + ExtenddbManifest::from_json_bytes(&manifest.to_json_bytes().unwrap()).unwrap_err(); + assert!(matches!(err, FormatError::InvalidDataFileKey { .. })); + } + + #[test] + fn valid_data_file_keys_pass() { + for key in [ + "data/000001.json.gz", + "data/42.json.gz", + "data/1000000.json.gz", + ] { + let entry = DataFileEntry { + key: key.to_owned(), + item_count: 1, + sha256: "0".repeat(64), + md5: String::new(), + bytes: 1, + uncompressed_bytes: 1, + key_range: None, + }; + entry + .validate_key() + .unwrap_or_else(|e| panic!("{key:?} rejected: {e}")); + } + } + + #[test] + fn data_file_key_rejects_bad_shapes() { + for key in [ + "../x", // traversal above the prefix + "/etc/passwd", // absolute path, no prefix + "other/f.json.gz", // wrong prefix + "data/../secret.json.gz", // traversal inside the prefix + "data/sub/000001.json.gz", // extra directory level + "data/", // no file + "data/000001.json", // wrong suffix + "data/000001.txt.gz", // wrong suffix + "data/abc.json.gz", // non-digit stem + "data/.json.gz", // empty stem + "data\\000001.json.gz", // backslash separator + "data/000001.json.gz.evil", // trailing suffix + ] { + let entry = DataFileEntry { + key: key.to_owned(), + item_count: 1, + sha256: "0".repeat(64), + md5: String::new(), + bytes: 1, + uncompressed_bytes: 1, + key_range: None, + }; + assert!( + matches!( + entry.validate_key(), + Err(FormatError::InvalidDataFileKey { .. }) + ), + "expected rejection for {key:?}" + ); + } + } + + #[test] + fn validate_data_files_reports_first_bad_key() { + let mut manifest = sample_manifest(); + manifest.data_files[0].key = "../escape.json.gz".to_owned(); + assert!(matches!( + manifest.validate_data_files(), + Err(FormatError::InvalidDataFileKey { .. }) + )); + } + + #[test] + fn entry_decode_rejects_traversal_key_before_reading_bytes() { + let entry = DataFileEntry { + key: "../escape.json.gz".to_owned(), + item_count: 0, + sha256: "0".repeat(64), + md5: String::new(), + bytes: 0, + uncompressed_bytes: 0, + key_range: None, + }; + // Bytes are irrelevant: the key check fires first. + let err = entry.decode(b"not even gzip").unwrap_err(); + assert!(matches!(err, FormatError::InvalidDataFileKey { .. })); + } +} diff --git a/crates/core/src/backup_format/mod.rs b/crates/core/src/backup_format/mod.rs new file mode 100644 index 000000000..76f9737ed --- /dev/null +++ b/crates/core/src/backup_format/mod.rs @@ -0,0 +1,62 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Portable backup format: pure types, serde, and checksum helpers. +//! +//! A backup is a store prefix holding the `ExtendDB` manifest +//! (`extenddb-manifest.json`) and its checksum sidecar +//! (`extenddb-manifest.sha256`), the two manifests Amazon `DynamoDB` writes for +//! `ExportTableToPointInTime` with `DYNAMODB_JSON` output +//! (`manifest-summary.json` and `manifest-files.json`), and gzip data files of +//! newline-delimited `{"Item": ...}` objects under a `data/` prefix. Because +//! the `DynamoDB`-shaped pieces follow the service's own export layout, the +//! `data/` prefix is a valid `ImportTable` source for the real service, and a +//! real export is restorable here. +//! +//! Write order: data files, `manifest-files.json`, `manifest-summary.json`, +//! `extenddb-manifest.sha256`, then `extenddb-manifest.json` last. The +//! manifest's presence is the commit record; a prefix with a manifest but no +//! sidecar, or whose sidecar does not match, is corrupt and is refused. +//! +//! The store key space is case-sensitive, as `DynamoDB` table names are +//! (`Music` and `music` are different tables). A filesystem store on a +//! case-insensitive filesystem (macOS APFS default volumes) cannot hold +//! backups of both; the store documents that constraint, the format does not +//! work around it. +//! +//! The `DynamoDB` export layout is documented at +//! , +//! which is the source of truth for the field sets and encodings in +//! [`ExportSummaryManifest`] and [`ExportFileEntry`]. +//! +//! This module performs no I/O and has no storage dependency: callers hand it +//! bytes and iterators, and it hands back bytes, types, and checksums. + +mod data_file; +mod ddb_export; +mod error; +pub mod key_order; +mod layout; +mod manifest; + +#[cfg(test)] +mod golden_tests; + +pub use data_file::{ + CompressedDigest, CompressedHasher, DataFileChecksums, DataFileDecoder, DataFileEncoder, + DataFileExpectations, GZIP_LEVEL, MAX_LINE_BYTES, decode_data_file, encode_data_file, +}; +pub use ddb_export::{ + DYNAMODB_JSON_OUTPUT_FORMAT, EXPORT_SUMMARY_VERSION, ExportFileEntry, ExportSummaryManifest, + parse_files_manifest, write_files_manifest, +}; +pub use error::FormatError; +pub use key_order::{KeyOrder, hash_key_text}; +pub use layout::{ + BackupArn, DATA_PREFIX, EXTENDDB_MANIFEST_FILE, FILES_MANIFEST_FILE, MANIFEST_CHECKSUM_FILE, + SUMMARY_MANIFEST_FILE, data_file_key, +}; +pub use manifest::{ + CURRENT_FORMAT_VERSION, DataFileEntry, ExtenddbManifest, ItemOrder, KeyRange, + SnapshotDescriptor, SourceTableSchema, manifest_checksum, verify_manifest_checksum, +}; diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index fdfd80b4d..9f2c5db25 100755 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -7,6 +7,7 @@ //! It defines the Virtual `DynamoDB` type system, request/response envelopes, //! validation logic, and error types shared across all other crates. +pub mod backup_format; pub mod error; pub mod expression; pub mod limits; diff --git a/crates/core/testdata/ddb-export/README.md b/crates/core/testdata/ddb-export/README.md new file mode 100644 index 000000000..cc1bea649 --- /dev/null +++ b/crates/core/testdata/ddb-export/README.md @@ -0,0 +1,25 @@ +# DynamoDB export fixture + +A small export in the layout Amazon DynamoDB writes for +`ExportTableToPointInTime` with `DYNAMODB_JSON` output. + +## Contents + +- `manifest-summary.json`: the summary manifest, one compact JSON document. +- `manifest-files.json`: the files manifest, JSON lines, one entry. +- `data/ka2sswm5ha4uejfqcmnjcbg6ru.json.gz`: one gzip data file holding three + items as newline-delimited `{"Item": ...}` objects. + +## Provenance + +Constructed, not captured from the live service. The fixture follows the +output format documented at +: +member names, the `2020-06-30` summary version, ISO 8601 timestamps, the +explicit `null` for `s3SseKmsKeyId`, the absence of `exportType` on a full +export, base64 `md5Checksum` per data file, and the marshalled DynamoDB JSON +item encoding. The summary field values reuse the documented example +(ProductCatalog); item counts and checksums are computed from the data file +in this directory, so the three files are mutually consistent. The data file +was gzipped with mtime 0 for reproducibility. Regenerating any file requires +recomputing `md5Checksum`, `etag`, and `itemCount` in the manifests. diff --git a/crates/core/testdata/ddb-export/data/ka2sswm5ha4uejfqcmnjcbg6ru.json.gz b/crates/core/testdata/ddb-export/data/ka2sswm5ha4uejfqcmnjcbg6ru.json.gz new file mode 100644 index 000000000..734ed0bd0 Binary files /dev/null and b/crates/core/testdata/ddb-export/data/ka2sswm5ha4uejfqcmnjcbg6ru.json.gz differ diff --git a/crates/core/testdata/ddb-export/manifest-files.json b/crates/core/testdata/ddb-export/manifest-files.json new file mode 100644 index 000000000..551a7ecec --- /dev/null +++ b/crates/core/testdata/ddb-export/manifest-files.json @@ -0,0 +1 @@ +{"itemCount":3,"md5Checksum":"1jUj2wzOC+iuxJN+LKy6Aw==","etag":"d63523db0cce0be8aec4937e2cacba03","dataFileS3Key":"AWSDynamoDB/01693685827463-2d8752fd/data/ka2sswm5ha4uejfqcmnjcbg6ru.json.gz"} diff --git a/crates/core/testdata/ddb-export/manifest-summary.json b/crates/core/testdata/ddb-export/manifest-summary.json new file mode 100644 index 000000000..e9ccd3668 --- /dev/null +++ b/crates/core/testdata/ddb-export/manifest-summary.json @@ -0,0 +1 @@ +{"version":"2020-06-30","exportArn":"arn:aws:dynamodb:us-east-1:123456789012:table/ProductCatalog/export/01693685827463-2d8752fd","startTime":"2020-11-04T07:28:34.028Z","endTime":"2020-11-04T07:33:43.897Z","tableArn":"arn:aws:dynamodb:us-east-1:123456789012:table/ProductCatalog","tableId":"12345a12-abcd-123a-ab12-1234abc12345","exportTime":"2020-11-04T07:28:34.028Z","s3Bucket":"ddb-productcatalog-export","s3Prefix":"2020-Nov","s3SseAlgorithm":"AES256","s3SseKmsKeyId":null,"manifestFilesS3Key":"AWSDynamoDB/01693685827463-2d8752fd/manifest-files.json","billedSizeBytes":0,"itemCount":3,"outputFormat":"DYNAMODB_JSON"} \ No newline at end of file diff --git a/crates/core/testdata/extenddb-manifest/README.md b/crates/core/testdata/extenddb-manifest/README.md new file mode 100644 index 000000000..a87e94b30 --- /dev/null +++ b/crates/core/testdata/extenddb-manifest/README.md @@ -0,0 +1,18 @@ +# Golden `extenddb-manifest.json`, format version 1 + +`v1.json` is the frozen on-disk shape of the ExtendDB backup manifest at +format version 1, with every optional member present, and `v1.sha256` is its +sidecar in `sha256sum` format. `golden_tests.rs` parses `v1.json` and +re-serializes it byte for byte. + +The nested members reuse the DynamoDB wire types from `crates/core/src/types`. +A change to one of those types (a rename, a `skip_serializing_if` toggle, a +new required member) changes this format without any other test noticing; +this fixture fails that change. Regenerate it only together with a +`format_version` bump: + +```sh +cargo test -p extenddb-core write_golden_manifest -- --ignored +``` + +The values are synthetic; no backup was taken to produce them. diff --git a/crates/core/testdata/extenddb-manifest/v1.json b/crates/core/testdata/extenddb-manifest/v1.json new file mode 100644 index 000000000..a85fa1f3e --- /dev/null +++ b/crates/core/testdata/extenddb-manifest/v1.json @@ -0,0 +1 @@ +{"format_version":1,"backup_arn":"arn:aws:dynamodb:us-east-1:123456789012:table/Music/backup/01757500000123-0badcafe","backup_name":"music-nightly","backup_creation_date_time":1757500000.123,"backup_size_bytes":61828,"source_backend":"postgres","source_extenddb_version":"0.1.13","snapshot":{"kind":"postgres_repeatable_read","marker":"1234:1234:"},"item_order":"key_ordered","source_table":{"table_name":"Music","table_id":"12345a12-abcd-123a-ab12-1234abc12345","table_arn":"arn:aws:dynamodb:us-east-1:123456789012:table/Music","table_creation_date_time":1757400000.0,"key_schema":[{"AttributeName":"Artist","KeyType":"HASH"},{"AttributeName":"SongTitle","KeyType":"RANGE"}],"attribute_definitions":[{"AttributeName":"Artist","AttributeType":"S"},{"AttributeName":"SongTitle","AttributeType":"S"},{"AttributeName":"Genre","AttributeType":"S"}],"billing_mode":"PROVISIONED","provisioned_throughput":{"ReadCapacityUnits":5,"WriteCapacityUnits":5},"on_demand_throughput":{"MaxReadRequestUnits":100,"MaxWriteRequestUnits":50},"table_class":"STANDARD","sse_description":{"Status":"ENABLED","SSEType":"KMS","KMSMasterKeyArn":"arn:aws:kms:us-east-1:123456789012:key/abcd"},"global_secondary_indexes":[{"IndexName":"GenreIndex","KeySchema":[{"AttributeName":"Genre","KeyType":"HASH"}],"Projection":{"ProjectionType":"INCLUDE","NonKeyAttributes":["Year"]},"ProvisionedThroughput":{"ReadCapacityUnits":1,"WriteCapacityUnits":1}}],"local_secondary_indexes":[{"IndexName":"TitleIndex","KeySchema":[{"AttributeName":"Artist","KeyType":"HASH"},{"AttributeName":"SongTitle","KeyType":"RANGE"}],"Projection":{"ProjectionType":"KEYS_ONLY"}}],"vector_indexes":[{"IndexName":"embeddings","Dimensions":1024,"DistanceFunction":"COSINE","VectorAttribute":{"AttributeName":"vec"}}],"item_count":100000,"table_size_bytes":123456},"data_files":[{"key":"data/000001.json.gz","item_count":50000,"sha256":"0000000000000000000000000000000000000000000000000000000000000000","md5":"1B2M2Y8AsgTpgAmY7PhCfg==","bytes":61728,"uncompressed_bytes":300000,"key_range":{"first":{"Artist":{"S":"A"},"SongTitle":{"S":"a"}},"last":{"Artist":{"S":"M"},"SongTitle":{"S":"z"}}}},{"key":"data/000002.json.gz","item_count":50000,"sha256":"1111111111111111111111111111111111111111111111111111111111111111","md5":"1B2M2Y8AsgTpgAmY7PhCfg==","bytes":100,"uncompressed_bytes":300000,"key_range":{"first":{"Artist":{"S":"N"},"SongTitle":{"S":"a"}},"last":{"Artist":{"S":"Z"},"SongTitle":{"S":"z"}}}}]} \ No newline at end of file diff --git a/crates/core/testdata/extenddb-manifest/v1.sha256 b/crates/core/testdata/extenddb-manifest/v1.sha256 new file mode 100644 index 000000000..a9e112f78 --- /dev/null +++ b/crates/core/testdata/extenddb-manifest/v1.sha256 @@ -0,0 +1 @@ +9982c814a6d2f59a4eb27738834f0b164b87160322aae0c4b90e665bba6a2ffe extenddb-manifest.json diff --git a/crates/storage-sqlite/src/number_key.rs b/crates/storage-sqlite/src/number_key.rs index 114e55688..f35b09ec8 100644 --- a/crates/storage-sqlite/src/number_key.rs +++ b/crates/storage-sqlite/src/number_key.rs @@ -222,4 +222,113 @@ mod tests { assert!(enc("0") < enc("0.0000000001")); assert_eq!(enc("0"), "1"); } + + /// Binds this encoding to the backup format's key comparator: a + /// `key_ordered` backup of a table with an `N` sort key promises the + /// order `KeyOrder::compare` defines, and this backend produces it with + /// `ORDER BY pk, sk_n` only while the byte order of `sk_n` equals that + /// comparison. A change to either side fails here. The corpus is a fixed + /// pseudo-random set of `DynamoDB`-valid numbers (1 to 38 significant + /// digits, both signs, exponents across the allowed range) plus the + /// boundary values the tests above use. + #[test] + fn number_key_order_matches_backup_format() { + use extenddb_core::backup_format::KeyOrder; + use extenddb_core::types::{ + AttributeDefinition, AttributeValue, Item, KeySchemaElement, KeyType, + ScalarAttributeType, + }; + use extenddb_core::validation::number::validate_and_normalize_number; + + let order = KeyOrder::resolve( + &[ + KeySchemaElement { + attribute_name: "pk".to_owned(), + key_type: KeyType::Hash, + }, + KeySchemaElement { + attribute_name: "sk".to_owned(), + key_type: KeyType::Range, + }, + ], + &[ + AttributeDefinition { + attribute_name: "pk".to_owned(), + attribute_type: ScalarAttributeType::S, + }, + AttributeDefinition { + attribute_name: "sk".to_owned(), + attribute_type: ScalarAttributeType::N, + }, + ], + ) + .unwrap(); + let item = |n: &str| { + let mut m = Item::new(); + m.insert("pk".to_owned(), AttributeValue::S("p".to_owned())); + m.insert("sk".to_owned(), AttributeValue::N(n.to_owned())); + m + }; + + let mut corpus: Vec = [ + "0", + "1", + "-1", + "1E-130", + "-1E-130", + "9.9999999999999999999999999999999999999E+125", + "-9.9999999999999999999999999999999999999E+125", + "1.2345678901234567890123456789012345678", + "1.2345678901234567890123456789012345679", + "0.12", + "0.123", + "-0.123", + "10", + "9", + ] + .iter() + .map(|s| validate_and_normalize_number(s).unwrap()) + .collect(); + let mut x: u64 = 0x2545_f491_4f6c_dd1d; + let mut next = || { + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + x + }; + while corpus.len() < 200 { + let digits = usize::try_from(next() % 38).unwrap() + 1; + let mut mantissa = String::new(); + for i in 0..digits { + let d = next() % 10; + // No leading zero so the digit count is the significant count. + let d = if i == 0 && d == 0 { 1 } else { d }; + mantissa.push(char::from(b'0' + u8::try_from(d).unwrap())); + } + // Exponent keeps the value within DynamoDB's magnitude range: + // 10^-130 <= |v| < 10^126, with `digits` digits before the point. + let exp = i64::try_from(next() % 250).unwrap() - 130; + let sign = if next() % 2 == 0 { "" } else { "-" }; + let candidate = format!( + "{sign}{mantissa}E{}", + exp - i64::try_from(digits).unwrap() + 1 + ); + if let Ok(normalized) = validate_and_normalize_number(&candidate) { + corpus.push(normalized); + } + } + corpus.sort(); + corpus.dedup(); + + for a in &corpus { + for b in &corpus { + let encoded = enc(a).cmp(&enc(b)); + let format = order.compare(&item(a), &item(b)).unwrap(); + assert_eq!( + encoded, format, + "{a} vs {b}: sk_n byte order {encoded:?}, backup format order {format:?}" + ); + } + } + } } diff --git a/crates/storage/src/util/key.rs b/crates/storage/src/util/key.rs index 1519afd9f..82ddd5dbf 100755 --- a/crates/storage/src/util/key.rs +++ b/crates/storage/src/util/key.rs @@ -4,8 +4,6 @@ //! Helper functions for transforming and parsing partition key (pk) and sort key (sk) values. use crate::error::StorageError; -use base64::Engine; -use base64::engine::general_purpose::STANDARD as BASE64; use extenddb_core::types::{ AttributeDefinition, AttributeValue, Item, KeySchemaElement, KeyType, ScalarAttributeType, }; @@ -76,14 +74,10 @@ pub fn parse_sk( /// Per design doc ยง5.1: partition keys are always stored as TEXT. /// S โ†’ direct (borrowed), N โ†’ string representation (borrowed), B โ†’ base64 (owned). pub fn pk_to_text(value: &AttributeValue) -> Result, StorageError> { - match value { - AttributeValue::S(s) => Ok(Cow::Borrowed(s)), - AttributeValue::N(n) => Ok(Cow::Borrowed(n)), - AttributeValue::B(b) => Ok(Cow::Owned(BASE64.encode(b))), - _ => Err(StorageError::Internal( - "partition key must be S, N, or B".to_owned(), - )), - } + // One encoding for the row key: the backup format's key order binds the + // same function, so an ORDER BY pk and the manifest's key ranges agree. + extenddb_core::backup_format::hash_key_text(value) + .ok_or_else(|| StorageError::Internal("partition key must be S, N, or B".to_owned())) } /// Determine which sort key column to use based on the attribute type.