From 09d5edfe5a07f1bdeffaf593cd25e97b06555511 Mon Sep 17 00:00:00 2001 From: tison Date: Sun, 8 Nov 2026 11:51:34 +0800 Subject: [PATCH 1/2] fix(xor): reserve a flags byte in the serialized header --- datasketches/src/codec/family.rs | 4 ++-- datasketches/src/xor/filter.rs | 21 ++++++++++++++++--- tests-integration/tests/serde_tests/xor.rs | 24 ++++++++++++---------- 3 files changed, 33 insertions(+), 16 deletions(-) diff --git a/datasketches/src/codec/family.rs b/datasketches/src/codec/family.rs index d613eec0..06a8f3d1 100644 --- a/datasketches/src/codec/family.rs +++ b/datasketches/src/codec/family.rs @@ -130,8 +130,8 @@ impl Family { pub const XORFILTER: Family = Family { id: 22, name: "XORFILTER", - min_pre_longs: 2, - max_pre_longs: 2, + min_pre_longs: 3, + max_pre_longs: 3, }; } diff --git a/datasketches/src/xor/filter.rs b/datasketches/src/xor/filter.rs index 3be436f0..a6247ff1 100644 --- a/datasketches/src/xor/filter.rs +++ b/datasketches/src/xor/filter.rs @@ -30,7 +30,7 @@ use crate::hash::XxHash64; const NUM_HASHES: u8 = 3; const SERIAL_VERSION: u8 = 1; -const PREAMBLE_BYTES: usize = 2 * size_of::(); +const PREAMBLE_BYTES: usize = 3 * size_of::(); const HASH_SEED: u64 = 0; const LOAD_FACTOR: f64 = 1.23; const CAPACITY_OFFSET: u64 = 32; @@ -336,9 +336,12 @@ impl XorFilter { bytes.write_u8(Family::XORFILTER.min_pre_longs); bytes.write_u8(SERIAL_VERSION); bytes.write_u8(Family::XORFILTER.id); + bytes.write_u8(0); // flags bytes.write_u8(self.filter_type().bits_per_fingerprint()); - bytes.write_i32_le(self.segment_length as i32); + bytes.write(&[0; 3]); // reserved bytes.write_u64_le(self.seed); + bytes.write_i32_le(self.segment_length as i32); + bytes.write_u32_le(0); // reserved match &self.fingerprints { Fingerprints::Xor8(fingerprints) => bytes.write(fingerprints), @@ -375,6 +378,12 @@ impl XorFilter { .read_u8() .map_err(insufficient_data("serial_version"))?; let family_id = cursor.read_u8().map_err(insufficient_data("family_id"))?; + let flags = cursor.read_u8().map_err(insufficient_data("flags"))?; + if flags != 0 { + return Err(Error::deserial(format!( + "unsupported xor filter flags: {flags}" + ))); + } let bits_per_fingerprint = cursor .read_u8() .map_err(insufficient_data("bits_per_fingerprint"))?; @@ -395,6 +404,10 @@ impl XorFilter { )); } }; + cursor + .read_exact(&mut [0; 3]) + .map_err(insufficient_data("reserved"))?; + let seed = cursor.read_u64_le().map_err(insufficient_data("seed"))?; let segment_length = cursor .read_i32_le() .map_err(insufficient_data("segment_length"))?; @@ -405,7 +418,9 @@ impl XorFilter { } let segment_length = segment_length as usize; - let seed = cursor.read_u64_le().map_err(insufficient_data("seed"))?; + cursor + .read_u32_le() + .map_err(insufficient_data("reserved"))?; let capacity = segment_length .checked_mul(usize::from(NUM_HASHES)) diff --git a/tests-integration/tests/serde_tests/xor.rs b/tests-integration/tests/serde_tests/xor.rs index de8dbe5d..2fe4d154 100644 --- a/tests-integration/tests/serde_tests/xor.rs +++ b/tests-integration/tests/serde_tests/xor.rs @@ -20,14 +20,14 @@ use datasketches::xor::XorFilter; use datasketches::xor::XorFilterBuilder; use datasketches::xor::XorFilterType; -// Generated from datasketches-java commit c179c5bc00d5aadaed8c68ce11d14d812abe08d6. +// Generated from datasketches-java commit 1ed3eab435a4767d672ce5797eb53f66b14d78bb. // XorFilterBuilder.create([1, 2, 3, Long.MIN_VALUE], bits, 42).toByteArray(). const JAVA_XOR8_IMAGE: &str = concat!( - "020116080c000000956eeb2f2632d7bd", + "0301160008000000956eeb2f2632d7bd0c00000000000000", "000000000000000400dd0000000000000000ce0000c50000000000000000000000000000" ); const JAVA_XOR16_IMAGE: &str = concat!( - "020116100c000000956eeb2f2632d7bd", + "0301160010000000956eeb2f2632d7bd0c00000000000000", "000000000000000000000000000004e80000dd6e00000000000000000000000000000000", "ce9500000000c5b600000000000000000000000000000000000000000000000000000000", ); @@ -109,13 +109,14 @@ fn invalid_preamble_fields_are_rejected() { for (offset, value) in [ (0, 1), - (0, 3), + (0, 2), (1, 0), (1, 2), (2, 21), - (3, 7), - (3, 24), - (3, 64), + (3, 1), + (4, 7), + (4, 24), + (4, 64), ] { let mut corrupted = valid.clone(); corrupted[offset] = value; @@ -134,7 +135,7 @@ fn unsafe_lengths_are_rejected_before_indexing() { for segment_length in [0_i32, -1, i32::MAX] { let mut corrupted = valid.clone(); - corrupted[4..8].copy_from_slice(&segment_length.to_le_bytes()); + corrupted[16..20].copy_from_slice(&segment_length.to_le_bytes()); assert_eq!( XorFilter::deserialize(&corrupted).unwrap_err().kind(), ErrorKind::InvalidData @@ -177,11 +178,11 @@ fn java_utf8_and_raw_byte_inputs_match() { let images = [ ( XorFilterType::Xor8, - "020116080c000000956eeb2f2632d7bdccdf00000000000000ef0091000000000000000000000000000000000000000000000000", + "0301160008000000956eeb2f2632d7bd0c00000000000000ccdf00000000000000ef0091000000000000000000000000000000000000000000000000", ), ( XorFilterType::Xor16, - "020116100c000000956eeb2f2632d7bdccb5df500000000000000000000000000000ef42000091b8000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "0301160010000000956eeb2f2632d7bd0c00000000000000ccb5df500000000000000000000000000000ef42000091b8000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", ), ]; for (kind, image) in images { @@ -206,8 +207,9 @@ fn xor32_image_preserves_full_width_little_endian_slots() { // Hand-authored layout fixture, independent of the construction algorithm: // family 22/version 1, width 32, three one-slot segments. let bytes = decode_hex(concat!( - "0201162001000000", + "0301160020000000", "efcdab8967452301", + "0100000000000000", "67452301efcdab899c5f4186", )); let filter = XorFilter::deserialize(&bytes).unwrap(); From 9c481f7843bf770b530de6dcde7f41c43736f47e Mon Sep 17 00:00:00 2001 From: tison Date: Sun, 8 Nov 2026 12:46:22 +0800 Subject: [PATCH 2/2] docs(xor): explain serialized header padding --- datasketches/src/xor/filter.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datasketches/src/xor/filter.rs b/datasketches/src/xor/filter.rs index a6247ff1..b378c8ae 100644 --- a/datasketches/src/xor/filter.rs +++ b/datasketches/src/xor/filter.rs @@ -338,10 +338,10 @@ impl XorFilter { bytes.write_u8(Family::XORFILTER.id); bytes.write_u8(0); // flags bytes.write_u8(self.filter_type().bits_per_fingerprint()); - bytes.write(&[0; 3]); // reserved + bytes.write(&[0; 3]); // Pad to the seed at byte 8. bytes.write_u64_le(self.seed); bytes.write_i32_le(self.segment_length as i32); - bytes.write_u32_le(0); // reserved + bytes.write_u32_le(0); // Pad the preamble to three 64-bit words. match &self.fingerprints { Fingerprints::Xor8(fingerprints) => bytes.write(fingerprints),