fix(xor): reserve a flags byte in the serialized header

datasketches/src/codec/family.rs

@@ -130,8 +130,8 @@ impl Family {
     pub const XORFILTER: Family = Family {
         id: 22,
         name: "XORFILTER",
-        min_pre_longs: 2,
-        max_pre_longs: 2,
+        min_pre_longs: 3,
+        max_pre_longs: 3,
     };
 }
 

datasketches/src/xor/filter.rs

@@ -30,7 +30,7 @@ use crate::hash::XxHash64;
 
 const NUM_HASHES: u8 = 3;
 const SERIAL_VERSION: u8 = 1;
-const PREAMBLE_BYTES: usize = 2 * size_of::<u64>();
+const PREAMBLE_BYTES: usize = 3 * size_of::<u64>();
 const HASH_SEED: u64 = 0;
 const LOAD_FACTOR: f64 = 1.23;
 const CAPACITY_OFFSET: u64 = 32;
@@ -336,9 +336,12 @@ impl XorFilter {
         bytes.write_u8(Family::XORFILTER.min_pre_longs);
         bytes.write_u8(SERIAL_VERSION);
         bytes.write_u8(Family::XORFILTER.id);
+        bytes.write_u8(0); // flags
         bytes.write_u8(self.filter_type().bits_per_fingerprint());
-        bytes.write_i32_le(self.segment_length as i32);
+        bytes.write(&[0; 3]); // Pad to the seed at byte 8.
         bytes.write_u64_le(self.seed);
+        bytes.write_i32_le(self.segment_length as i32);
+        bytes.write_u32_le(0); // Pad the preamble to three 64-bit words.
 
         match &self.fingerprints {
             Fingerprints::Xor8(fingerprints) => bytes.write(fingerprints),
@@ -375,6 +378,12 @@ impl XorFilter {
             .read_u8()
             .map_err(insufficient_data("serial_version"))?;
         let family_id = cursor.read_u8().map_err(insufficient_data("family_id"))?;
+        let flags = cursor.read_u8().map_err(insufficient_data("flags"))?;
+        if flags != 0 {
+            return Err(Error::deserial(format!(
+                "unsupported xor filter flags: {flags}"
+            )));
+        }
         let bits_per_fingerprint = cursor
             .read_u8()
             .map_err(insufficient_data("bits_per_fingerprint"))?;
@@ -395,6 +404,10 @@ impl XorFilter {
                 ));
             }
         };
+        cursor
+            .read_exact(&mut [0; 3])
+            .map_err(insufficient_data("reserved"))?;
+        let seed = cursor.read_u64_le().map_err(insufficient_data("seed"))?;
         let segment_length = cursor
             .read_i32_le()
             .map_err(insufficient_data("segment_length"))?;
@@ -405,7 +418,9 @@ impl XorFilter {
         }
         let segment_length = segment_length as usize;
 
-        let seed = cursor.read_u64_le().map_err(insufficient_data("seed"))?;
+        cursor
+            .read_u32_le()
+            .map_err(insufficient_data("reserved"))?;
 
         let capacity = segment_length
             .checked_mul(usize::from(NUM_HASHES))

tests-integration/tests/serde_tests/xor.rs

@@ -20,14 +20,14 @@ use datasketches::xor::XorFilter;
 use datasketches::xor::XorFilterBuilder;
 use datasketches::xor::XorFilterType;
 
-// Generated from datasketches-java commit c179c5bc00d5aadaed8c68ce11d14d812abe08d6.
+// Generated from datasketches-java commit 1ed3eab435a4767d672ce5797eb53f66b14d78bb.
 // XorFilterBuilder.create([1, 2, 3, Long.MIN_VALUE], bits, 42).toByteArray().
 const JAVA_XOR8_IMAGE: &str = concat!(
-    "020116080c000000956eeb2f2632d7bd",
+    "0301160008000000956eeb2f2632d7bd0c00000000000000",
     "000000000000000400dd0000000000000000ce0000c50000000000000000000000000000"
 );
 const JAVA_XOR16_IMAGE: &str = concat!(
-    "020116100c000000956eeb2f2632d7bd",
+    "0301160010000000956eeb2f2632d7bd0c00000000000000",
     "000000000000000000000000000004e80000dd6e00000000000000000000000000000000",
     "ce9500000000c5b600000000000000000000000000000000000000000000000000000000",
 );
@@ -109,13 +109,14 @@ fn invalid_preamble_fields_are_rejected() {
 
     for (offset, value) in [
         (0, 1),
-        (0, 3),
+        (0, 2),
         (1, 0),
         (1, 2),
         (2, 21),
-        (3, 7),
-        (3, 24),
-        (3, 64),
+        (3, 1),
+        (4, 7),
+        (4, 24),
+        (4, 64),
     ] {
         let mut corrupted = valid.clone();
         corrupted[offset] = value;
@@ -134,7 +135,7 @@ fn unsafe_lengths_are_rejected_before_indexing() {
 
     for segment_length in [0_i32, -1, i32::MAX] {
         let mut corrupted = valid.clone();
-        corrupted[4..8].copy_from_slice(&segment_length.to_le_bytes());
+        corrupted[16..20].copy_from_slice(&segment_length.to_le_bytes());
         assert_eq!(
             XorFilter::deserialize(&corrupted).unwrap_err().kind(),
             ErrorKind::InvalidData
@@ -177,11 +178,11 @@ fn java_utf8_and_raw_byte_inputs_match() {
     let images = [
         (
             XorFilterType::Xor8,
-            "020116080c000000956eeb2f2632d7bdccdf00000000000000ef0091000000000000000000000000000000000000000000000000",
+            "0301160008000000956eeb2f2632d7bd0c00000000000000ccdf00000000000000ef0091000000000000000000000000000000000000000000000000",
         ),
         (
             XorFilterType::Xor16,
-            "020116100c000000956eeb2f2632d7bdccb5df500000000000000000000000000000ef42000091b8000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
+            "0301160010000000956eeb2f2632d7bd0c00000000000000ccb5df500000000000000000000000000000ef42000091b8000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
         ),
     ];
     for (kind, image) in images {
@@ -206,8 +207,9 @@ fn xor32_image_preserves_full_width_little_endian_slots() {
     // Hand-authored layout fixture, independent of the construction algorithm:
     // family 22/version 1, width 32, three one-slot segments.
     let bytes = decode_hex(concat!(
-        "0201162001000000",
+        "0301160020000000",
         "efcdab8967452301",
+        "0100000000000000",
         "67452301efcdab899c5f4186",
     ));
     let filter = XorFilter::deserialize(&bytes).unwrap();