Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 21 additions & 1 deletion fastfilter/src/main/java/org/fastfilter/Filter.java
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package org.fastfilter;

import java.nio.ByteBuffer;

/**
* An approximate membership filter.
*/
Expand All @@ -14,7 +16,7 @@ public interface Filter {
boolean mayContain(long key);

/**
* Get the number of bits in thhe filter.
* Get the number of bits in the filter.
*
* @return the number of bits
*/
Expand Down Expand Up @@ -65,4 +67,22 @@ default long cardinality() {
return -1;
}

/**
* Get the serialized size of the filter.
*
* @return the size in bytes
*/
default int getSerializedSize() {
return -1;
}

/**
* Serializes the filter state into the provided {@code ByteBuffer}.
*
* @param buffer the byte buffer where the serialized state of the filter will be written
* @throws UnsupportedOperationException if the operation is not supported by the filter implementation
*/
default void serialize(ByteBuffer buffer) {
throw new UnsupportedOperationException();
}
}
1 change: 0 additions & 1 deletion fastfilter/src/main/java/org/fastfilter/utils/Hash.java
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
import java.util.Random;

public class Hash {

private static Random random = new Random();

public static void setSeed(long seed) {
Expand Down
74 changes: 63 additions & 11 deletions fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse16.java
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package org.fastfilter.xor;

import java.nio.ByteBuffer;
import java.util.Arrays;

import org.fastfilter.Filter;
import org.fastfilter.utils.Hash;

Expand All @@ -20,19 +20,25 @@ public class XorBinaryFuse16 implements Filter {
private final short[] fingerprints;
private long seed;

public XorBinaryFuse16(int segmentCount, int segmentLength) {
private XorBinaryFuse16(int segmentCount, int segmentLength, long seed, short[] fingerprints) {
if (segmentLength < 0 || Integer.bitCount(segmentLength) != 1) {
throw new IllegalArgumentException("Segment length needs to be a power of 2, is " + segmentLength);
}
if (segmentCount <= 0) {
throw new IllegalArgumentException("Illegal segment count: " + segmentCount);
}
this.segmentLength = segmentLength;

this.segmentCount = segmentCount;
this.segmentLengthMask = segmentLength - 1;
this.segmentCountLength = segmentCount * segmentLength;
this.arrayLength = (segmentCount + ARITY - 1) * segmentLength;
this.fingerprints = new short[arrayLength];
this.segmentLength = segmentLength;
this.segmentLengthMask = segmentLength - 1;
this.arrayLength = fingerprints.length;
this.fingerprints = fingerprints;
this.seed = seed;
}

public XorBinaryFuse16(int segmentCount, int segmentLength) {
this(segmentCount, segmentLength, 0L, new short[(segmentCount + ARITY - 1) * segmentLength]);
}

public long getBitCount() {
Expand Down Expand Up @@ -202,14 +208,13 @@ private void addAll(long[] keys) {
// if construction doesn't succeed eventually,
// then there is likely a problem with the hash function
// let us not crash the system:
for(int i = 0; i < fingerprints.length; i++) {
fingerprints[i] = (short)0xFFFF;
}
Arrays.fill(fingerprints, (short) 0xFFFF);
return;
}
// use a new random numbers
// use a new random number
seed = Hash.randomSeed();
}

alone = null;
t2count = null;
t2hash = null;
Expand Down Expand Up @@ -261,4 +266,51 @@ private short fingerprint(long hash) {
return (short) hash;
}

}
@Override
public int getSerializedSize() {
return 2 * Integer.BYTES + Long.BYTES + Integer.BYTES + fingerprints.length * Short.BYTES;
}

@Override
public void serialize(ByteBuffer buffer) {
if (buffer.remaining() < getSerializedSize()) {
throw new IllegalArgumentException("Buffer too small");
}

buffer.putInt(segmentLength);
buffer.putInt(segmentCountLength);
buffer.putLong(seed);
buffer.putInt(fingerprints.length);
for (final short fp : fingerprints) {
buffer.putShort(fp);
}
}

public static XorBinaryFuse16 deserialize(ByteBuffer buffer) {
// Check minimum size for header (2 ints + 1 long + 1 int for length)
if (buffer.remaining() < 2 * Integer.BYTES + Long.BYTES + Integer.BYTES) {
throw new IllegalArgumentException("Buffer too small");
}

final int segmentLength = buffer.getInt();
final int segmentCountLength = buffer.getInt();
final long seed = buffer.getLong();

final int len = buffer.getInt();

// Check if buffer has enough bytes for all fingerprints
if (buffer.remaining() < len * Short.BYTES) {
throw new IllegalArgumentException("Buffer too small");
}

final short[] fingerprints = new short[len];
for (int i = 0; i < len; i++) {
fingerprints[i] = buffer.getShort();
}

// Calculate segmentCount from segmentCountLength and segmentLength
final int segmentCount = segmentCountLength / segmentLength;

return new XorBinaryFuse16(segmentCount, segmentLength, seed, fingerprints);
}
}
Loading