From 983f9720f440ec38a2e1d0cfb4f3dde16418db63 Mon Sep 17 00:00:00 2001 From: Prateek Gaur Date: Tue, 21 Apr 2026 00:15:09 +0000 Subject: [PATCH 1/7] Add the PFOR encoding for INT32 and INT64 PFOR splits a page into vectors, 1024 values by default. Each vector subtracts its minimum, bit-packs the offsets at the width a cost model picks, and stores the values that need more bits as exceptions with their positions. An offset array locates every vector, so a reader decodes a vector on first access and skipping does not unpack the ones in between. The reader checks each header field and exception position against the page before using it. Tests cover widths from 0 up to the full value width, partial vectors and malformed pages. --- .../org/apache/parquet/column/Encoding.java | 23 + .../column/values/pfor/PforConstants.java | 85 +++ .../values/pfor/PforEncoderDecoder.java | 155 ++++++ .../column/values/pfor/PforValuesReader.java | 217 ++++++++ .../values/pfor/PforValuesReaderForInt.java | 146 ++++++ .../values/pfor/PforValuesReaderForLong.java | 146 ++++++ .../column/values/pfor/PforValuesWriter.java | 487 ++++++++++++++++++ .../values/pfor/PforAdversarialTest.java | 461 +++++++++++++++++ .../values/pfor/PforBitPackingTest.java | 260 ++++++++++ .../values/pfor/PforEncoderDecoderTest.java | 197 +++++++ .../values/pfor/PforValuesEndToEndTest.java | 433 ++++++++++++++++ 11 files changed, 2610 insertions(+) create mode 100644 parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforConstants.java create mode 100644 parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforEncoderDecoder.java create mode 100644 parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforValuesReader.java create mode 100644 parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforValuesReaderForInt.java create mode 100644 parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforValuesReaderForLong.java create mode 100644 parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforValuesWriter.java create mode 100644 parquet-column/src/test/java/org/apache/parquet/column/values/pfor/PforAdversarialTest.java create mode 100644 parquet-column/src/test/java/org/apache/parquet/column/values/pfor/PforBitPackingTest.java create mode 100644 parquet-column/src/test/java/org/apache/parquet/column/values/pfor/PforEncoderDecoderTest.java create mode 100644 parquet-column/src/test/java/org/apache/parquet/column/values/pfor/PforValuesEndToEndTest.java diff --git a/parquet-column/src/main/java/org/apache/parquet/column/Encoding.java b/parquet-column/src/main/java/org/apache/parquet/column/Encoding.java index 874c99fded..280aa417e3 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/Encoding.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/Encoding.java @@ -45,6 +45,8 @@ import org.apache.parquet.column.values.dictionary.PlainValuesDictionary.PlainFloatDictionary; import org.apache.parquet.column.values.dictionary.PlainValuesDictionary.PlainIntegerDictionary; import org.apache.parquet.column.values.dictionary.PlainValuesDictionary.PlainLongDictionary; +import org.apache.parquet.column.values.pfor.PforValuesReaderForInt; +import org.apache.parquet.column.values.pfor.PforValuesReaderForLong; import org.apache.parquet.column.values.plain.BinaryPlainValuesReader; import org.apache.parquet.column.values.plain.BooleanPlainValuesReader; import org.apache.parquet.column.values.plain.FixedLenByteArrayPlainValuesReader; @@ -147,6 +149,27 @@ public ValuesReader getValuesReader(ColumnDescriptor descriptor, ValuesType valu } }, + /** + * PFOR (Patched Frame of Reference) encoding for INT32 and INT64 types. + * Compresses integer columns by subtracting the minimum value, selecting an + * optimal bit width via a cost model, bit-packing the deltas, and storing + * outlier values as exceptions. + */ + PFOR { + @Override + public ValuesReader getValuesReader(ColumnDescriptor descriptor, ValuesType valuesType) { + switch (descriptor.getType()) { + case INT32: + return new PforValuesReaderForInt(); + case INT64: + return new PforValuesReaderForLong(); + default: + throw new ParquetDecodingException( + "PFOR encoding is only supported for INT32 and INT64, not " + descriptor.getType()); + } + } + }, + /** * @deprecated This is no longer used, and has been replaced by {@link #RLE} * which is combination of bit packing and rle diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforConstants.java b/parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforConstants.java new file mode 100644 index 0000000000..a0f0f24222 --- /dev/null +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforConstants.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.values.pfor; + +import org.apache.parquet.Preconditions; + +/** + * Constants for the PFOR (Patched Frame of Reference) encoding. + * + *

PFOR encoding compresses integer columns (INT32/INT64) by: + *

    + *
  1. Subtracting the minimum value (Frame of Reference)
    2. + *
  2. Choosing an optimal bit width via a cost model
    3. + *
  3. Bit-packing the deltas at the chosen width
    4. + *
  4. Storing outlier values (exceptions) separately with their positions
    5. + *
+ */ +public final class PforConstants { + + private PforConstants() { + // Utility class + } + + // Page header fields (7 bytes total) + public static final int PFOR_PACKING_MODE_FOR = 0; + public static final int PFOR_HEADER_SIZE = 7; + + public static final int DEFAULT_VECTOR_SIZE = 1024; + public static final int DEFAULT_VECTOR_SIZE_LOG = 10; + + // Capped at 15 (vectorSize=32768) because num_exceptions is uint16, + // so vectorSize must not exceed 65535 to avoid overflow when all values are exceptions. + static final int MAX_LOG_VECTOR_SIZE = 15; + static final int MIN_LOG_VECTOR_SIZE = 3; + + // Maximum exceptions per vector (uint16) + public static final int MAX_EXCEPTIONS = 65535; + + // The bit width occupies bits 0..6 of its byte; bit 7 is reserved and must be + // masked off before the width is used or range-checked. + public static final int BIT_WIDTH_MASK = 0x7F; + + // Per-vector metadata sizes in bytes + // INT32: frame_of_reference(4) + bit_width(1) + num_exceptions(2) = 7 + public static final int INT32_VECTOR_INFO_SIZE = 7; + // INT64: frame_of_reference(8) + bit_width(1) + num_exceptions(2) = 11 + public static final int INT64_VECTOR_INFO_SIZE = 11; + + // Value byte widths + public static final int INT32_VALUE_BYTE_WIDTH = 4; + public static final int INT64_VALUE_BYTE_WIDTH = 8; + + /** Validates vector size: must be a power of 2 in [2^MIN_LOG .. 2^MAX_LOG]. */ + static int validateVectorSize(int vectorSize) { + Preconditions.checkArgument( + vectorSize > 0 && (vectorSize & (vectorSize - 1)) == 0, + "Vector size must be a power of 2, got: %s", + vectorSize); + int logSize = Integer.numberOfTrailingZeros(vectorSize); + Preconditions.checkArgument( + logSize >= MIN_LOG_VECTOR_SIZE && logSize <= MAX_LOG_VECTOR_SIZE, + "Vector size log2 must be between %s and %s, got: %s (vectorSize=%s)", + MIN_LOG_VECTOR_SIZE, + MAX_LOG_VECTOR_SIZE, + logSize, + vectorSize); + return vectorSize; + } +} diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforEncoderDecoder.java b/parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforEncoderDecoder.java new file mode 100644 index 0000000000..414ce8f89b --- /dev/null +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforEncoderDecoder.java @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.values.pfor; + +/** + * Core PFOR encoding/decoding logic with histogram-based cost model. + * + *

The cost model selects the optimal bit width by evaluating: + * 

+ * total_cost(b) = num_elements * b + num_exceptions(b) * (16 + value_bits)
+ *
+ * where {@code value_bits} is 32 for INT32 and 64 for INT64, and + * {@code num_exceptions(b)} is the count of deltas requiring more than {@code b} bits. + */ +public final class PforEncoderDecoder { + + private PforEncoderDecoder() { + // Utility class + } + + /** Result of the optimal bit width search. */ + public static final class BitWidthResult { + public final int bitWidth; + public final int numExceptions; + + BitWidthResult(int bitWidth, int numExceptions) { + this.bitWidth = bitWidth; + this.numExceptions = numExceptions; + } + } + + /** + * Find the optimal bit width for packing INT32 unsigned deltas. + * + *

Builds a histogram of bits required per delta, then evaluates each + * candidate bit width from 0 to 32 using the cost model. + * + * @param deltas unsigned deltas (values[] - min), treated as unsigned int + * @param numElements number of elements + * @return the optimal bit width and resulting number of exceptions + */ + public static BitWidthResult findOptimalBitWidthForInt(int[] deltas, int numElements) { + // Histogram: bitsHist[b] = count of deltas that need exactly b bits + int[] bitsHist = new int[33]; // 0..32 + for (int i = 0; i < numElements; i++) { + bitsHist[bitWidthForInt(deltas[i])]++; + } + + // Exception cost per exception: position(16 bits) + value(32 bits) = 48 bits + final long exceptionBitsPerValue = 16 + 32; + + long bestCost = Long.MAX_VALUE; + int bestBitWidth = 0; + int bestExceptions = 0; + + // At candidate width b, the deltas needing more than b bits are exceptions. + // A zero delta needs no bits, so at b=0 that is every nonzero delta. + int exceptionsAbove = numElements - bitsHist[0]; + + for (int b = 0; b <= 32; b++) { + long packingCost = (long) numElements * b; + long exceptionCost = (long) exceptionsAbove * exceptionBitsPerValue; + long totalCost = packingCost + exceptionCost; + + if (totalCost < bestCost) { + bestCost = totalCost; + bestBitWidth = b; + bestExceptions = exceptionsAbove; + } + + // Move to next candidate: values requiring exactly (b+1) bits are no longer exceptions + if (b < 32) { + exceptionsAbove -= bitsHist[b + 1]; + } + } + + return new BitWidthResult(bestBitWidth, bestExceptions); + } + + /** + * Find the optimal bit width for packing INT64 unsigned deltas. + * + * @param deltas unsigned deltas (values[] - min), treated as unsigned long + * @param numElements number of elements + * @return the optimal bit width and resulting number of exceptions + */ + public static BitWidthResult findOptimalBitWidthForLong(long[] deltas, int numElements) { + // Histogram: bitsHist[b] = count of deltas that need exactly b bits + int[] bitsHist = new int[65]; // 0..64 + for (int i = 0; i < numElements; i++) { + bitsHist[bitWidthForLong(deltas[i])]++; + } + + // Exception cost per exception: position(16 bits) + value(64 bits) = 80 bits + final long exceptionBitsPerValue = 16 + 64; + + long bestCost = Long.MAX_VALUE; + int bestBitWidth = 0; + int bestExceptions = 0; + + int exceptionsAbove = numElements - bitsHist[0]; + + for (int b = 0; b <= 64; b++) { + long packingCost = (long) numElements * b; + long exceptionCost = (long) exceptionsAbove * exceptionBitsPerValue; + long totalCost = packingCost + exceptionCost; + + if (totalCost < bestCost) { + bestCost = totalCost; + bestBitWidth = b; + bestExceptions = exceptionsAbove; + } + + if (b < 64) { + exceptionsAbove -= bitsHist[b + 1]; + } + } + + return new BitWidthResult(bestBitWidth, bestExceptions); + } + + /** + * Returns the number of bits required to represent an unsigned int value. + * Returns 0 for value == 0. + */ + public static int bitWidthForInt(int value) { + if (value == 0) return 0; + return 32 - Integer.numberOfLeadingZeros(value); + } + + /** + * Returns the number of bits required to represent an unsigned long value. + * Returns 0 for value == 0. + */ + public static int bitWidthForLong(long value) { + if (value == 0) return 0; + return 64 - Long.numberOfLeadingZeros(value); + } +} diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforValuesReader.java b/parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforValuesReader.java new file mode 100644 index 0000000000..4fabcae56b --- /dev/null +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforValuesReader.java @@ -0,0 +1,217 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.values.pfor; + +import static org.apache.parquet.column.values.pfor.PforConstants.*; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import org.apache.parquet.bytes.ByteBufferInputStream; +import org.apache.parquet.column.values.ValuesReader; +import org.apache.parquet.io.ParquetDecodingException; + +/** + * Abstract base class for PFOR values readers with lazy per-vector decoding. + * + *

Reads PFOR-encoded values from the page layout: + * 

+ * ┌─────────┬──────────────────────┬──────────────┬──────────────┬─────┐
+ * │ Header  │ Offset Array         │ Vector 0     │ Vector 1     │ ... │
+ * │ 7 bytes │ 4B x numVectors      │ info + data  │ info + data  │     │
+ * └─────────┴──────────────────────┴──────────────┴──────────────┴─────┘
+ *
+ * + *

Each vector is decoded lazily on first access. Skipping values does not + * trigger decoding of intermediate vectors. + */ +abstract class PforValuesReader extends ValuesReader { + + protected int vectorSize; + protected int totalCount; + protected int numVectors; + protected int currentIndex; + protected int currentVectorIndex; + protected int valueByteWidth; + protected int vectorInfoSize; + + protected int[] vectorOffsets; + protected ByteBuffer vectorsData; + protected int offsetArraySize; + + PforValuesReader() { + this.currentIndex = 0; + this.totalCount = 0; + this.currentVectorIndex = -1; + } + + @Override + public void initFromPage(int valuesCount, ByteBufferInputStream stream) + throws ParquetDecodingException, IOException { + ByteBuffer headerBuf = stream.slice(PFOR_HEADER_SIZE).order(ByteOrder.LITTLE_ENDIAN); + int packingMode = headerBuf.get() & 0xFF; + int logVectorSize = headerBuf.get() & 0xFF; + int valueBW = headerBuf.get() & 0xFF; + int numElements = headerBuf.getInt(); + + if (packingMode != PFOR_PACKING_MODE_FOR) { + throw new ParquetDecodingException("Unsupported PFOR packing mode: " + packingMode); + } + if (logVectorSize < MIN_LOG_VECTOR_SIZE || logVectorSize > MAX_LOG_VECTOR_SIZE) { + throw new ParquetDecodingException("Invalid PFOR log vector size: " + logVectorSize + ", must be between " + + MIN_LOG_VECTOR_SIZE + " and " + MAX_LOG_VECTOR_SIZE); + } + if (valueBW != INT32_VALUE_BYTE_WIDTH && valueBW != INT64_VALUE_BYTE_WIDTH) { + throw new ParquetDecodingException("Invalid PFOR value byte width: " + valueBW + ", must be 4 or 8"); + } + if (numElements < 0) { + throw new ParquetDecodingException("Invalid PFOR element count: " + numElements); + } + if (numElements > valuesCount) { + throw new ParquetDecodingException( + "PFOR header element count " + numElements + " exceeds page valuesCount " + valuesCount); + } + + this.vectorSize = 1 << logVectorSize; + this.totalCount = numElements; + this.valueByteWidth = valueBW; + this.vectorInfoSize = valueBW == INT32_VALUE_BYTE_WIDTH ? INT32_VECTOR_INFO_SIZE : INT64_VECTOR_INFO_SIZE; + this.numVectors = (numElements + vectorSize - 1) / vectorSize; + this.currentIndex = 0; + this.currentVectorIndex = -1; + + this.offsetArraySize = numVectors * Integer.BYTES; + ByteBuffer offsetBuf = stream.slice(offsetArraySize).order(ByteOrder.LITTLE_ENDIAN); + this.vectorOffsets = new int[numVectors]; + for (int v = 0; v < numVectors; v++) { + vectorOffsets[v] = offsetBuf.getInt(); + } + + // Slice remaining bytes into a 0-based view so decodeVector can use + // absolute get methods (vectorsData.get(pos)) directly. + int remainingBytes = (int) stream.available(); + ByteBuffer rawSlice = stream.slice(remainingBytes); + this.vectorsData = rawSlice.slice().order(ByteOrder.LITTLE_ENDIAN); + + allocateDecodedBuffer(vectorSize); + } + + protected int getVectorLength(int vectorIdx) { + if (vectorIdx < numVectors - 1) { + return vectorSize; + } + // Last vector may be partial + int lastVectorLen = totalCount % vectorSize; + return lastVectorLen == 0 ? vectorSize : lastVectorLen; + } + + // Offsets in the page are relative to the compression body (after header), + // but vectorsData starts after the offset array, so adjust. The offset came off + // the wire, so it has to leave room for the vector info it points at. + protected int getVectorDataPosition(int vectorIdx) { + int pos = vectorOffsets[vectorIdx] - offsetArraySize; + if (pos < 0 || pos + vectorInfoSize > vectorsData.limit()) { + throw new ParquetDecodingException("PFOR vector " + vectorIdx + " offset " + + vectorOffsets[vectorIdx] + " is outside a page body of " + vectorsData.limit() + + " bytes"); + } + return pos; + } + + /** + * Checks a vector's header fields against the vector they describe and the bytes + * that remain. All three come off the wire and size every read and write that + * follows, including the writes into the fixed-size decode buffers. + * + * @param pos position of the packed values, that is, just past the vector info + */ + protected void checkVectorInfo(int pos, int bitWidth, int numExceptions, int vectorLen) { + int maxBitWidth = valueByteWidth * Byte.SIZE; + if (bitWidth > maxBitWidth) { + throw new ParquetDecodingException("PFOR bit width " + bitWidth + " exceeds " + maxBitWidth); + } + if (numExceptions > vectorLen) { + throw new ParquetDecodingException( + "PFOR vector has " + numExceptions + " exceptions but only " + vectorLen + " elements"); + } + long needed = ((long) vectorLen * bitWidth + 7) / 8 + (long) numExceptions * (Short.BYTES + valueByteWidth); + long remaining = vectorsData.limit() - (long) pos; + if (needed > remaining) { + throw new ParquetDecodingException( + "PFOR vector needs " + needed + " bytes but only " + remaining + " remain"); + } + } + + /** Exception positions index the decode buffer, so one past the end is a bad write. */ + protected static void checkExceptionPosition(int position, int vectorLen) { + if (position >= vectorLen) { + throw new ParquetDecodingException( + "PFOR exception position " + position + " is outside a vector of " + vectorLen + " elements"); + } + } + + @Override + public void skip() { + skip(1); + } + + @Override + public void skip(int n) { + if (n < 0 || currentIndex + n > totalCount) { + throw new ParquetDecodingException(String.format( + "Cannot skip this many elements. Current index: %d. Skip %d. Total count: %d", + currentIndex, n, totalCount)); + } + currentIndex += n; + } + + protected void ensureVectorDecoded() { + int vectorIdx = currentIndex / vectorSize; + if (vectorIdx != currentVectorIndex) { + decodeVector(vectorIdx); + currentVectorIndex = vectorIdx; + } + } + + protected abstract void allocateDecodedBuffer(int capacity); + + protected abstract void decodeVector(int vectorIdx); + + protected static int getShortLE(ByteBuffer buf, int pos) { + return (buf.get(pos) & 0xFF) | ((buf.get(pos + 1) & 0xFF) << 8); + } + + protected static int getIntLE(ByteBuffer buf, int pos) { + return (buf.get(pos) & 0xFF) + | ((buf.get(pos + 1) & 0xFF) << 8) + | ((buf.get(pos + 2) & 0xFF) << 16) + | ((buf.get(pos + 3) & 0xFF) << 24); + } + + protected static long getLongLE(ByteBuffer buf, int pos) { + return (buf.get(pos) & 0xFFL) + | ((buf.get(pos + 1) & 0xFFL) << 8) + | ((buf.get(pos + 2) & 0xFFL) << 16) + | ((buf.get(pos + 3) & 0xFFL) << 24) + | ((buf.get(pos + 4) & 0xFFL) << 32) + | ((buf.get(pos + 5) & 0xFFL) << 40) + | ((buf.get(pos + 6) & 0xFFL) << 48) + | ((buf.get(pos + 7) & 0xFFL) << 56); + } +} diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforValuesReaderForInt.java b/parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforValuesReaderForInt.java new file mode 100644 index 0000000000..d80ef79cc5 --- /dev/null +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforValuesReaderForInt.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.values.pfor; + +import static org.apache.parquet.column.values.pfor.PforConstants.*; + +import java.nio.ByteBuffer; +import org.apache.parquet.column.values.bitpacking.BytePacker; +import org.apache.parquet.column.values.bitpacking.Packer; +import org.apache.parquet.io.ParquetDecodingException; + +/** + * PFOR values reader for INT32 type with lazy per-vector decoding. + * + *

Reads PFOR-encoded int values from the PFOR page layout. + * Each vector is decoded on first access using BytePacker-based unpacking. + * + *

Per-vector format: + * 

+ * PforVectorInfo (7B): frame_of_reference(4) + bit_width(1) + num_exceptions(2)
+ * PackedValues: ceil(N * bit_width / 8) bytes
+ * ExceptionPositions: num_exceptions * 2 bytes
+ * ExceptionValues: num_exceptions * 4 bytes
+ *
+ */ +public class PforValuesReaderForInt extends PforValuesReader { + + private int[] decodedValues; + + // Reusable per-vector decode buffers + private int[] deltasBuffer; + private int[] excPositionsBuffer; + private byte[] unpackPadBuf; + private int[] unpackTempBuf; + + public PforValuesReaderForInt() { + super(); + } + + @Override + protected void allocateDecodedBuffer(int capacity) { + this.decodedValues = new int[capacity]; + this.deltasBuffer = new int[capacity]; + this.excPositionsBuffer = new int[capacity]; + this.unpackPadBuf = new byte[Integer.SIZE]; // max bit width = 32 bytes + this.unpackTempBuf = new int[8]; + } + + @Override + public int readInteger() { + if (currentIndex >= totalCount) { + throw new ParquetDecodingException("PFOR int data was already exhausted."); + } + ensureVectorDecoded(); + int indexInVector = currentIndex % vectorSize; + currentIndex++; + return decodedValues[indexInVector]; + } + + @Override + protected void decodeVector(int vectorIdx) { + int vectorLen = getVectorLength(vectorIdx); + int pos = getVectorDataPosition(vectorIdx); + + // Read PforVectorInfo (7 bytes) + int frameOfReference = getIntLE(vectorsData, pos); + int bitWidth = vectorsData.get(pos + 4) & BIT_WIDTH_MASK; + int numExceptions = getShortLE(vectorsData, pos + 5) & 0xFFFF; + pos += INT32_VECTOR_INFO_SIZE; + checkVectorInfo(pos, bitWidth, numExceptions, vectorLen); + + // Unpack bit-packed deltas into reusable buffer + if (bitWidth > 0) { + pos = unpackIntsWithBytePacker(vectorsData, pos, deltasBuffer, vectorLen, bitWidth); + } else { + for (int i = 0; i < vectorLen; i++) { + deltasBuffer[i] = 0; + } + } + + // Add frame of reference to reconstruct values + for (int i = 0; i < vectorLen; i++) { + decodedValues[i] = deltasBuffer[i] + frameOfReference; + } + + // Overwrite exception slots with their original values + if (numExceptions > 0) { + for (int e = 0; e < numExceptions; e++) { + int position = getShortLE(vectorsData, pos) & 0xFFFF; + checkExceptionPosition(position, vectorLen); + excPositionsBuffer[e] = position; + pos += Short.BYTES; + } + for (int e = 0; e < numExceptions; e++) { + decodedValues[excPositionsBuffer[e]] = getIntLE(vectorsData, pos); + pos += Integer.BYTES; + } + } + } + + private int unpackIntsWithBytePacker(ByteBuffer buf, int pos, int[] output, int count, int bitWidth) { + BytePacker packer = Packer.LITTLE_ENDIAN.newBytePacker(bitWidth); + int numFullGroups = count / 8; + int remaining = count % 8; + + for (int g = 0; g < numFullGroups; g++) { + packer.unpack8Values(buf, pos, output, g * 8); + pos += bitWidth; + } + + if (remaining > 0) { + int totalPackedBytes = (count * bitWidth + 7) / 8; + int alreadyRead = numFullGroups * bitWidth; + int partialBytes = totalPackedBytes - alreadyRead; + + for (int i = 0; i < partialBytes; i++) { + unpackPadBuf[i] = buf.get(pos + i); + } + for (int i = partialBytes; i < bitWidth; i++) { + unpackPadBuf[i] = 0; + } + + packer.unpack8Values(unpackPadBuf, 0, unpackTempBuf, 0); + System.arraycopy(unpackTempBuf, 0, output, numFullGroups * 8, remaining); + pos += partialBytes; + } + + return pos; + } +} diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforValuesReaderForLong.java b/parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforValuesReaderForLong.java new file mode 100644 index 0000000000..1ce51611a1 --- /dev/null +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforValuesReaderForLong.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.values.pfor; + +import static org.apache.parquet.column.values.pfor.PforConstants.*; + +import java.nio.ByteBuffer; +import org.apache.parquet.column.values.bitpacking.BytePackerForLong; +import org.apache.parquet.column.values.bitpacking.Packer; +import org.apache.parquet.io.ParquetDecodingException; + +/** + * PFOR values reader for INT64 type with lazy per-vector decoding. + * + *

Reads PFOR-encoded long values from the PFOR page layout. + * Each vector is decoded on first access using BytePackerForLong-based unpacking. + * + *

Per-vector format: + * 

+ * PforVectorInfo (11B): frame_of_reference(8) + bit_width(1) + num_exceptions(2)
+ * PackedValues: ceil(N * bit_width / 8) bytes
+ * ExceptionPositions: num_exceptions * 2 bytes
+ * ExceptionValues: num_exceptions * 8 bytes
+ *
+ */ +public class PforValuesReaderForLong extends PforValuesReader { + + private long[] decodedValues; + + // Reusable per-vector decode buffers + private long[] deltasBuffer; + private int[] excPositionsBuffer; + private byte[] unpackPadBuf; + private long[] unpackTempBuf; + + public PforValuesReaderForLong() { + super(); + } + + @Override + protected void allocateDecodedBuffer(int capacity) { + this.decodedValues = new long[capacity]; + this.deltasBuffer = new long[capacity]; + this.excPositionsBuffer = new int[capacity]; + this.unpackPadBuf = new byte[Long.SIZE]; // max bit width = 64 bytes + this.unpackTempBuf = new long[8]; + } + + @Override + public long readLong() { + if (currentIndex >= totalCount) { + throw new ParquetDecodingException("PFOR long data was already exhausted."); + } + ensureVectorDecoded(); + int indexInVector = currentIndex % vectorSize; + currentIndex++; + return decodedValues[indexInVector]; + } + + @Override + protected void decodeVector(int vectorIdx) { + int vectorLen = getVectorLength(vectorIdx); + int pos = getVectorDataPosition(vectorIdx); + + // Read PforVectorInfo (11 bytes) + long frameOfReference = getLongLE(vectorsData, pos); + int bitWidth = vectorsData.get(pos + 8) & BIT_WIDTH_MASK; + int numExceptions = getShortLE(vectorsData, pos + 9) & 0xFFFF; + pos += INT64_VECTOR_INFO_SIZE; + checkVectorInfo(pos, bitWidth, numExceptions, vectorLen); + + // Unpack bit-packed deltas into reusable buffer + if (bitWidth > 0) { + pos = unpackLongsWithBytePacker(vectorsData, pos, deltasBuffer, vectorLen, bitWidth); + } else { + for (int i = 0; i < vectorLen; i++) { + deltasBuffer[i] = 0; + } + } + + // Add frame of reference to reconstruct values + for (int i = 0; i < vectorLen; i++) { + decodedValues[i] = deltasBuffer[i] + frameOfReference; + } + + // Overwrite exception slots with their original values + if (numExceptions > 0) { + for (int e = 0; e < numExceptions; e++) { + int position = getShortLE(vectorsData, pos) & 0xFFFF; + checkExceptionPosition(position, vectorLen); + excPositionsBuffer[e] = position; + pos += Short.BYTES; + } + for (int e = 0; e < numExceptions; e++) { + decodedValues[excPositionsBuffer[e]] = getLongLE(vectorsData, pos); + pos += Long.BYTES; + } + } + } + + private int unpackLongsWithBytePacker(ByteBuffer buf, int pos, long[] output, int count, int bitWidth) { + BytePackerForLong packer = Packer.LITTLE_ENDIAN.newBytePackerForLong(bitWidth); + int numFullGroups = count / 8; + int remaining = count % 8; + + for (int g = 0; g < numFullGroups; g++) { + packer.unpack8Values(buf, pos, output, g * 8); + pos += bitWidth; + } + + if (remaining > 0) { + int totalPackedBytes = (count * bitWidth + 7) / 8; + int alreadyRead = numFullGroups * bitWidth; + int partialBytes = totalPackedBytes - alreadyRead; + + for (int i = 0; i < partialBytes; i++) { + unpackPadBuf[i] = buf.get(pos + i); + } + for (int i = partialBytes; i < bitWidth; i++) { + unpackPadBuf[i] = 0; + } + + packer.unpack8Values(unpackPadBuf, 0, unpackTempBuf, 0); + System.arraycopy(unpackTempBuf, 0, output, numFullGroups * 8, remaining); + pos += partialBytes; + } + + return pos; + } +} diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforValuesWriter.java b/parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforValuesWriter.java new file mode 100644 index 0000000000..cb6524afce --- /dev/null +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforValuesWriter.java @@ -0,0 +1,487 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.values.pfor; + +import static org.apache.parquet.column.values.pfor.PforConstants.*; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.ArrayList; +import java.util.List; +import org.apache.parquet.bytes.ByteBufferAllocator; +import org.apache.parquet.bytes.BytesInput; +import org.apache.parquet.bytes.CapacityByteArrayOutputStream; +import org.apache.parquet.column.Encoding; +import org.apache.parquet.column.values.ValuesWriter; +import org.apache.parquet.column.values.bitpacking.BytePacker; +import org.apache.parquet.column.values.bitpacking.BytePackerForLong; +import org.apache.parquet.column.values.bitpacking.Packer; + +/** + * PFOR (Patched Frame of Reference) values writer for INT32 and INT64 columns. + * + *

PFOR compresses integer columns by subtracting the minimum value (FOR), + * selecting an optimal bit width via a histogram-based cost model, bit-packing + * the deltas, and storing outlier values (exceptions) separately. + * + *

Writing is incremental: values are buffered in a fixed-size vector buffer, + * and each full vector is encoded and flushed to the output stream immediately. + * On {@link #getBytes()}, any remaining partial vector is flushed, and the + * final page bytes are assembled. + * + *

Page layout: + * 

+ * ┌─────────┬──────────────────────┬──────────────┬──────────────┬─────┐
+ * │ Header  │ Offset Array         │ Vector 0     │ Vector 1     │ ... │
+ * │ 7 bytes │ 4B x numVectors      │ info + data  │ info + data  │     │
+ * └─────────┴──────────────────────┴──────────────┴──────────────┴─────┘
+ *
+ * + *

A vector carries its own info ahead of its data, so it can be decoded + * without reading any other vector: PforVectorInfo (7B for INT32, 11B for + * INT64), then the packed values, then the exception positions, then the + * exception values. + */ +public abstract class PforValuesWriter extends ValuesWriter { + + protected final int initialCapacity; + protected final int pageSize; + protected final ByteBufferAllocator allocator; + protected final int vectorSize; + protected final int logVectorSize; + + PforValuesWriter(int initialCapacity, int pageSize, ByteBufferAllocator allocator, int vectorSize) { + PforConstants.validateVectorSize(vectorSize); + this.initialCapacity = initialCapacity; + this.pageSize = pageSize; + this.allocator = allocator; + this.vectorSize = vectorSize; + this.logVectorSize = Integer.numberOfTrailingZeros(vectorSize); + } + + @Override + public Encoding getEncoding() { + return Encoding.PFOR; + } + + /** INT32 writer. Buffers one vector at a time, encodes and flushes when full. */ + public static class IntPforValuesWriter extends PforValuesWriter { + private final int[] vectorBuffer; + private int bufferCount; + private int totalCount; + private CapacityByteArrayOutputStream encodedVectors; + private final List vectorByteSizes; + + // Reusable per-vector buffers to avoid allocations on every encodeAndFlushVector call + private final int[] deltasBuffer; + private final short[] excPosBuffer; + private final int[] excValBuffer; + private final byte[] metadataBuf; + private final byte[] packBuf; + private final int[] packPadBuf; + + public IntPforValuesWriter(int initialCapacity, int pageSize, ByteBufferAllocator allocator) { + this(initialCapacity, pageSize, allocator, DEFAULT_VECTOR_SIZE); + } + + public IntPforValuesWriter(int initialCapacity, int pageSize, ByteBufferAllocator allocator, int vectorSize) { + super(initialCapacity, pageSize, allocator, vectorSize); + this.vectorBuffer = new int[vectorSize]; + this.bufferCount = 0; + this.totalCount = 0; + this.encodedVectors = new CapacityByteArrayOutputStream(initialCapacity, pageSize, allocator); + this.vectorByteSizes = new ArrayList<>(); + this.deltasBuffer = new int[vectorSize]; + this.excPosBuffer = new short[vectorSize]; + this.excValBuffer = new int[vectorSize]; + this.metadataBuf = new byte[INT32_VECTOR_INFO_SIZE]; + this.packBuf = new byte[Integer.SIZE]; // max bit width for int = 32 bytes + this.packPadBuf = new int[8]; + } + + @Override + public void writeInteger(int v) { + vectorBuffer[bufferCount++] = v; + totalCount++; + if (bufferCount == vectorSize) { + encodeAndFlushVector(bufferCount); + bufferCount = 0; + } + } + + private void encodeAndFlushVector(int vectorLen) { + // Find minimum value (frame of reference) + int minValue = vectorBuffer[0]; + for (int i = 1; i < vectorLen; i++) { + if (vectorBuffer[i] < minValue) { + minValue = vectorBuffer[i]; + } + } + + // Compute unsigned deltas into reusable buffer + for (int i = 0; i < vectorLen; i++) { + deltasBuffer[i] = vectorBuffer[i] - minValue; + } + + // Find optimal bit width via cost model + PforEncoderDecoder.BitWidthResult result = + PforEncoderDecoder.findOptimalBitWidthForInt(deltasBuffer, vectorLen); + int bitWidth = result.bitWidth; + int numExceptions = result.numExceptions; + + // Collect exceptions: values whose delta doesn't fit in bitWidth bits + int excIdx = 0; + if (numExceptions > 0) { + int mask = (bitWidth == 32) ? -1 : (1 << bitWidth) - 1; + for (int i = 0; i < vectorLen; i++) { + if (Integer.compareUnsigned(deltasBuffer[i], mask) > 0) { + excPosBuffer[excIdx] = (short) i; + excValBuffer[excIdx] = vectorBuffer[i]; + excIdx++; + deltasBuffer[i] = 0; + } + } + } + + long startSize = encodedVectors.size(); + + // PforVectorInfo: frame_of_reference(4) + bit_width(1) + num_exceptions(2) = 7B + metadataBuf[0] = (byte) (minValue & 0xFF); + metadataBuf[1] = (byte) ((minValue >>> 8) & 0xFF); + metadataBuf[2] = (byte) ((minValue >>> 16) & 0xFF); + metadataBuf[3] = (byte) ((minValue >>> 24) & 0xFF); + metadataBuf[4] = (byte) bitWidth; + metadataBuf[5] = (byte) (numExceptions & 0xFF); + metadataBuf[6] = (byte) ((numExceptions >>> 8) & 0xFF); + encodedVectors.write(metadataBuf, 0, INT32_VECTOR_INFO_SIZE); + + // Pack deltas + if (bitWidth > 0) { + packIntsWithBytePacker(deltasBuffer, vectorLen, bitWidth); + } + + // Exception positions then values + if (numExceptions > 0) { + for (int i = 0; i < numExceptions; i++) { + int pos = excPosBuffer[i] & 0xFFFF; + metadataBuf[0] = (byte) (pos & 0xFF); + metadataBuf[1] = (byte) ((pos >>> 8) & 0xFF); + encodedVectors.write(metadataBuf, 0, Short.BYTES); + } + + for (int i = 0; i < numExceptions; i++) { + int val = excValBuffer[i]; + metadataBuf[0] = (byte) (val & 0xFF); + metadataBuf[1] = (byte) ((val >>> 8) & 0xFF); + metadataBuf[2] = (byte) ((val >>> 16) & 0xFF); + metadataBuf[3] = (byte) ((val >>> 24) & 0xFF); + encodedVectors.write(metadataBuf, 0, Integer.BYTES); + } + } + + vectorByteSizes.add((int) (encodedVectors.size() - startSize)); + } + + private void packIntsWithBytePacker(int[] values, int count, int bitWidth) { + BytePacker packer = Packer.LITTLE_ENDIAN.newBytePacker(bitWidth); + int numFullGroups = count / 8; + int remaining = count % 8; + + for (int g = 0; g < numFullGroups; g++) { + packer.pack8Values(values, g * 8, packBuf, 0); + encodedVectors.write(packBuf, 0, bitWidth); + } + + if (remaining > 0) { + System.arraycopy(values, numFullGroups * 8, packPadBuf, 0, remaining); + for (int i = remaining; i < 8; i++) { + packPadBuf[i] = 0; + } + packer.pack8Values(packPadBuf, 0, packBuf, 0); + int totalPackedBytes = (count * bitWidth + 7) / 8; + int alreadyWritten = numFullGroups * bitWidth; + encodedVectors.write(packBuf, 0, totalPackedBytes - alreadyWritten); + } + } + + @Override + public long getBufferedSize() { + return encodedVectors.size() + (long) bufferCount * Integer.BYTES; + } + + @Override + public BytesInput getBytes() { + if (bufferCount > 0) { + encodeAndFlushVector(bufferCount); + bufferCount = 0; + } + + int numVectors = vectorByteSizes.size(); + + // Header: packing_mode(1) + log_vector_size(1) + value_byte_width(1) + num_elements(4) = 7B + ByteBuffer header = ByteBuffer.allocate(PFOR_HEADER_SIZE).order(ByteOrder.LITTLE_ENDIAN); + header.put((byte) PFOR_PACKING_MODE_FOR); + header.put((byte) logVectorSize); + header.put((byte) INT32_VALUE_BYTE_WIDTH); + header.putInt(totalCount); + + if (totalCount == 0) { + return BytesInput.from(header.array()); + } + + int offsetArraySize = numVectors * Integer.BYTES; + ByteBuffer offsets = ByteBuffer.allocate(offsetArraySize).order(ByteOrder.LITTLE_ENDIAN); + int currentOffset = offsetArraySize; + for (int v = 0; v < numVectors; v++) { + offsets.putInt(currentOffset); + currentOffset += vectorByteSizes.get(v); + } + + return BytesInput.concat( + BytesInput.from(header.array()), BytesInput.from(offsets.array()), BytesInput.from(encodedVectors)); + } + + @Override + public void reset() { + bufferCount = 0; + totalCount = 0; + encodedVectors.reset(); + vectorByteSizes.clear(); + } + + @Override + public void close() { + encodedVectors.close(); + } + + @Override + public long getAllocatedSize() { + return (long) vectorBuffer.length * Integer.BYTES + encodedVectors.getCapacity(); + } + + @Override + public String memUsageString(String prefix) { + return String.format( + "%s IntPforValuesWriter %d values, %d bytes allocated", prefix, totalCount, getAllocatedSize()); + } + } + + /** INT64 writer. Same structure as IntPforValuesWriter but uses longs. */ + public static class LongPforValuesWriter extends PforValuesWriter { + private final long[] vectorBuffer; + private int bufferCount; + private int totalCount; + private CapacityByteArrayOutputStream encodedVectors; + private final List vectorByteSizes; + + // Reusable per-vector buffers + private final long[] deltasBuffer; + private final short[] excPosBuffer; + private final long[] excValBuffer; + private final byte[] metadataBuf; + private final byte[] packBuf; + private final long[] packPadBuf; + + public LongPforValuesWriter(int initialCapacity, int pageSize, ByteBufferAllocator allocator) { + this(initialCapacity, pageSize, allocator, DEFAULT_VECTOR_SIZE); + } + + public LongPforValuesWriter(int initialCapacity, int pageSize, ByteBufferAllocator allocator, int vectorSize) { + super(initialCapacity, pageSize, allocator, vectorSize); + this.vectorBuffer = new long[vectorSize]; + this.bufferCount = 0; + this.totalCount = 0; + this.encodedVectors = new CapacityByteArrayOutputStream(initialCapacity, pageSize, allocator); + this.vectorByteSizes = new ArrayList<>(); + this.deltasBuffer = new long[vectorSize]; + this.excPosBuffer = new short[vectorSize]; + this.excValBuffer = new long[vectorSize]; + this.metadataBuf = new byte[INT64_VECTOR_INFO_SIZE]; + this.packBuf = new byte[Long.SIZE]; // max bit width for long = 64 bytes + this.packPadBuf = new long[8]; + } + + @Override + public void writeLong(long v) { + vectorBuffer[bufferCount++] = v; + totalCount++; + if (bufferCount == vectorSize) { + encodeAndFlushVector(bufferCount); + bufferCount = 0; + } + } + + private void encodeAndFlushVector(int vectorLen) { + long minValue = vectorBuffer[0]; + for (int i = 1; i < vectorLen; i++) { + if (vectorBuffer[i] < minValue) { + minValue = vectorBuffer[i]; + } + } + + for (int i = 0; i < vectorLen; i++) { + deltasBuffer[i] = vectorBuffer[i] - minValue; + } + + PforEncoderDecoder.BitWidthResult result = + PforEncoderDecoder.findOptimalBitWidthForLong(deltasBuffer, vectorLen); + int bitWidth = result.bitWidth; + int numExceptions = result.numExceptions; + + int excIdx = 0; + if (numExceptions > 0) { + long mask = (bitWidth == 64) ? -1L : (1L << bitWidth) - 1L; + for (int i = 0; i < vectorLen; i++) { + if (Long.compareUnsigned(deltasBuffer[i], mask) > 0) { + excPosBuffer[excIdx] = (short) i; + excValBuffer[excIdx] = vectorBuffer[i]; + excIdx++; + deltasBuffer[i] = 0; + } + } + } + + long startSize = encodedVectors.size(); + + // PforVectorInfo: frame_of_reference(8) + bit_width(1) + num_exceptions(2) = 11B + metadataBuf[0] = (byte) (minValue & 0xFF); + metadataBuf[1] = (byte) ((minValue >>> 8) & 0xFF); + metadataBuf[2] = (byte) ((minValue >>> 16) & 0xFF); + metadataBuf[3] = (byte) ((minValue >>> 24) & 0xFF); + metadataBuf[4] = (byte) ((minValue >>> 32) & 0xFF); + metadataBuf[5] = (byte) ((minValue >>> 40) & 0xFF); + metadataBuf[6] = (byte) ((minValue >>> 48) & 0xFF); + metadataBuf[7] = (byte) ((minValue >>> 56) & 0xFF); + metadataBuf[8] = (byte) bitWidth; + metadataBuf[9] = (byte) (numExceptions & 0xFF); + metadataBuf[10] = (byte) ((numExceptions >>> 8) & 0xFF); + encodedVectors.write(metadataBuf, 0, INT64_VECTOR_INFO_SIZE); + + if (bitWidth > 0) { + packLongsWithBytePacker(deltasBuffer, vectorLen, bitWidth); + } + + if (numExceptions > 0) { + for (int i = 0; i < numExceptions; i++) { + int pos = excPosBuffer[i] & 0xFFFF; + metadataBuf[0] = (byte) (pos & 0xFF); + metadataBuf[1] = (byte) ((pos >>> 8) & 0xFF); + encodedVectors.write(metadataBuf, 0, Short.BYTES); + } + + for (int i = 0; i < numExceptions; i++) { + long val = excValBuffer[i]; + metadataBuf[0] = (byte) (val & 0xFF); + metadataBuf[1] = (byte) ((val >>> 8) & 0xFF); + metadataBuf[2] = (byte) ((val >>> 16) & 0xFF); + metadataBuf[3] = (byte) ((val >>> 24) & 0xFF); + metadataBuf[4] = (byte) ((val >>> 32) & 0xFF); + metadataBuf[5] = (byte) ((val >>> 40) & 0xFF); + metadataBuf[6] = (byte) ((val >>> 48) & 0xFF); + metadataBuf[7] = (byte) ((val >>> 56) & 0xFF); + encodedVectors.write(metadataBuf, 0, Long.BYTES); + } + } + + vectorByteSizes.add((int) (encodedVectors.size() - startSize)); + } + + private void packLongsWithBytePacker(long[] values, int count, int bitWidth) { + BytePackerForLong packer = Packer.LITTLE_ENDIAN.newBytePackerForLong(bitWidth); + int numFullGroups = count / 8; + int remaining = count % 8; + + for (int g = 0; g < numFullGroups; g++) { + packer.pack8Values(values, g * 8, packBuf, 0); + encodedVectors.write(packBuf, 0, bitWidth); + } + + if (remaining > 0) { + System.arraycopy(values, numFullGroups * 8, packPadBuf, 0, remaining); + for (int i = remaining; i < 8; i++) { + packPadBuf[i] = 0; + } + packer.pack8Values(packPadBuf, 0, packBuf, 0); + int totalPackedBytes = (count * bitWidth + 7) / 8; + int alreadyWritten = numFullGroups * bitWidth; + encodedVectors.write(packBuf, 0, totalPackedBytes - alreadyWritten); + } + } + + @Override + public long getBufferedSize() { + return encodedVectors.size() + (long) bufferCount * Long.BYTES; + } + + @Override + public BytesInput getBytes() { + if (bufferCount > 0) { + encodeAndFlushVector(bufferCount); + bufferCount = 0; + } + + int numVectors = vectorByteSizes.size(); + + ByteBuffer header = ByteBuffer.allocate(PFOR_HEADER_SIZE).order(ByteOrder.LITTLE_ENDIAN); + header.put((byte) PFOR_PACKING_MODE_FOR); + header.put((byte) logVectorSize); + header.put((byte) INT64_VALUE_BYTE_WIDTH); + header.putInt(totalCount); + + if (totalCount == 0) { + return BytesInput.from(header.array()); + } + + int offsetArraySize = numVectors * Integer.BYTES; + ByteBuffer offsets = ByteBuffer.allocate(offsetArraySize).order(ByteOrder.LITTLE_ENDIAN); + int currentOffset = offsetArraySize; + for (int v = 0; v < numVectors; v++) { + offsets.putInt(currentOffset); + currentOffset += vectorByteSizes.get(v); + } + + return BytesInput.concat( + BytesInput.from(header.array()), BytesInput.from(offsets.array()), BytesInput.from(encodedVectors)); + } + + @Override + public void reset() { + bufferCount = 0; + totalCount = 0; + encodedVectors.reset(); + vectorByteSizes.clear(); + } + + @Override + public void close() { + encodedVectors.close(); + } + + @Override + public long getAllocatedSize() { + return (long) vectorBuffer.length * Long.BYTES + encodedVectors.getCapacity(); + } + + @Override + public String memUsageString(String prefix) { + return String.format( + "%s LongPforValuesWriter %d values, %d bytes allocated", prefix, totalCount, getAllocatedSize()); + } + } +} diff --git a/parquet-column/src/test/java/org/apache/parquet/column/values/pfor/PforAdversarialTest.java b/parquet-column/src/test/java/org/apache/parquet/column/values/pfor/PforAdversarialTest.java new file mode 100644 index 0000000000..5ff5780c82 --- /dev/null +++ b/parquet-column/src/test/java/org/apache/parquet/column/values/pfor/PforAdversarialTest.java @@ -0,0 +1,461 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.values.pfor; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.nio.ByteBuffer; +import org.apache.parquet.bytes.ByteBufferInputStream; +import org.apache.parquet.bytes.BytesInput; +import org.apache.parquet.bytes.DirectByteBufferAllocator; +import org.apache.parquet.io.ParquetDecodingException; +import org.junit.Test; + +/** + * Adversarial tests for PFOR readers: feed malformed page bytes and assert the reader + * fails cleanly rather than crashing, producing silent garbage, or hanging. + * + *

Covers both explicitly-validated cases (ParquetDecodingException with message) + * and currently-unvalidated cases (IndexOutOfBoundsException or BufferUnderflowException + * from the underlying ByteBuffer). + */ +public class PforAdversarialTest { + + private static final int VECTOR_SIZE = PforConstants.DEFAULT_VECTOR_SIZE; + + // The five-element pages below hold one vector, so the page is a 7-byte header, + // a single 4-byte offset, and then that vector's info. + private static final int VECTOR_START = PforConstants.PFOR_HEADER_SIZE + Integer.BYTES; + private static final int OUTLIER_VECTOR_LEN = 5; + + private static byte[] validIntPage(int valueCount, int vectorSize) throws Exception { + PforValuesWriter.IntPforValuesWriter writer = null; + try { + int cap = Math.max(512, valueCount * 8); + writer = new PforValuesWriter.IntPforValuesWriter(cap, cap, new DirectByteBufferAllocator(), vectorSize); + for (int i = 0; i < valueCount; i++) { + writer.writeInteger(i * 7 + 3); + } + BytesInput bi = writer.getBytes(); + ByteBuffer bb = bi.toByteBuffer(); + byte[] out = new byte[bb.remaining()]; + bb.duplicate().get(out); + return out; + } finally { + if (writer != null) { + writer.reset(); + writer.close(); + } + } + } + + private static byte[] validLongPage(int valueCount, int vectorSize) throws Exception { + PforValuesWriter.LongPforValuesWriter writer = null; + try { + int cap = Math.max(512, valueCount * 16); + writer = new PforValuesWriter.LongPforValuesWriter(cap, cap, new DirectByteBufferAllocator(), vectorSize); + for (int i = 0; i < valueCount; i++) { + writer.writeLong((long) i * 13 + 5); + } + BytesInput bi = writer.getBytes(); + ByteBuffer bb = bi.toByteBuffer(); + byte[] out = new byte[bb.remaining()]; + bb.duplicate().get(out); + return out; + } finally { + if (writer != null) { + writer.reset(); + writer.close(); + } + } + } + + // One value far above the cluster, which is the shape the cost model stores as an + // exception; a low outlier would become the frame of reference instead. + private static final int[] OUTLIER_INTS = {100, 101, 102, 103, 50000}; + + // An INT64 exception costs 80 bits, so the outlier has to sit further out still: + // 50,000 would be cheaper to pack at full width than to store as an exception. + private static final long[] OUTLIER_LONGS = {100L, 101L, 102L, 103L, 100L + (1L << 40)}; + + private static byte[] outlierIntPage() throws Exception { + PforValuesWriter.IntPforValuesWriter writer = null; + try { + writer = new PforValuesWriter.IntPforValuesWriter(512, 512, new DirectByteBufferAllocator(), 8); + for (int value : OUTLIER_INTS) { + writer.writeInteger(value); + } + return toBytes(writer.getBytes()); + } finally { + if (writer != null) { + writer.reset(); + writer.close(); + } + } + } + + private static byte[] outlierLongPage() throws Exception { + PforValuesWriter.LongPforValuesWriter writer = null; + try { + writer = new PforValuesWriter.LongPforValuesWriter(512, 512, new DirectByteBufferAllocator(), 8); + for (long value : OUTLIER_LONGS) { + writer.writeLong(value); + } + return toBytes(writer.getBytes()); + } finally { + if (writer != null) { + writer.reset(); + writer.close(); + } + } + } + + private static byte[] toBytes(BytesInput bytes) throws Exception { + ByteBuffer bb = bytes.toByteBuffer(); + byte[] out = new byte[bb.remaining()]; + bb.duplicate().get(out); + return out; + } + + private static int bitWidthOffset(int vectorInfoSize) { + return VECTOR_START + vectorInfoSize - 3; + } + + private static int numExceptionsOffset(int vectorInfoSize) { + return VECTOR_START + vectorInfoSize - 2; + } + + // Offset of the first stored exception position: past the vector info and the + // packed deltas, whose length depends on the width the writer chose. + private static int exceptionPositionOffset(byte[] page, int vectorInfoSize) { + int bitWidth = page[bitWidthOffset(vectorInfoSize)] & PforConstants.BIT_WIDTH_MASK; + int packedBytes = (OUTLIER_VECTOR_LEN * bitWidth + 7) / 8; + return VECTOR_START + vectorInfoSize + packedBytes; + } + + private static int shortLE(byte[] page, int pos) { + return (page[pos] & 0xFF) | ((page[pos + 1] & 0xFF) << 8); + } + + private static byte[] putShortLE(byte[] original, int pos, int value) { + byte[] copy = original.clone(); + copy[pos] = (byte) (value & 0xFF); + copy[pos + 1] = (byte) ((value >>> 8) & 0xFF); + return copy; + } + + private static byte[] mutate(byte[] original, int offset, byte value) { + byte[] copy = original.clone(); + copy[offset] = value; + return copy; + } + + private static byte[] truncate(byte[] original, int newLen) { + byte[] copy = new byte[newLen]; + System.arraycopy(original, 0, copy, 0, newLen); + return copy; + } + + private static void initIntReader(byte[] page, int valuesCount) throws Exception { + PforValuesReaderForInt reader = new PforValuesReaderForInt(); + reader.initFromPage(valuesCount, ByteBufferInputStream.wrap(ByteBuffer.wrap(page))); + reader.readInteger(); + } + + private static void initLongReader(byte[] page, int valuesCount) throws Exception { + PforValuesReaderForLong reader = new PforValuesReaderForLong(); + reader.initFromPage(valuesCount, ByteBufferInputStream.wrap(ByteBuffer.wrap(page))); + reader.readLong(); + } + + // Sanity: valid pages decode cleanly + + @Test + public void sanityBaselineDecodesClean() throws Exception { + byte[] page = validIntPage(2048, VECTOR_SIZE); + PforValuesReaderForInt reader = new PforValuesReaderForInt(); + reader.initFromPage(2048, ByteBufferInputStream.wrap(ByteBuffer.wrap(page))); + for (int i = 0; i < 2048; i++) { + reader.readInteger(); + } + } + + @Test + public void sanityBaselineLongDecodesClean() throws Exception { + byte[] page = validLongPage(2048, VECTOR_SIZE); + PforValuesReaderForLong reader = new PforValuesReaderForLong(); + reader.initFromPage(2048, ByteBufferInputStream.wrap(ByteBuffer.wrap(page))); + for (int i = 0; i < 2048; i++) { + reader.readLong(); + } + } + + // Header validation + + @Test + public void rejectsBadPackingMode() throws Exception { + byte[] page = validIntPage(1024, VECTOR_SIZE); + byte[] bad = mutate(page, 0, (byte) 99); + assertThrows(ParquetDecodingException.class, () -> initIntReader(bad, 1024)); + } + + @Test + public void rejectsLogVectorSizeTooLarge() throws Exception { + byte[] page = validIntPage(1024, VECTOR_SIZE); + byte[] bad = mutate(page, 1, (byte) 16); // MAX_LOG_VECTOR_SIZE is 15 + assertThrows(ParquetDecodingException.class, () -> initIntReader(bad, 1024)); + } + + @Test + public void rejectsLogVectorSizeTooSmall() throws Exception { + byte[] page = validIntPage(1024, VECTOR_SIZE); + byte[] bad = mutate(page, 1, (byte) 2); // MIN_LOG_VECTOR_SIZE is 3 + assertThrows(ParquetDecodingException.class, () -> initIntReader(bad, 1024)); + } + + @Test + public void rejectsBadValueByteWidth() throws Exception { + byte[] page = validIntPage(1024, VECTOR_SIZE); + byte[] bad = mutate(page, 2, (byte) 3); // must be 4 or 8 + assertThrows(ParquetDecodingException.class, () -> initIntReader(bad, 1024)); + } + + @Test + public void rejectsNegativeNumElements() throws Exception { + byte[] page = validIntPage(1024, VECTOR_SIZE); + // Overwrite num_elements (bytes 3-6) with -1 (0xFFFFFFFF) + byte[] bad = page.clone(); + bad[3] = (byte) 0xFF; + bad[4] = (byte) 0xFF; + bad[5] = (byte) 0xFF; + bad[6] = (byte) 0xFF; + assertThrows(ParquetDecodingException.class, () -> initIntReader(bad, 1024)); + } + + @Test + public void rejectsNumElementsGreaterThanValuesCount() throws Exception { + byte[] page = validIntPage(1024, VECTOR_SIZE); + // page header says 1024 elements but we pass valuesCount=500 + assertThrows(ParquetDecodingException.class, () -> initIntReader(page, 500)); + } + + // Truncation / corruption + + @Test + public void rejectsHeaderOnlyPage() { + byte[] page = new byte[PforConstants.PFOR_HEADER_SIZE]; + page[0] = (byte) PforConstants.PFOR_PACKING_MODE_FOR; + page[1] = (byte) PforConstants.DEFAULT_VECTOR_SIZE_LOG; + page[2] = (byte) PforConstants.INT32_VALUE_BYTE_WIDTH; + // num_elements = 100 in LE + page[3] = 100; + page[4] = 0; + page[5] = 0; + page[6] = 0; + + try { + initIntReader(page, 100); + fail("Expected exception for header-only page"); + } catch (Throwable t) { + assertNotNull(t); + } + } + + @Test + public void rejectsPageTruncatedMidOffsetArray() throws Exception { + byte[] page = validIntPage(2048, VECTOR_SIZE); + // Truncate inside the offset array (header=7 + partial offsets) + byte[] bad = truncate(page, PforConstants.PFOR_HEADER_SIZE + 2); + try { + initIntReader(bad, 2048); + fail("Expected exception for page truncated mid offset array"); + } catch (Throwable t) { + assertNotNull(t); + } + } + + @Test + public void rejectsPageTruncatedMidVectorData() throws Exception { + byte[] page = validIntPage(2048, VECTOR_SIZE); + // Keep header + offset array but truncate vector data + int offsetArrayEnd = PforConstants.PFOR_HEADER_SIZE + 2 * Integer.BYTES; + byte[] bad = truncate(page, offsetArrayEnd + 3); + try { + initIntReader(bad, 2048); + fail("Expected exception for page truncated mid vector data"); + } catch (Throwable t) { + assertNotNull(t); + } + } + + @Test + public void rejectsCorruptedOffsetPointingPastEnd() throws Exception { + byte[] page = validIntPage(1024, VECTOR_SIZE); + // The offset array starts at byte 7. Overwrite first offset to point past buffer end. + byte[] bad = page.clone(); + int hugeOffset = page.length * 2; + bad[7] = (byte) (hugeOffset & 0xFF); + bad[8] = (byte) ((hugeOffset >>> 8) & 0xFF); + bad[9] = (byte) ((hugeOffset >>> 16) & 0xFF); + bad[10] = (byte) ((hugeOffset >>> 24) & 0xFF); + try { + initIntReader(bad, 1024); + fail("Expected exception for corrupted offset"); + } catch (Throwable t) { + assertNotNull(t); + } + } + + // Per-vector info validation + + @Test + public void sanityOutlierPagesCarryAnException() throws Exception { + assertTrue( + "the tests below relocate a stored exception, so the int page must have one", + shortLE(outlierIntPage(), numExceptionsOffset(PforConstants.INT32_VECTOR_INFO_SIZE)) > 0); + assertTrue( + "the tests below relocate a stored exception, so the long page must have one", + shortLE(outlierLongPage(), numExceptionsOffset(PforConstants.INT64_VECTOR_INFO_SIZE)) > 0); + } + + @Test + public void rejectsExceptionCountAboveVectorLength() throws Exception { + byte[] page = outlierIntPage(); + byte[] bad = putShortLE(page, numExceptionsOffset(PforConstants.INT32_VECTOR_INFO_SIZE), 6); + assertThrows(ParquetDecodingException.class, () -> initIntReader(bad, OUTLIER_VECTOR_LEN)); + } + + @Test + public void rejectsExceptionPositionPastEndOfVector() throws Exception { + byte[] page = outlierIntPage(); + byte[] bad = putShortLE(page, exceptionPositionOffset(page, PforConstants.INT32_VECTOR_INFO_SIZE), 100); + assertThrows(ParquetDecodingException.class, () -> initIntReader(bad, OUTLIER_VECTOR_LEN)); + } + + @Test + public void rejectsExceptionPositionPastEndOfVectorLong() throws Exception { + byte[] page = outlierLongPage(); + byte[] bad = putShortLE(page, exceptionPositionOffset(page, PforConstants.INT64_VECTOR_INFO_SIZE), 100); + assertThrows(ParquetDecodingException.class, () -> initLongReader(bad, OUTLIER_VECTOR_LEN)); + } + + @Test + public void rejectsBitWidthAboveValueWidth() throws Exception { + byte[] page = outlierIntPage(); + byte[] bad = mutate(page, bitWidthOffset(PforConstants.INT32_VECTOR_INFO_SIZE), (byte) 33); + assertThrows(ParquetDecodingException.class, () -> initIntReader(bad, OUTLIER_VECTOR_LEN)); + } + + @Test + public void rejectsExceptionValuesTruncated() throws Exception { + byte[] page = outlierIntPage(); + byte[] bad = truncate(page, page.length - Integer.BYTES); + assertThrows(ParquetDecodingException.class, () -> initIntReader(bad, OUTLIER_VECTOR_LEN)); + } + + // Bit 7 of the bit width byte is reserved, so a writer that sets it must not + // change what a reader decodes. + @Test + public void ignoresReservedBitInBitWidth() throws Exception { + byte[] page = outlierIntPage(); + int at = bitWidthOffset(PforConstants.INT32_VECTOR_INFO_SIZE); + byte[] withReservedBit = mutate(page, at, (byte) (page[at] | 0x80)); + + PforValuesReaderForInt reader = new PforValuesReaderForInt(); + reader.initFromPage(OUTLIER_VECTOR_LEN, ByteBufferInputStream.wrap(ByteBuffer.wrap(withReservedBit))); + for (int expected : OUTLIER_INTS) { + assertEquals(expected, reader.readInteger()); + } + } + + // Skip/read bounds + + @Test + public void rejectsSkipPastEnd() throws Exception { + byte[] page = validIntPage(100, 8); + PforValuesReaderForInt reader = new PforValuesReaderForInt(); + reader.initFromPage(100, ByteBufferInputStream.wrap(ByteBuffer.wrap(page))); + assertThrows(ParquetDecodingException.class, () -> reader.skip(101)); + } + + @Test + public void rejectsNegativeSkip() throws Exception { + byte[] page = validIntPage(100, 8); + PforValuesReaderForInt reader = new PforValuesReaderForInt(); + reader.initFromPage(100, ByteBufferInputStream.wrap(ByteBuffer.wrap(page))); + assertThrows(ParquetDecodingException.class, () -> reader.skip(-1)); + } + + @Test + public void rejectsReadPastEnd() throws Exception { + byte[] page = validIntPage(10, 8); + PforValuesReaderForInt reader = new PforValuesReaderForInt(); + reader.initFromPage(10, ByteBufferInputStream.wrap(ByteBuffer.wrap(page))); + for (int i = 0; i < 10; i++) { + reader.readInteger(); + } + assertThrows(ParquetDecodingException.class, reader::readInteger); + } + + @Test + public void rejectsLongReadPastEnd() throws Exception { + byte[] page = validLongPage(10, 8); + PforValuesReaderForLong reader = new PforValuesReaderForLong(); + reader.initFromPage(10, ByteBufferInputStream.wrap(ByteBuffer.wrap(page))); + for (int i = 0; i < 10; i++) { + reader.readLong(); + } + assertThrows(ParquetDecodingException.class, reader::readLong); + } + + // Skip across vector boundaries works correctly + + @Test + public void skipAcrossVectorBoundary() throws Exception { + int vectorSize = 8; + int count = 30; + byte[] page = validIntPage(count, vectorSize); + PforValuesReaderForInt reader = new PforValuesReaderForInt(); + reader.initFromPage(count, ByteBufferInputStream.wrap(ByteBuffer.wrap(page))); + + // Skip past first two vectors (16 values), read from third + reader.skip(16); + int val = reader.readInteger(); + // Expected: 16 * 7 + 3 = 115 + assertTrue("Value after skip should be 115, got: " + val, val == 115); + } + + @Test + public void skipAcrossVectorBoundaryLong() throws Exception { + int vectorSize = 8; + int count = 30; + byte[] page = validLongPage(count, vectorSize); + PforValuesReaderForLong reader = new PforValuesReaderForLong(); + reader.initFromPage(count, ByteBufferInputStream.wrap(ByteBuffer.wrap(page))); + + reader.skip(16); + long val = reader.readLong(); + // Expected: 16 * 13 + 5 = 213 + assertTrue("Value after skip should be 213, got: " + val, val == 213L); + } +} diff --git a/parquet-column/src/test/java/org/apache/parquet/column/values/pfor/PforBitPackingTest.java b/parquet-column/src/test/java/org/apache/parquet/column/values/pfor/PforBitPackingTest.java new file mode 100644 index 0000000000..037eae534f --- /dev/null +++ b/parquet-column/src/test/java/org/apache/parquet/column/values/pfor/PforBitPackingTest.java @@ -0,0 +1,260 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.values.pfor; + +import static org.junit.Assert.*; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import org.apache.parquet.bytes.ByteBufferInputStream; +import org.apache.parquet.bytes.BytesInput; +import org.apache.parquet.bytes.DirectByteBufferAllocator; +import org.junit.Test; + +/** + * Tests focused on PFOR bit-packing correctness across different bit widths + * and vector sizes. + */ +public class PforBitPackingTest { + + // Round-trip helper that verifies bit-packing for int values in a given range + private void verifyIntRoundTrip(int[] values) throws Exception { + int capacity = Math.max(256, values.length * 8); + PforValuesWriter.IntPforValuesWriter writer = + new PforValuesWriter.IntPforValuesWriter(capacity, capacity, new DirectByteBufferAllocator()); + + for (int v : values) { + writer.writeInteger(v); + } + + BytesInput bytes = writer.getBytes(); + PforValuesReaderForInt reader = new PforValuesReaderForInt(); + reader.initFromPage(values.length, ByteBufferInputStream.wrap(bytes.toByteBuffer())); + + for (int i = 0; i < values.length; i++) { + assertEquals("Mismatch at index " + i, values[i], reader.readInteger()); + } + + writer.close(); + } + + private void verifyLongRoundTrip(long[] values) throws Exception { + int capacity = Math.max(512, values.length * 16); + PforValuesWriter.LongPforValuesWriter writer = + new PforValuesWriter.LongPforValuesWriter(capacity, capacity, new DirectByteBufferAllocator()); + + for (long v : values) { + writer.writeLong(v); + } + + BytesInput bytes = writer.getBytes(); + PforValuesReaderForLong reader = new PforValuesReaderForLong(); + reader.initFromPage(values.length, ByteBufferInputStream.wrap(bytes.toByteBuffer())); + + for (int i = 0; i < values.length; i++) { + assertEquals("Mismatch at index " + i, values[i], reader.readLong()); + } + + writer.close(); + } + + @Test + public void testIntBitWidth0() throws Exception { + // All same value -> bitWidth=0, no packed bytes + int[] values = new int[100]; + java.util.Arrays.fill(values, 777); + verifyIntRoundTrip(values); + } + + @Test + public void testIntBitWidth1() throws Exception { + // Values: base + {0, 1} + int[] values = new int[100]; + for (int i = 0; i < 100; i++) { + values[i] = 1000 + (i % 2); + } + verifyIntRoundTrip(values); + } + + @Test + public void testIntBitWidth8() throws Exception { + int[] values = new int[1024]; + for (int i = 0; i < 1024; i++) { + values[i] = 5000 + (i % 256); + } + verifyIntRoundTrip(values); + } + + @Test + public void testIntBitWidth16() throws Exception { + int[] values = new int[1024]; + for (int i = 0; i < 1024; i++) { + values[i] = i; + } + verifyIntRoundTrip(values); + } + + @Test + public void testIntBitWidth32() throws Exception { + // Full range int values + int[] values = {Integer.MIN_VALUE, -1, 0, 1, Integer.MAX_VALUE, 0x7FFFFFFF, 0x40000000, -2147483648}; + verifyIntRoundTrip(values); + } + + @Test + public void testIntPartialGroup() throws Exception { + // 13 values: 1 full group of 8 + 5 remaining + int[] values = new int[13]; + for (int i = 0; i < 13; i++) { + values[i] = i * 100; + } + verifyIntRoundTrip(values); + } + + @Test + public void testIntExactlyOneGroup() throws Exception { + // Exactly 8 values + int[] values = {10, 20, 30, 40, 50, 60, 70, 80}; + verifyIntRoundTrip(values); + } + + @Test + public void testIntSevenValues() throws Exception { + // Less than one full group + int[] values = {1, 2, 3, 4, 5, 6, 7}; + verifyIntRoundTrip(values); + } + + @Test + public void testLongBitWidth0() throws Exception { + long[] values = new long[100]; + java.util.Arrays.fill(values, 123456789L); + verifyLongRoundTrip(values); + } + + @Test + public void testLongBitWidth1() throws Exception { + long[] values = new long[100]; + for (int i = 0; i < 100; i++) { + values[i] = 1_000_000L + (i % 2); + } + verifyLongRoundTrip(values); + } + + @Test + public void testLongBitWidth32() throws Exception { + long[] values = new long[1024]; + for (int i = 0; i < 1024; i++) { + values[i] = (long) i * 1_000_000L; + } + verifyLongRoundTrip(values); + } + + @Test + public void testLongBitWidth64() throws Exception { + long[] values = {Long.MIN_VALUE, Long.MAX_VALUE, 0L, -1L, 1L}; + verifyLongRoundTrip(values); + } + + @Test + public void testLongPartialGroup() throws Exception { + long[] values = new long[13]; + for (int i = 0; i < 13; i++) { + values[i] = i * 100_000L; + } + verifyLongRoundTrip(values); + } + + @Test + public void testIntPageHeaderFormat() throws Exception { + int[] values = new int[100]; + for (int i = 0; i < 100; i++) { + values[i] = i; + } + + PforValuesWriter.IntPforValuesWriter writer = + new PforValuesWriter.IntPforValuesWriter(1024, 1024, new DirectByteBufferAllocator()); + for (int v : values) { + writer.writeInteger(v); + } + + ByteBuffer buf = writer.getBytes().toByteBuffer().order(ByteOrder.LITTLE_ENDIAN); + + // Verify header + assertEquals("packing_mode", 0, buf.get(0) & 0xFF); + assertEquals("log_vector_size", 10, buf.get(1) & 0xFF); + assertEquals("value_byte_width", 4, buf.get(2) & 0xFF); + + buf.position(3); + assertEquals("num_elements", 100, buf.getInt()); + + writer.close(); + } + + @Test + public void testLongPageHeaderFormat() throws Exception { + long[] values = new long[100]; + for (int i = 0; i < 100; i++) { + values[i] = i; + } + + PforValuesWriter.LongPforValuesWriter writer = + new PforValuesWriter.LongPforValuesWriter(1024, 1024, new DirectByteBufferAllocator()); + for (long v : values) { + writer.writeLong(v); + } + + ByteBuffer buf = writer.getBytes().toByteBuffer().order(ByteOrder.LITTLE_ENDIAN); + + assertEquals("packing_mode", 0, buf.get(0) & 0xFF); + assertEquals("log_vector_size", 10, buf.get(1) & 0xFF); + assertEquals("value_byte_width", 8, buf.get(2) & 0xFF); + + buf.position(3); + assertEquals("num_elements", 100, buf.getInt()); + + writer.close(); + } + + @Test + public void testIntManyExceptions() throws Exception { + // More than half are outliers — cost model should widen bit width + int[] values = new int[100]; + for (int i = 0; i < 100; i++) { + values[i] = (i % 2 == 0) ? i : 1_000_000 + i; + } + verifyIntRoundTrip(values); + } + + @Test + public void testIntAllExceptionsScenario() throws Exception { + // Very wide range but few values: might make everything exceptions or wide pack + int[] values = {0, Integer.MAX_VALUE, Integer.MIN_VALUE, 42, -42}; + verifyIntRoundTrip(values); + } + + @Test + public void testLongManyExceptions() throws Exception { + long[] values = new long[100]; + for (int i = 0; i < 100; i++) { + values[i] = (i % 2 == 0) ? i : Long.MAX_VALUE - i; + } + verifyLongRoundTrip(values); + } +} diff --git a/parquet-column/src/test/java/org/apache/parquet/column/values/pfor/PforEncoderDecoderTest.java b/parquet-column/src/test/java/org/apache/parquet/column/values/pfor/PforEncoderDecoderTest.java new file mode 100644 index 0000000000..f458ddf106 --- /dev/null +++ b/parquet-column/src/test/java/org/apache/parquet/column/values/pfor/PforEncoderDecoderTest.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.values.pfor; + +import static org.junit.Assert.*; + +import org.junit.Test; + +/** + * Tests for PFOR cost model and bit width utilities. + */ +public class PforEncoderDecoderTest { + + @Test + public void testBitWidthForInt() { + assertEquals(0, PforEncoderDecoder.bitWidthForInt(0)); + assertEquals(1, PforEncoderDecoder.bitWidthForInt(1)); + assertEquals(2, PforEncoderDecoder.bitWidthForInt(2)); + assertEquals(2, PforEncoderDecoder.bitWidthForInt(3)); + assertEquals(3, PforEncoderDecoder.bitWidthForInt(4)); + assertEquals(8, PforEncoderDecoder.bitWidthForInt(255)); + assertEquals(9, PforEncoderDecoder.bitWidthForInt(256)); + assertEquals(16, PforEncoderDecoder.bitWidthForInt(65535)); + assertEquals(31, PforEncoderDecoder.bitWidthForInt(Integer.MAX_VALUE)); + // Unsigned: -1 == 0xFFFFFFFF -> 32 bits + assertEquals(32, PforEncoderDecoder.bitWidthForInt(-1)); + } + + @Test + public void testBitWidthForLong() { + assertEquals(0, PforEncoderDecoder.bitWidthForLong(0L)); + assertEquals(1, PforEncoderDecoder.bitWidthForLong(1L)); + assertEquals(2, PforEncoderDecoder.bitWidthForLong(2L)); + assertEquals(2, PforEncoderDecoder.bitWidthForLong(3L)); + assertEquals(3, PforEncoderDecoder.bitWidthForLong(4L)); + assertEquals(8, PforEncoderDecoder.bitWidthForLong(255L)); + assertEquals(9, PforEncoderDecoder.bitWidthForLong(256L)); + assertEquals(16, PforEncoderDecoder.bitWidthForLong(65535L)); + assertEquals(31, PforEncoderDecoder.bitWidthForLong((long) Integer.MAX_VALUE)); + assertEquals(63, PforEncoderDecoder.bitWidthForLong(Long.MAX_VALUE)); + // Unsigned: -1 == 0xFFFFFFFFFFFFFFFF -> 64 bits + assertEquals(64, PforEncoderDecoder.bitWidthForLong(-1L)); + } + + @Test + public void testOptimalBitWidthAllIdentical() { + // All deltas are 0 -> bit_width=0, no exceptions + int[] deltas = new int[1024]; + PforEncoderDecoder.BitWidthResult result = PforEncoderDecoder.findOptimalBitWidthForInt(deltas, 1024); + assertEquals(0, result.bitWidth); + assertEquals(0, result.numExceptions); + } + + @Test + public void testOptimalBitWidthNoOutliers() { + // Deltas 0..255 -> all fit in 8 bits, no exceptions + int[] deltas = new int[256]; + for (int i = 0; i < 256; i++) { + deltas[i] = i; + } + PforEncoderDecoder.BitWidthResult result = PforEncoderDecoder.findOptimalBitWidthForInt(deltas, 256); + assertEquals(8, result.bitWidth); + assertEquals(0, result.numExceptions); + } + + @Test + public void testOptimalBitWidthSingleOutlier() { + // 1023 values fit in 8 bits (0..255), 1 outlier at 100000 + // Cost at bw=8: 1024*8 + 1*48 = 8240 + // Cost at bw=17: 1024*17 + 0 = 17408 + // Cost at bw=0: 1020 nonzero deltas * 48 = 48960 + int[] deltas = new int[1024]; + for (int i = 0; i < 1023; i++) { + deltas[i] = i % 256; + } + deltas[1023] = 100000; + PforEncoderDecoder.BitWidthResult result = PforEncoderDecoder.findOptimalBitWidthForInt(deltas, 1024); + assertEquals(8, result.bitWidth); + assertEquals(1, result.numExceptions); + } + + @Test + public void testOptimalBitWidthManyOutliers() { + // All values need 32 bits -> bit_width=32, no exceptions + int[] deltas = new int[100]; + for (int i = 0; i < 100; i++) { + deltas[i] = Integer.MAX_VALUE - i; + } + PforEncoderDecoder.BitWidthResult result = PforEncoderDecoder.findOptimalBitWidthForInt(deltas, 100); + assertEquals(31, result.bitWidth); + assertEquals(0, result.numExceptions); + } + + @Test + public void testOptimalBitWidthSingleElement() { + int[] deltas = {42}; + PforEncoderDecoder.BitWidthResult result = PforEncoderDecoder.findOptimalBitWidthForInt(deltas, 1); + assertEquals(0, result.numExceptions); + assertTrue(result.bitWidth >= 6); // 42 needs 6 bits + } + + @Test + public void testOptimalBitWidthAllZeros() { + int[] deltas = new int[512]; + PforEncoderDecoder.BitWidthResult result = PforEncoderDecoder.findOptimalBitWidthForInt(deltas, 512); + assertEquals(0, result.bitWidth); + assertEquals(0, result.numExceptions); + } + + @Test + public void testOptimalBitWidthLongAllIdentical() { + long[] deltas = new long[1024]; + PforEncoderDecoder.BitWidthResult result = PforEncoderDecoder.findOptimalBitWidthForLong(deltas, 1024); + assertEquals(0, result.bitWidth); + assertEquals(0, result.numExceptions); + } + + @Test + public void testOptimalBitWidthLongNoOutliers() { + long[] deltas = new long[256]; + for (int i = 0; i < 256; i++) { + deltas[i] = i; + } + PforEncoderDecoder.BitWidthResult result = PforEncoderDecoder.findOptimalBitWidthForLong(deltas, 256); + assertEquals(8, result.bitWidth); + assertEquals(0, result.numExceptions); + } + + @Test + public void testOptimalBitWidthLongSingleOutlier() { + long[] deltas = new long[1024]; + for (int i = 0; i < 1023; i++) { + deltas[i] = i % 256; + } + deltas[1023] = 10_000_000_000L; + PforEncoderDecoder.BitWidthResult result = PforEncoderDecoder.findOptimalBitWidthForLong(deltas, 1024); + assertEquals(8, result.bitWidth); + assertEquals(1, result.numExceptions); + } + + @Test + public void testOptimalBitWidthLongLargeValues() { + long[] deltas = new long[100]; + for (int i = 0; i < 100; i++) { + deltas[i] = Long.MAX_VALUE - i; + } + PforEncoderDecoder.BitWidthResult result = PforEncoderDecoder.findOptimalBitWidthForLong(deltas, 100); + assertEquals(63, result.bitWidth); + assertEquals(0, result.numExceptions); + } + + @Test + public void testCostModelPrefersFewerExceptions() { + // With 50% outliers, the cost model should widen bit width rather than + // storing half the values as exceptions + int[] deltas = new int[100]; + for (int i = 0; i < 50; i++) { + deltas[i] = i; // 0..49 fit in 6 bits + } + for (int i = 50; i < 100; i++) { + deltas[i] = 1000 + i; // need ~10 bits + } + PforEncoderDecoder.BitWidthResult result = PforEncoderDecoder.findOptimalBitWidthForInt(deltas, 100); + // Should choose to pack everything (10-11 bits) rather than 50 exceptions + assertEquals(0, result.numExceptions); + } + + @Test + public void testCostModelNeverExceedsMaxBitWidth() { + int[] deltas = {-1}; // 0xFFFFFFFF, needs 32 bits unsigned + PforEncoderDecoder.BitWidthResult result = PforEncoderDecoder.findOptimalBitWidthForInt(deltas, 1); + assertTrue(result.bitWidth <= 32); + } + + @Test + public void testCostModelLongNeverExceedsMaxBitWidth() { + long[] deltas = {-1L}; // 0xFFFFFFFFFFFFFFFF, needs 64 bits unsigned + PforEncoderDecoder.BitWidthResult result = PforEncoderDecoder.findOptimalBitWidthForLong(deltas, 1); + assertTrue(result.bitWidth <= 64); + } +} diff --git a/parquet-column/src/test/java/org/apache/parquet/column/values/pfor/PforValuesEndToEndTest.java b/parquet-column/src/test/java/org/apache/parquet/column/values/pfor/PforValuesEndToEndTest.java new file mode 100644 index 0000000000..4b1ef9447d --- /dev/null +++ b/parquet-column/src/test/java/org/apache/parquet/column/values/pfor/PforValuesEndToEndTest.java @@ -0,0 +1,433 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.values.pfor; + +import static org.junit.Assert.*; + +import java.util.Random; +import org.apache.parquet.bytes.ByteBufferInputStream; +import org.apache.parquet.bytes.BytesInput; +import org.apache.parquet.bytes.DirectByteBufferAllocator; +import org.apache.parquet.column.Encoding; +import org.junit.Test; + +/** + * End-to-end tests for PFOR encoding and decoding pipeline. + * Tests the full writer -> serialized bytes -> reader round-trip. + */ +public class PforValuesEndToEndTest { + + private static final int DEFAULT_VECTOR_SIZE = PforConstants.DEFAULT_VECTOR_SIZE; + + private void roundTripInt(int[] values) throws Exception { + roundTripInt(values, DEFAULT_VECTOR_SIZE); + } + + private void roundTripInt(int[] values, int vectorSize) throws Exception { + PforValuesWriter.IntPforValuesWriter writer = null; + try { + int capacity = Math.max(256, values.length * 8); + writer = new PforValuesWriter.IntPforValuesWriter( + capacity, capacity, new DirectByteBufferAllocator(), vectorSize); + + for (int v : values) { + writer.writeInteger(v); + } + + assertEquals(Encoding.PFOR, writer.getEncoding()); + + BytesInput input = writer.getBytes(); + PforValuesReaderForInt reader = new PforValuesReaderForInt(); + reader.initFromPage(values.length, ByteBufferInputStream.wrap(input.toByteBuffer())); + + for (int i = 0; i < values.length; i++) { + assertEquals("Value mismatch at index " + i, values[i], reader.readInteger()); + } + } finally { + if (writer != null) { + writer.reset(); + writer.close(); + } + } + } + + private void roundTripLong(long[] values) throws Exception { + roundTripLong(values, DEFAULT_VECTOR_SIZE); + } + + private void roundTripLong(long[] values, int vectorSize) throws Exception { + PforValuesWriter.LongPforValuesWriter writer = null; + try { + int capacity = Math.max(512, values.length * 16); + writer = new PforValuesWriter.LongPforValuesWriter( + capacity, capacity, new DirectByteBufferAllocator(), vectorSize); + + for (long v : values) { + writer.writeLong(v); + } + + assertEquals(Encoding.PFOR, writer.getEncoding()); + + BytesInput input = writer.getBytes(); + PforValuesReaderForLong reader = new PforValuesReaderForLong(); + reader.initFromPage(values.length, ByteBufferInputStream.wrap(input.toByteBuffer())); + + for (int i = 0; i < values.length; i++) { + assertEquals("Value mismatch at index " + i, values[i], reader.readLong()); + } + } finally { + if (writer != null) { + writer.reset(); + writer.close(); + } + } + } + + @Test + public void testIntSimpleSequence() throws Exception { + int[] values = new int[100]; + for (int i = 0; i < 100; i++) { + values[i] = i; + } + roundTripInt(values); + } + + @Test + public void testIntAllIdentical() throws Exception { + int[] values = new int[1024]; + java.util.Arrays.fill(values, 42); + roundTripInt(values); + } + + @Test + public void testIntAllZeros() throws Exception { + int[] values = new int[1024]; + roundTripInt(values); + } + + @Test + public void testIntSingleElement() throws Exception { + roundTripInt(new int[] {12345}); + } + + @Test + public void testIntNegativeValues() throws Exception { + int[] values = {-100, -50, -1, 0, 1, 50, 100}; + roundTripInt(values); + } + + @Test + public void testIntMinMaxValues() throws Exception { + int[] values = {Integer.MIN_VALUE, Integer.MAX_VALUE, 0, -1, 1}; + roundTripInt(values); + } + + @Test + public void testIntAlternatingMinMax() throws Exception { + int[] values = new int[100]; + for (int i = 0; i < 100; i++) { + values[i] = (i % 2 == 0) ? Integer.MIN_VALUE : Integer.MAX_VALUE; + } + roundTripInt(values); + } + + @Test + public void testIntExactOneVector() throws Exception { + int[] values = new int[1024]; + for (int i = 0; i < 1024; i++) { + values[i] = i * 3; + } + roundTripInt(values); + } + + @Test + public void testIntMultipleVectors() throws Exception { + int[] values = new int[3000]; + for (int i = 0; i < 3000; i++) { + values[i] = i * 7 - 10000; + } + roundTripInt(values); + } + + @Test + public void testIntPartialLastVector() throws Exception { + // 1025 values = 1 full vector + 1 partial + int[] values = new int[1025]; + for (int i = 0; i < 1025; i++) { + values[i] = i; + } + roundTripInt(values); + } + + @Test + public void testIntWithOutliers() throws Exception { + // Mostly small values with a few large outliers + int[] values = new int[1024]; + for (int i = 0; i < 1024; i++) { + values[i] = i % 100; + } + values[0] = 1_000_000; + values[512] = -1_000_000; + values[1023] = Integer.MAX_VALUE; + roundTripInt(values); + } + + @Test + public void testIntLargeRandom() throws Exception { + Random rng = new Random(42); + int[] values = new int[10000]; + for (int i = 0; i < 10000; i++) { + values[i] = rng.nextInt(); + } + roundTripInt(values); + } + + @Test + public void testIntSmallVectorSize() throws Exception { + int[] values = new int[100]; + for (int i = 0; i < 100; i++) { + values[i] = i * 11; + } + roundTripInt(values, 8); // smallest valid vector size + } + + @Test + public void testIntTpcdsLikeDateKeys() throws Exception { + // Simulate TPC-DS date dimension keys: mostly sequential with a few gaps + int[] values = new int[2048]; + for (int i = 0; i < 2048; i++) { + values[i] = 2450815 + i + (i % 100 == 0 ? 100 : 0); + } + roundTripInt(values); + } + + @Test + public void testLongSimpleSequence() throws Exception { + long[] values = new long[100]; + for (int i = 0; i < 100; i++) { + values[i] = i; + } + roundTripLong(values); + } + + @Test + public void testLongAllIdentical() throws Exception { + long[] values = new long[1024]; + java.util.Arrays.fill(values, 999999999999L); + roundTripLong(values); + } + + @Test + public void testLongAllZeros() throws Exception { + long[] values = new long[1024]; + roundTripLong(values); + } + + @Test + public void testLongSingleElement() throws Exception { + roundTripLong(new long[] {Long.MAX_VALUE}); + } + + @Test + public void testLongNegativeValues() throws Exception { + long[] values = {-100_000_000_000L, -1L, 0L, 1L, 100_000_000_000L}; + roundTripLong(values); + } + + @Test + public void testLongMinMaxValues() throws Exception { + long[] values = {Long.MIN_VALUE, Long.MAX_VALUE, 0L, -1L, 1L}; + roundTripLong(values); + } + + @Test + public void testLongMultipleVectors() throws Exception { + long[] values = new long[3000]; + for (int i = 0; i < 3000; i++) { + values[i] = (long) i * 1_000_000L - 1_500_000_000L; + } + roundTripLong(values); + } + + @Test + public void testLongPartialLastVector() throws Exception { + long[] values = new long[1025]; + for (int i = 0; i < 1025; i++) { + values[i] = i * 17L; + } + roundTripLong(values); + } + + @Test + public void testLongWithOutliers() throws Exception { + long[] values = new long[1024]; + for (int i = 0; i < 1024; i++) { + values[i] = i % 100; + } + values[0] = Long.MAX_VALUE; + values[512] = Long.MIN_VALUE; + values[1023] = 10_000_000_000_000L; + roundTripLong(values); + } + + @Test + public void testLongLargeRandom() throws Exception { + Random rng = new Random(42); + long[] values = new long[10000]; + for (int i = 0; i < 10000; i++) { + values[i] = rng.nextLong(); + } + roundTripLong(values); + } + + @Test + public void testLongSmallVectorSize() throws Exception { + long[] values = new long[100]; + for (int i = 0; i < 100; i++) { + values[i] = i * 1_000_000L; + } + roundTripLong(values, 8); + } + + @Test + public void testIntWriterReset() throws Exception { + PforValuesWriter.IntPforValuesWriter writer = + new PforValuesWriter.IntPforValuesWriter(1024, 1024, new DirectByteBufferAllocator()); + + // First batch + for (int i = 0; i < 100; i++) { + writer.writeInteger(i); + } + BytesInput bytes1 = writer.getBytes(); + assertTrue(bytes1.size() > 0); + + // Reset and second batch + writer.reset(); + for (int i = 0; i < 50; i++) { + writer.writeInteger(i * 2); + } + BytesInput bytes2 = writer.getBytes(); + assertTrue(bytes2.size() > 0); + + // Verify second batch reads correctly + PforValuesReaderForInt reader = new PforValuesReaderForInt(); + reader.initFromPage(50, ByteBufferInputStream.wrap(bytes2.toByteBuffer())); + for (int i = 0; i < 50; i++) { + assertEquals(i * 2, reader.readInteger()); + } + + writer.close(); + } + + @Test + public void testLongWriterReset() throws Exception { + PforValuesWriter.LongPforValuesWriter writer = + new PforValuesWriter.LongPforValuesWriter(1024, 1024, new DirectByteBufferAllocator()); + + for (int i = 0; i < 100; i++) { + writer.writeLong(i * 1000L); + } + writer.getBytes(); + + writer.reset(); + for (int i = 0; i < 50; i++) { + writer.writeLong(i * 2000L); + } + BytesInput bytes2 = writer.getBytes(); + + PforValuesReaderForLong reader = new PforValuesReaderForLong(); + reader.initFromPage(50, ByteBufferInputStream.wrap(bytes2.toByteBuffer())); + for (int i = 0; i < 50; i++) { + assertEquals(i * 2000L, reader.readLong()); + } + + writer.close(); + } + + @Test + public void testIntSkip() throws Exception { + int[] values = new int[2048]; + for (int i = 0; i < 2048; i++) { + values[i] = i; + } + + PforValuesWriter.IntPforValuesWriter writer = + new PforValuesWriter.IntPforValuesWriter(4096, 4096, new DirectByteBufferAllocator()); + for (int v : values) { + writer.writeInteger(v); + } + BytesInput bytes = writer.getBytes(); + + PforValuesReaderForInt reader = new PforValuesReaderForInt(); + reader.initFromPage(2048, ByteBufferInputStream.wrap(bytes.toByteBuffer())); + + // Skip first 1000 + reader.skip(1000); + assertEquals(1000, reader.readInteger()); + assertEquals(1001, reader.readInteger()); + + // Skip more + reader.skip(500); + assertEquals(1502, reader.readInteger()); + + writer.close(); + } + + @Test + public void testLongSkip() throws Exception { + long[] values = new long[2048]; + for (int i = 0; i < 2048; i++) { + values[i] = i * 100L; + } + + PforValuesWriter.LongPforValuesWriter writer = + new PforValuesWriter.LongPforValuesWriter(4096, 4096, new DirectByteBufferAllocator()); + for (long v : values) { + writer.writeLong(v); + } + BytesInput bytes = writer.getBytes(); + + PforValuesReaderForLong reader = new PforValuesReaderForLong(); + reader.initFromPage(2048, ByteBufferInputStream.wrap(bytes.toByteBuffer())); + + reader.skip(1000); + assertEquals(100000L, reader.readLong()); + + writer.close(); + } + + @Test + public void testIntEmptyInput() throws Exception { + PforValuesWriter.IntPforValuesWriter writer = + new PforValuesWriter.IntPforValuesWriter(256, 256, new DirectByteBufferAllocator()); + BytesInput bytes = writer.getBytes(); + // Empty page still emits a valid 7-byte header (numElements=0) + assertEquals(PforConstants.PFOR_HEADER_SIZE, bytes.size()); + writer.close(); + } + + @Test + public void testLongEmptyInput() throws Exception { + PforValuesWriter.LongPforValuesWriter writer = + new PforValuesWriter.LongPforValuesWriter(256, 256, new DirectByteBufferAllocator()); + BytesInput bytes = writer.getBytes(); + assertEquals(PforConstants.PFOR_HEADER_SIZE, bytes.size()); + writer.close(); + } +} From d6560935683e5e019b9202feb6f11887251f7409 Mon Sep 17 00:00:00 2001 From: Prateek Gaur Date: Tue, 21 Apr 2026 00:15:22 +0000 Subject: [PATCH 2/7] Let writers enable PFOR per column The writer properties gain a PFOR switch, global or per column, which the V2 values writer factory checks before BYTE_STREAM_SPLIT for INT32 and INT64 columns. It is off by default. The format has no PFOR value yet, so the metadata converter refuses to write one with a message that says so instead of failing in an enum lookup. --- .../parquet/column/ParquetProperties.java | 46 +++++++++++++++++++ .../factory/DefaultV2ValuesWriterFactory.java | 15 +++++- .../converter/ParquetMetadataConverter.java | 5 ++ 3 files changed, 64 insertions(+), 2 deletions(-) diff --git a/parquet-column/src/main/java/org/apache/parquet/column/ParquetProperties.java b/parquet-column/src/main/java/org/apache/parquet/column/ParquetProperties.java index f29214b458..4036d35e34 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/ParquetProperties.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/ParquetProperties.java @@ -50,6 +50,7 @@ public class ParquetProperties { public static final int DEFAULT_DICTIONARY_PAGE_SIZE = DEFAULT_PAGE_SIZE; public static final boolean DEFAULT_IS_DICTIONARY_ENABLED = true; public static final boolean DEFAULT_IS_BYTE_STREAM_SPLIT_ENABLED = false; + public static final boolean DEFAULT_IS_PFOR_ENABLED = false; public static final WriterVersion DEFAULT_WRITER_VERSION = WriterVersion.PARQUET_1_0; public static final boolean DEFAULT_ESTIMATE_ROW_COUNT_FOR_PAGE_SIZE_CHECK = true; public static final int DEFAULT_MINIMUM_RECORD_COUNT_FOR_CHECK = 100; @@ -132,6 +133,7 @@ public static WriterVersion fromString(String name) { private final int pageRowCountLimit; private final boolean pageWriteChecksumEnabled; private final ColumnProperty byteStreamSplitEnabled; + private final ColumnProperty pforEnabled; private final Map extraMetaData; private final ColumnProperty statistics; private final ColumnProperty sizeStatistics; @@ -164,6 +166,7 @@ private ParquetProperties(Builder builder) { this.pageRowCountLimit = builder.pageRowCountLimit; this.pageWriteChecksumEnabled = builder.pageWriteChecksumEnabled; this.byteStreamSplitEnabled = builder.byteStreamSplitEnabled.build(); + this.pforEnabled = builder.pforEnabled.build(); this.extraMetaData = builder.extraMetaData; this.statistics = builder.statistics.build(); this.sizeStatistics = builder.sizeStatistics.build(); @@ -259,6 +262,23 @@ public boolean isByteStreamSplitEnabled(ColumnDescriptor column) { } } + /** + * Check if PFOR encoding is enabled for the given column. + * PFOR encoding is only supported for INT32 and INT64 types. + * + * @param column the column descriptor + * @return true if PFOR encoding is enabled for this column + */ + public boolean isPforEnabled(ColumnDescriptor column) { + switch (column.getPrimitiveType().getPrimitiveTypeName()) { + case INT32: + case INT64: + return pforEnabled.getValue(column); + default: + return false; + } + } + public ByteBufferAllocator getAllocator() { return allocator; } @@ -416,6 +436,7 @@ public static class Builder { private int pageRowCountLimit = DEFAULT_PAGE_ROW_COUNT_LIMIT; private boolean pageWriteChecksumEnabled = DEFAULT_PAGE_WRITE_CHECKSUM_ENABLED; private final ColumnProperty.Builder byteStreamSplitEnabled; + private final ColumnProperty.Builder pforEnabled; private Map extraMetaData = new HashMap<>(); private final ColumnProperty.Builder statistics; private final ColumnProperty.Builder sizeStatistics; @@ -427,6 +448,7 @@ private Builder() { DEFAULT_IS_BYTE_STREAM_SPLIT_ENABLED ? ByteStreamSplitMode.FLOATING_POINT : ByteStreamSplitMode.NONE); + pforEnabled = ColumnProperty.builder().withDefaultValue(DEFAULT_IS_PFOR_ENABLED); bloomFilterEnabled = ColumnProperty.builder().withDefaultValue(DEFAULT_BLOOM_FILTER_ENABLED); bloomFilterNDVs = ColumnProperty.builder().withDefaultValue(null); bloomFilterFPPs = ColumnProperty.builder().withDefaultValue(DEFAULT_BLOOM_FILTER_FPP); @@ -457,6 +479,7 @@ private Builder(ParquetProperties toCopy) { this.numBloomFilterCandidates = ColumnProperty.builder(toCopy.numBloomFilterCandidates); this.maxBloomFilterBytes = toCopy.maxBloomFilterBytes; this.byteStreamSplitEnabled = ColumnProperty.builder(toCopy.byteStreamSplitEnabled); + this.pforEnabled = ColumnProperty.builder(toCopy.pforEnabled); this.extraMetaData = toCopy.extraMetaData; this.statistics = ColumnProperty.builder(toCopy.statistics); this.sizeStatistics = ColumnProperty.builder(toCopy.sizeStatistics); @@ -534,6 +557,29 @@ public Builder withExtendedByteStreamSplitEncoding(boolean enable) { return this; } + /** + * Enable or disable PFOR encoding for INT32 and INT64 columns. + * + * @param enable whether PFOR encoding should be enabled + * @return this builder for method chaining. + */ + public Builder withPforEncoding(boolean enable) { + this.pforEnabled.withDefaultValue(enable); + return this; + } + + /** + * Enable or disable PFOR encoding for the specified column. + * + * @param columnPath the path of the column (dot-string) + * @param enable whether PFOR encoding should be enabled + * @return this builder for method chaining. + */ + public Builder withPforEncoding(String columnPath, boolean enable) { + this.pforEnabled.withValue(columnPath, enable); + return this; + } + /** * Set the Parquet format dictionary page size. * diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultV2ValuesWriterFactory.java b/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultV2ValuesWriterFactory.java index c50b4e49c5..b215e0fe14 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultV2ValuesWriterFactory.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultV2ValuesWriterFactory.java @@ -29,6 +29,7 @@ import org.apache.parquet.column.values.delta.DeltaBinaryPackingValuesWriterForInteger; import org.apache.parquet.column.values.delta.DeltaBinaryPackingValuesWriterForLong; import org.apache.parquet.column.values.deltastrings.DeltaByteArrayWriter; +import org.apache.parquet.column.values.pfor.PforValuesWriter; import org.apache.parquet.column.values.plain.FixedLenByteArrayPlainValuesWriter; import org.apache.parquet.column.values.plain.PlainValuesWriter; import org.apache.parquet.column.values.rle.RunLengthBitPackingHybridValuesWriter; @@ -115,7 +116,12 @@ private ValuesWriter getBinaryValuesWriter(ColumnDescriptor path) { private ValuesWriter getInt32ValuesWriter(ColumnDescriptor path) { final ValuesWriter fallbackWriter; - if (parquetProperties.isByteStreamSplitEnabled(path)) { + if (this.parquetProperties.isPforEnabled(path)) { + fallbackWriter = new PforValuesWriter.IntPforValuesWriter( + parquetProperties.getInitialSlabSize(), + parquetProperties.getPageSizeThreshold(), + parquetProperties.getAllocator()); + } else if (parquetProperties.isByteStreamSplitEnabled(path)) { fallbackWriter = new ByteStreamSplitValuesWriter.IntegerByteStreamSplitValuesWriter( parquetProperties.getInitialSlabSize(), parquetProperties.getPageSizeThreshold(), @@ -132,7 +138,12 @@ private ValuesWriter getInt32ValuesWriter(ColumnDescriptor path) { private ValuesWriter getInt64ValuesWriter(ColumnDescriptor path) { final ValuesWriter fallbackWriter; - if (parquetProperties.isByteStreamSplitEnabled(path)) { + if (this.parquetProperties.isPforEnabled(path)) { + fallbackWriter = new PforValuesWriter.LongPforValuesWriter( + parquetProperties.getInitialSlabSize(), + parquetProperties.getPageSizeThreshold(), + parquetProperties.getAllocator()); + } else if (parquetProperties.isByteStreamSplitEnabled(path)) { fallbackWriter = new ByteStreamSplitValuesWriter.LongByteStreamSplitValuesWriter( parquetProperties.getInitialSlabSize(), parquetProperties.getPageSizeThreshold(), diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java b/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java index 60150439a6..2d4c5ed9af 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java @@ -748,6 +748,11 @@ public org.apache.parquet.column.Encoding getEncoding(Encoding encoding) { } public Encoding getEncoding(org.apache.parquet.column.Encoding encoding) { + // PFOR encoding is not yet part of the parquet-format specification + if (encoding == org.apache.parquet.column.Encoding.PFOR) { + throw new IllegalArgumentException( + "PFOR encoding is not yet supported in the parquet-format specification"); + } return Encoding.valueOf(encoding.name()); } From d80d5c3238c3221ddf9b8e62f0a514f659577ae1 Mon Sep 17 00:00:00 2001 From: Prateek Gaur Date: Sat, 5 Sep 2026 01:18:40 +0000 Subject: [PATCH 3/7] Add a Hadoop configuration key for PFOR parquet.enable.pfor turns PFOR on for jobs configured through a Hadoop Configuration, exposed the same way as parquet.enable.bytestreamsplit. It defaults to off. --- .../parquet/hadoop/ParquetOutputFormat.java | 9 ++ .../parquet/hadoop/TestPforConfiguration.java | 85 +++++++++++++++++++ 2 files changed, 94 insertions(+) create mode 100644 parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestPforConfiguration.java diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java index 868ae634c1..41f690f517 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java @@ -83,6 +83,9 @@ * # To enable/disable BYTE_STREAM_SPLIT encoding * parquet.enable.bytestreamsplit=false # true to enable BYTE_STREAM_SPLIT encoding * + * # To enable/disable PFOR encoding for INT32 and INT64 columns + * parquet.enable.pfor=false # true to enable PFOR encoding + * * # To enable/disable summary metadata aggregation at the end of a MR job * # The default is true (enabled) * parquet.enable.summary-metadata=true # false to disable summary aggregation @@ -141,6 +144,7 @@ public static enum JobSummaryLevel { public static final String DICTIONARY_PAGE_SIZE = "parquet.dictionary.page.size"; public static final String ENABLE_DICTIONARY = "parquet.enable.dictionary"; public static final String ENABLE_BYTE_STREAM_SPLIT = "parquet.enable.bytestreamsplit"; + public static final String ENABLE_PFOR = "parquet.enable.pfor"; public static final String VALIDATION = "parquet.validation"; public static final String WRITER_VERSION = "parquet.writer.version"; public static final String MEMORY_POOL_RATIO = "parquet.memory.pool.ratio"; @@ -279,6 +283,10 @@ public static boolean getByteStreamSplitEnabled(Configuration configuration) { ENABLE_BYTE_STREAM_SPLIT, ParquetProperties.DEFAULT_IS_BYTE_STREAM_SPLIT_ENABLED); } + public static boolean getPforEnabled(Configuration configuration) { + return configuration.getBoolean(ENABLE_PFOR, ParquetProperties.DEFAULT_IS_PFOR_ENABLED); + } + public static int getMinRowCountForPageSizeCheck(Configuration configuration) { return configuration.getInt( MIN_ROW_COUNT_FOR_PAGE_SIZE_CHECK, ParquetProperties.DEFAULT_MINIMUM_RECORD_COUNT_FOR_CHECK); @@ -513,6 +521,7 @@ public RecordWriter getRecordWriter(Configuration conf, Path file, Comp .withDictionaryPageSize(getDictionaryPageSize(conf)) .withDictionaryEncoding(getEnableDictionary(conf)) .withByteStreamSplitEncoding(getByteStreamSplitEnabled(conf)) + .withPforEncoding(getPforEnabled(conf)) .withWriterVersion(getWriterVersion(conf)) .estimateRowCountForPageSizeCheck(getEstimatePageSizeCheck(conf)) .withMinRowCountForPageSizeCheck(getMinRowCountForPageSizeCheck(conf)) diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestPforConfiguration.java b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestPforConfiguration.java new file mode 100644 index 0000000000..3e161e5f54 --- /dev/null +++ b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestPforConfiguration.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.hadoop; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import org.apache.hadoop.conf.Configuration; +import org.apache.parquet.column.ColumnDescriptor; +import org.apache.parquet.column.ParquetProperties; +import org.apache.parquet.schema.MessageType; +import org.apache.parquet.schema.MessageTypeParser; +import org.junit.Test; + +public class TestPforConfiguration { + + private static final MessageType SCHEMA = + MessageTypeParser.parseMessageType("message m { required int32 i; required double d; }"); + + private static ColumnDescriptor intColumn() { + return SCHEMA.getColumns().get(0); + } + + private static ColumnDescriptor doubleColumn() { + return SCHEMA.getColumns().get(1); + } + + @Test + public void testDefault() throws Exception { + Configuration conf = new Configuration(); + // PFOR is off unless a job asks for it + assertEquals(ParquetProperties.DEFAULT_IS_PFOR_ENABLED, ParquetOutputFormat.getPforEnabled(conf)); + } + + @Test + public void testTheKeyNameIsTheDocumentedOne() throws Exception { + // This string is the public surface; the class javadoc documents it + assertEquals("parquet.enable.pfor", ParquetOutputFormat.ENABLE_PFOR); + } + + @Test + public void testSetTrue() throws Exception { + Configuration conf = new Configuration(); + conf.setBoolean(ParquetOutputFormat.ENABLE_PFOR, true); + assertTrue(ParquetOutputFormat.getPforEnabled(conf)); + } + + @Test + public void testSetFalse() throws Exception { + Configuration conf = new Configuration(); + conf.setBoolean(ParquetOutputFormat.ENABLE_PFOR, false); + assertFalse(ParquetOutputFormat.getPforEnabled(conf)); + } + + @Test + public void testTheKeyReachesTheWriterProperties() throws Exception { + Configuration conf = new Configuration(); + conf.setBoolean(ParquetOutputFormat.ENABLE_PFOR, true); + + ParquetProperties props = ParquetProperties.builder() + .withPforEncoding(ParquetOutputFormat.getPforEnabled(conf)) + .build(); + + assertTrue(props.isPforEnabled(intColumn())); + // PFOR encodes INT32 and INT64 only, whatever the configuration says + assertFalse(props.isPforEnabled(doubleColumn())); + } +} From 6df1b0920eb28ad5c1c3754b462caf9e6192c358 Mon Sep 17 00:00:00 2001 From: Prateek Gaur Date: Sat, 5 Sep 2026 01:19:00 +0000 Subject: [PATCH 4/7] Add a PFOR encoding benchmark Encodes and decodes INT32 and INT64 columns drawn from constant, sequential, clustered, outlier-heavy, random and TPC-DS-like distributions, and prints each one's compression ratio. Surefire's benchmark exclusion keeps it out of normal test runs. --- .../pfor/benchmark/BenchmarkPforEncoding.java | 521 ++++++++++++++++++ 1 file changed, 521 insertions(+) create mode 100644 parquet-column/src/test/java/org/apache/parquet/column/values/pfor/benchmark/BenchmarkPforEncoding.java diff --git a/parquet-column/src/test/java/org/apache/parquet/column/values/pfor/benchmark/BenchmarkPforEncoding.java b/parquet-column/src/test/java/org/apache/parquet/column/values/pfor/benchmark/BenchmarkPforEncoding.java new file mode 100644 index 0000000000..cca6c6c3d8 --- /dev/null +++ b/parquet-column/src/test/java/org/apache/parquet/column/values/pfor/benchmark/BenchmarkPforEncoding.java @@ -0,0 +1,521 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.values.pfor.benchmark; + +import com.carrotsearch.junitbenchmarks.BenchmarkOptions; +import com.carrotsearch.junitbenchmarks.BenchmarkRule; +import com.carrotsearch.junitbenchmarks.annotation.AxisRange; +import com.carrotsearch.junitbenchmarks.annotation.BenchmarkMethodChart; +import java.io.IOException; +import java.util.Random; +import org.apache.parquet.bytes.ByteBufferInputStream; +import org.apache.parquet.bytes.DirectByteBufferAllocator; +import org.apache.parquet.column.values.pfor.PforValuesReaderForInt; +import org.apache.parquet.column.values.pfor.PforValuesReaderForLong; +import org.apache.parquet.column.values.pfor.PforValuesWriter; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; + +/** + * PFOR encoding/decoding benchmarks for INT32 and INT64. + * + *

Each data distribution exercises the cost model differently: + *

+ * + *

Excluded from normal test runs via surefire's benchmark exclusion pattern. + */ +@AxisRange(min = 0, max = 1) +@BenchmarkMethodChart(filePrefix = "benchmark-pfor-encoding") +public class BenchmarkPforEncoding { + + private static final int NUM_VALUES = 500_000; + + @Rule + public org.junit.rules.TestRule benchmarkRun = new BenchmarkRule(); + + private static int[] intConstant; + private static int[] intSequential; + private static int[] intSmallRange; + private static int[] intHighBaseSmallRange; + private static int[] intWithOutliers; + private static int[] intRandom; + private static int[] intTpcdsSoldDateSk; + private static int[] intTpcdsQuantity; + + private static long[] longConstant; + private static long[] longSequential; + private static long[] longSmallRange; + private static long[] longHighBaseSmallRange; + private static long[] longWithOutliers; + private static long[] longRandom; + private static long[] longTpcdsSoldDateSk; + + // Pre-encoded bytes for decode benchmarks + private static byte[] intConstantBytes; + private static byte[] intSequentialBytes; + private static byte[] intSmallRangeBytes; + private static byte[] intHighBaseSmallRangeBytes; + private static byte[] intWithOutliersBytes; + private static byte[] intRandomBytes; + private static byte[] intTpcdsSoldDateSkBytes; + private static byte[] intTpcdsQuantityBytes; + + private static byte[] longConstantBytes; + private static byte[] longSequentialBytes; + private static byte[] longSmallRangeBytes; + private static byte[] longHighBaseSmallRangeBytes; + private static byte[] longWithOutliersBytes; + private static byte[] longRandomBytes; + private static byte[] longTpcdsSoldDateSkBytes; + + @BeforeClass + public static void prepare() throws IOException { + // Generate INT32 distributions + intConstant = genIntConstant(NUM_VALUES); + intSequential = genIntSequential(NUM_VALUES); + intSmallRange = genIntSmallRange(NUM_VALUES); + intHighBaseSmallRange = genIntHighBaseSmallRange(NUM_VALUES); + intWithOutliers = genIntWithOutliers(NUM_VALUES); + intRandom = genIntRandom(NUM_VALUES); + intTpcdsSoldDateSk = genIntTpcdsSoldDateSk(NUM_VALUES); + intTpcdsQuantity = genIntTpcdsQuantity(NUM_VALUES); + + // Generate INT64 distributions + longConstant = genLongConstant(NUM_VALUES); + longSequential = genLongSequential(NUM_VALUES); + longSmallRange = genLongSmallRange(NUM_VALUES); + longHighBaseSmallRange = genLongHighBaseSmallRange(NUM_VALUES); + longWithOutliers = genLongWithOutliers(NUM_VALUES); + longRandom = genLongRandom(NUM_VALUES); + longTpcdsSoldDateSk = genLongTpcdsSoldDateSk(NUM_VALUES); + + // Pre-encode for decode benchmarks + intConstantBytes = encodeInts(intConstant); + intSequentialBytes = encodeInts(intSequential); + intSmallRangeBytes = encodeInts(intSmallRange); + intHighBaseSmallRangeBytes = encodeInts(intHighBaseSmallRange); + intWithOutliersBytes = encodeInts(intWithOutliers); + intRandomBytes = encodeInts(intRandom); + intTpcdsSoldDateSkBytes = encodeInts(intTpcdsSoldDateSk); + intTpcdsQuantityBytes = encodeInts(intTpcdsQuantity); + + longConstantBytes = encodeLongs(longConstant); + longSequentialBytes = encodeLongs(longSequential); + longSmallRangeBytes = encodeLongs(longSmallRange); + longHighBaseSmallRangeBytes = encodeLongs(longHighBaseSmallRange); + longWithOutliersBytes = encodeLongs(longWithOutliers); + longRandomBytes = encodeLongs(longRandom); + longTpcdsSoldDateSkBytes = encodeLongs(longTpcdsSoldDateSk); + + // Print compression ratios + System.out.println("=== PFOR Compression Ratios (compressed / uncompressed) ==="); + printIntRatio("Constant", intConstantBytes, intConstant.length); + printIntRatio("Sequential", intSequentialBytes, intSequential.length); + printIntRatio("SmallRange", intSmallRangeBytes, intSmallRange.length); + printIntRatio("HighBaseSmallRange", intHighBaseSmallRangeBytes, intHighBaseSmallRange.length); + printIntRatio("WithOutliers", intWithOutliersBytes, intWithOutliers.length); + printIntRatio("Random", intRandomBytes, intRandom.length); + printIntRatio("TpcdsSoldDateSk", intTpcdsSoldDateSkBytes, intTpcdsSoldDateSk.length); + printIntRatio("TpcdsQuantity", intTpcdsQuantityBytes, intTpcdsQuantity.length); + System.out.println("---"); + printLongRatio("Constant", longConstantBytes, longConstant.length); + printLongRatio("Sequential", longSequentialBytes, longSequential.length); + printLongRatio("SmallRange", longSmallRangeBytes, longSmallRange.length); + printLongRatio("HighBaseSmallRange", longHighBaseSmallRangeBytes, longHighBaseSmallRange.length); + printLongRatio("WithOutliers", longWithOutliersBytes, longWithOutliers.length); + printLongRatio("Random", longRandomBytes, longRandom.length); + printLongRatio("TpcdsSoldDateSk", longTpcdsSoldDateSkBytes, longTpcdsSoldDateSk.length); + } + + @BenchmarkOptions(benchmarkRounds = 20, warmupRounds = 4) + @Test + public void encodeIntConstant() throws IOException { + benchmarkIntEncode(intConstant); + } + + @BenchmarkOptions(benchmarkRounds = 20, warmupRounds = 4) + @Test + public void encodeIntSequential() throws IOException { + benchmarkIntEncode(intSequential); + } + + @BenchmarkOptions(benchmarkRounds = 20, warmupRounds = 4) + @Test + public void encodeIntSmallRange() throws IOException { + benchmarkIntEncode(intSmallRange); + } + + @BenchmarkOptions(benchmarkRounds = 20, warmupRounds = 4) + @Test + public void encodeIntHighBaseSmallRange() throws IOException { + benchmarkIntEncode(intHighBaseSmallRange); + } + + @BenchmarkOptions(benchmarkRounds = 20, warmupRounds = 4) + @Test + public void encodeIntWithOutliers() throws IOException { + benchmarkIntEncode(intWithOutliers); + } + + @BenchmarkOptions(benchmarkRounds = 20, warmupRounds = 4) + @Test + public void encodeIntRandom() throws IOException { + benchmarkIntEncode(intRandom); + } + + @BenchmarkOptions(benchmarkRounds = 20, warmupRounds = 4) + @Test + public void encodeIntTpcdsSoldDateSk() throws IOException { + benchmarkIntEncode(intTpcdsSoldDateSk); + } + + @BenchmarkOptions(benchmarkRounds = 20, warmupRounds = 4) + @Test + public void encodeIntTpcdsQuantity() throws IOException { + benchmarkIntEncode(intTpcdsQuantity); + } + + @BenchmarkOptions(benchmarkRounds = 20, warmupRounds = 4) + @Test + public void decodeIntConstant() throws IOException { + benchmarkIntDecode(intConstantBytes, intConstant.length); + } + + @BenchmarkOptions(benchmarkRounds = 20, warmupRounds = 4) + @Test + public void decodeIntSequential() throws IOException { + benchmarkIntDecode(intSequentialBytes, intSequential.length); + } + + @BenchmarkOptions(benchmarkRounds = 20, warmupRounds = 4) + @Test + public void decodeIntSmallRange() throws IOException { + benchmarkIntDecode(intSmallRangeBytes, intSmallRange.length); + } + + @BenchmarkOptions(benchmarkRounds = 20, warmupRounds = 4) + @Test + public void decodeIntHighBaseSmallRange() throws IOException { + benchmarkIntDecode(intHighBaseSmallRangeBytes, intHighBaseSmallRange.length); + } + + @BenchmarkOptions(benchmarkRounds = 20, warmupRounds = 4) + @Test + public void decodeIntWithOutliers() throws IOException { + benchmarkIntDecode(intWithOutliersBytes, intWithOutliers.length); + } + + @BenchmarkOptions(benchmarkRounds = 20, warmupRounds = 4) + @Test + public void decodeIntRandom() throws IOException { + benchmarkIntDecode(intRandomBytes, intRandom.length); + } + + @BenchmarkOptions(benchmarkRounds = 20, warmupRounds = 4) + @Test + public void decodeIntTpcdsSoldDateSk() throws IOException { + benchmarkIntDecode(intTpcdsSoldDateSkBytes, intTpcdsSoldDateSk.length); + } + + @BenchmarkOptions(benchmarkRounds = 20, warmupRounds = 4) + @Test + public void decodeIntTpcdsQuantity() throws IOException { + benchmarkIntDecode(intTpcdsQuantityBytes, intTpcdsQuantity.length); + } + + @BenchmarkOptions(benchmarkRounds = 20, warmupRounds = 4) + @Test + public void encodeLongConstant() throws IOException { + benchmarkLongEncode(longConstant); + } + + @BenchmarkOptions(benchmarkRounds = 20, warmupRounds = 4) + @Test + public void encodeLongSequential() throws IOException { + benchmarkLongEncode(longSequential); + } + + @BenchmarkOptions(benchmarkRounds = 20, warmupRounds = 4) + @Test + public void encodeLongSmallRange() throws IOException { + benchmarkLongEncode(longSmallRange); + } + + @BenchmarkOptions(benchmarkRounds = 20, warmupRounds = 4) + @Test + public void encodeLongHighBaseSmallRange() throws IOException { + benchmarkLongEncode(longHighBaseSmallRange); + } + + @BenchmarkOptions(benchmarkRounds = 20, warmupRounds = 4) + @Test + public void encodeLongWithOutliers() throws IOException { + benchmarkLongEncode(longWithOutliers); + } + + @BenchmarkOptions(benchmarkRounds = 20, warmupRounds = 4) + @Test + public void encodeLongRandom() throws IOException { + benchmarkLongEncode(longRandom); + } + + @BenchmarkOptions(benchmarkRounds = 20, warmupRounds = 4) + @Test + public void encodeLongTpcdsSoldDateSk() throws IOException { + benchmarkLongEncode(longTpcdsSoldDateSk); + } + + @BenchmarkOptions(benchmarkRounds = 20, warmupRounds = 4) + @Test + public void decodeLongConstant() throws IOException { + benchmarkLongDecode(longConstantBytes, longConstant.length); + } + + @BenchmarkOptions(benchmarkRounds = 20, warmupRounds = 4) + @Test + public void decodeLongSequential() throws IOException { + benchmarkLongDecode(longSequentialBytes, longSequential.length); + } + + @BenchmarkOptions(benchmarkRounds = 20, warmupRounds = 4) + @Test + public void decodeLongSmallRange() throws IOException { + benchmarkLongDecode(longSmallRangeBytes, longSmallRange.length); + } + + @BenchmarkOptions(benchmarkRounds = 20, warmupRounds = 4) + @Test + public void decodeLongHighBaseSmallRange() throws IOException { + benchmarkLongDecode(longHighBaseSmallRangeBytes, longHighBaseSmallRange.length); + } + + @BenchmarkOptions(benchmarkRounds = 20, warmupRounds = 4) + @Test + public void decodeLongWithOutliers() throws IOException { + benchmarkLongDecode(longWithOutliersBytes, longWithOutliers.length); + } + + @BenchmarkOptions(benchmarkRounds = 20, warmupRounds = 4) + @Test + public void decodeLongRandom() throws IOException { + benchmarkLongDecode(longRandomBytes, longRandom.length); + } + + @BenchmarkOptions(benchmarkRounds = 20, warmupRounds = 4) + @Test + public void decodeLongTpcdsSoldDateSk() throws IOException { + benchmarkLongDecode(longTpcdsSoldDateSkBytes, longTpcdsSoldDateSk.length); + } + + private void benchmarkIntEncode(int[] values) throws IOException { + int capacity = Math.max(256, values.length * 8); + PforValuesWriter.IntPforValuesWriter writer = + new PforValuesWriter.IntPforValuesWriter(capacity, capacity, new DirectByteBufferAllocator()); + for (int v : values) { + writer.writeInteger(v); + } + writer.getBytes(); + writer.close(); + } + + private void benchmarkIntDecode(byte[] encoded, int numValues) throws IOException { + PforValuesReaderForInt reader = new PforValuesReaderForInt(); + reader.initFromPage(numValues, ByteBufferInputStream.wrap(java.nio.ByteBuffer.wrap(encoded))); + for (int i = 0; i < numValues; i++) { + reader.readInteger(); + } + } + + private void benchmarkLongEncode(long[] values) throws IOException { + int capacity = Math.max(512, values.length * 16); + PforValuesWriter.LongPforValuesWriter writer = + new PforValuesWriter.LongPforValuesWriter(capacity, capacity, new DirectByteBufferAllocator()); + for (long v : values) { + writer.writeLong(v); + } + writer.getBytes(); + writer.close(); + } + + private void benchmarkLongDecode(byte[] encoded, int numValues) throws IOException { + PforValuesReaderForLong reader = new PforValuesReaderForLong(); + reader.initFromPage(numValues, ByteBufferInputStream.wrap(java.nio.ByteBuffer.wrap(encoded))); + for (int i = 0; i < numValues; i++) { + reader.readLong(); + } + } + + private static int[] genIntConstant(int n) { + int[] v = new int[n]; + java.util.Arrays.fill(v, 42); + return v; + } + + private static int[] genIntSequential(int n) { + int[] v = new int[n]; + for (int i = 0; i < n; i++) v[i] = i; + return v; + } + + private static int[] genIntSmallRange(int n) { + int[] v = new int[n]; + Random rng = new Random(12345); + for (int i = 0; i < n; i++) v[i] = 100000 + rng.nextInt(100001); + return v; + } + + private static int[] genIntHighBaseSmallRange(int n) { + int[] v = new int[n]; + Random rng = new Random(12345); + for (int i = 0; i < n; i++) v[i] = 1704067200 + rng.nextInt(1001); + return v; + } + + private static int[] genIntWithOutliers(int n) { + int[] v = new int[n]; + Random rng = new Random(42); + for (int i = 0; i < n; i++) v[i] = 1000 + rng.nextInt(256); + // ~1% outliers + int numOutliers = Math.max(1, n / 100); + for (int i = 0; i < numOutliers; i++) { + v[rng.nextInt(n)] = Integer.MAX_VALUE / 2 + i; + } + return v; + } + + private static int[] genIntRandom(int n) { + int[] v = new int[n]; + Random rng = new Random(99); + for (int i = 0; i < n; i++) v[i] = rng.nextInt(); + return v; + } + + private static int[] genIntTpcdsSoldDateSk(int n) { + int[] v = new int[n]; + Random rng = new Random(12345); + for (int i = 0; i < n; i++) v[i] = 2450815 + rng.nextInt(1821); + return v; + } + + private static int[] genIntTpcdsQuantity(int n) { + int[] v = new int[n]; + Random rng = new Random(12345); + for (int i = 0; i < n; i++) { + v[i] = (rng.nextInt(100) < 90) ? (1 + rng.nextInt(10)) : (11 + rng.nextInt(90)); + } + return v; + } + + private static long[] genLongConstant(int n) { + long[] v = new long[n]; + java.util.Arrays.fill(v, 42L); + return v; + } + + private static long[] genLongSequential(int n) { + long[] v = new long[n]; + for (int i = 0; i < n; i++) v[i] = i; + return v; + } + + private static long[] genLongSmallRange(int n) { + long[] v = new long[n]; + Random rng = new Random(12345); + for (int i = 0; i < n; i++) v[i] = 100000L + rng.nextInt(100001); + return v; + } + + private static long[] genLongHighBaseSmallRange(int n) { + long[] v = new long[n]; + Random rng = new Random(12345); + for (int i = 0; i < n; i++) v[i] = 1704067200L + rng.nextInt(1001); + return v; + } + + private static long[] genLongWithOutliers(int n) { + long[] v = new long[n]; + Random rng = new Random(42); + for (int i = 0; i < n; i++) v[i] = 1000L + rng.nextInt(256); + int numOutliers = Math.max(1, n / 100); + for (int i = 0; i < numOutliers; i++) { + v[rng.nextInt(n)] = Long.MAX_VALUE / 2 + i; + } + return v; + } + + private static long[] genLongRandom(int n) { + long[] v = new long[n]; + Random rng = new Random(99); + for (int i = 0; i < n; i++) v[i] = rng.nextLong(); + return v; + } + + private static long[] genLongTpcdsSoldDateSk(int n) { + long[] v = new long[n]; + Random rng = new Random(12345); + for (int i = 0; i < n; i++) v[i] = 2450815L + rng.nextInt(1821); + return v; + } + + private static byte[] encodeInts(int[] values) throws IOException { + int capacity = Math.max(256, values.length * 8); + PforValuesWriter.IntPforValuesWriter writer = + new PforValuesWriter.IntPforValuesWriter(capacity, capacity, new DirectByteBufferAllocator()); + for (int v : values) { + writer.writeInteger(v); + } + byte[] result = writer.getBytes().toByteArray(); + writer.close(); + return result; + } + + private static byte[] encodeLongs(long[] values) throws IOException { + int capacity = Math.max(512, values.length * 16); + PforValuesWriter.LongPforValuesWriter writer = + new PforValuesWriter.LongPforValuesWriter(capacity, capacity, new DirectByteBufferAllocator()); + for (long v : values) { + writer.writeLong(v); + } + byte[] result = writer.getBytes().toByteArray(); + writer.close(); + return result; + } + + private static void printIntRatio(String name, byte[] encoded, int numValues) { + double ratio = 100.0 * encoded.length / (numValues * 4); + System.out.printf( + " INT32 %-25s: %6d bytes -> %6d bytes (%.1f%%)\n", name, numValues * 4, encoded.length, ratio); + } + + private static void printLongRatio(String name, byte[] encoded, int numValues) { + double ratio = 100.0 * encoded.length / (numValues * 8); + System.out.printf( + " INT64 %-25s: %6d bytes -> %6d bytes (%.1f%%)\n", name, numValues * 8, encoded.length, ratio); + } +} From 62165bc524ff145e830fe1b235627fe859152e07 Mon Sep 17 00:00:00 2001 From: Prateek Gaur Date: Fri, 4 Sep 2026 17:03:36 +0000 Subject: [PATCH 5/7] Add an optional delta mode to PFOR A sorted or slowly changing column packs much narrower as differences between successive values than as offsets from a frame. The writer now costs each vector both ways and keeps the cheaper, using a sampled estimate to skip the full delta pass where it cannot win. Bit 7 of the bit width byte, reserved until now, marks a delta vector, and such a vector stores its first value so it still decodes without the one before it. The reader patches exceptions before the prefix sum. The mode is on by default and can be turned off globally or per column. --- .../parquet/column/ParquetProperties.java | 43 ++ .../factory/DefaultV2ValuesWriterFactory.java | 6 +- .../column/values/pfor/PforConstants.java | 26 +- .../values/pfor/PforEncoderDecoder.java | 313 +++++++-- .../column/values/pfor/PforValuesReader.java | 20 +- .../values/pfor/PforValuesReaderForInt.java | 42 +- .../values/pfor/PforValuesReaderForLong.java | 42 +- .../column/values/pfor/PforValuesWriter.java | 184 +++-- .../values/pfor/PforAdversarialTest.java | 136 +++- .../column/values/pfor/PforDeltaModeTest.java | 652 ++++++++++++++++++ 10 files changed, 1326 insertions(+), 138 deletions(-) create mode 100644 parquet-column/src/test/java/org/apache/parquet/column/values/pfor/PforDeltaModeTest.java diff --git a/parquet-column/src/main/java/org/apache/parquet/column/ParquetProperties.java b/parquet-column/src/main/java/org/apache/parquet/column/ParquetProperties.java index 4036d35e34..6ca2dea037 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/ParquetProperties.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/ParquetProperties.java @@ -51,6 +51,7 @@ public class ParquetProperties { public static final boolean DEFAULT_IS_DICTIONARY_ENABLED = true; public static final boolean DEFAULT_IS_BYTE_STREAM_SPLIT_ENABLED = false; public static final boolean DEFAULT_IS_PFOR_ENABLED = false; + public static final boolean DEFAULT_IS_PFOR_DELTA_ENABLED = true; public static final WriterVersion DEFAULT_WRITER_VERSION = WriterVersion.PARQUET_1_0; public static final boolean DEFAULT_ESTIMATE_ROW_COUNT_FOR_PAGE_SIZE_CHECK = true; public static final int DEFAULT_MINIMUM_RECORD_COUNT_FOR_CHECK = 100; @@ -134,6 +135,7 @@ public static WriterVersion fromString(String name) { private final boolean pageWriteChecksumEnabled; private final ColumnProperty byteStreamSplitEnabled; private final ColumnProperty pforEnabled; + private final ColumnProperty pforDeltaEnabled; private final Map extraMetaData; private final ColumnProperty statistics; private final ColumnProperty sizeStatistics; @@ -167,6 +169,7 @@ private ParquetProperties(Builder builder) { this.pageWriteChecksumEnabled = builder.pageWriteChecksumEnabled; this.byteStreamSplitEnabled = builder.byteStreamSplitEnabled.build(); this.pforEnabled = builder.pforEnabled.build(); + this.pforDeltaEnabled = builder.pforDeltaEnabled.build(); this.extraMetaData = builder.extraMetaData; this.statistics = builder.statistics.build(); this.sizeStatistics = builder.sizeStatistics.build(); @@ -279,6 +282,17 @@ public boolean isPforEnabled(ColumnDescriptor column) { } } + /** + * Check whether a PFOR writer may encode a vector as the differences between its + * successive values. Only consulted where PFOR itself is enabled. + * + * @param column the column descriptor + * @return true if the PFOR delta mode is enabled for this column + */ + public boolean isPforDeltaEnabled(ColumnDescriptor column) { + return pforDeltaEnabled.getValue(column); + } + public ByteBufferAllocator getAllocator() { return allocator; } @@ -437,6 +451,7 @@ public static class Builder { private boolean pageWriteChecksumEnabled = DEFAULT_PAGE_WRITE_CHECKSUM_ENABLED; private final ColumnProperty.Builder byteStreamSplitEnabled; private final ColumnProperty.Builder pforEnabled; + private final ColumnProperty.Builder pforDeltaEnabled; private Map extraMetaData = new HashMap<>(); private final ColumnProperty.Builder statistics; private final ColumnProperty.Builder sizeStatistics; @@ -449,6 +464,7 @@ private Builder() { ? ByteStreamSplitMode.FLOATING_POINT : ByteStreamSplitMode.NONE); pforEnabled = ColumnProperty.builder().withDefaultValue(DEFAULT_IS_PFOR_ENABLED); + pforDeltaEnabled = ColumnProperty.builder().withDefaultValue(DEFAULT_IS_PFOR_DELTA_ENABLED); bloomFilterEnabled = ColumnProperty.builder().withDefaultValue(DEFAULT_BLOOM_FILTER_ENABLED); bloomFilterNDVs = ColumnProperty.builder().withDefaultValue(null); bloomFilterFPPs = ColumnProperty.builder().withDefaultValue(DEFAULT_BLOOM_FILTER_FPP); @@ -480,6 +496,7 @@ private Builder(ParquetProperties toCopy) { this.maxBloomFilterBytes = toCopy.maxBloomFilterBytes; this.byteStreamSplitEnabled = ColumnProperty.builder(toCopy.byteStreamSplitEnabled); this.pforEnabled = ColumnProperty.builder(toCopy.pforEnabled); + this.pforDeltaEnabled = ColumnProperty.builder(toCopy.pforDeltaEnabled); this.extraMetaData = toCopy.extraMetaData; this.statistics = ColumnProperty.builder(toCopy.statistics); this.sizeStatistics = ColumnProperty.builder(toCopy.sizeStatistics); @@ -580,6 +597,32 @@ public Builder withPforEncoding(String columnPath, boolean enable) { return this; } + /** + * Enable or disable the PFOR delta mode, in which a vector may be encoded as the + * differences between its successive values rather than as the values. Enabled by + * default: the writer costs both and keeps the cheaper, per vector, so leaving it on + * cannot make a page larger than leaving it off, only slower to write. + * + * @param enable whether the PFOR delta mode should be enabled + * @return this builder for method chaining. + */ + public Builder withPforDeltaEncoding(boolean enable) { + this.pforDeltaEnabled.withDefaultValue(enable); + return this; + } + + /** + * Enable or disable the PFOR delta mode for the specified column. + * + * @param columnPath the path of the column (dot-string) + * @param enable whether the PFOR delta mode should be enabled + * @return this builder for method chaining. + */ + public Builder withPforDeltaEncoding(String columnPath, boolean enable) { + this.pforDeltaEnabled.withValue(columnPath, enable); + return this; + } + /** * Set the Parquet format dictionary page size. * diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultV2ValuesWriterFactory.java b/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultV2ValuesWriterFactory.java index b215e0fe14..e3f9c7827f 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultV2ValuesWriterFactory.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultV2ValuesWriterFactory.java @@ -120,7 +120,8 @@ private ValuesWriter getInt32ValuesWriter(ColumnDescriptor path) { fallbackWriter = new PforValuesWriter.IntPforValuesWriter( parquetProperties.getInitialSlabSize(), parquetProperties.getPageSizeThreshold(), - parquetProperties.getAllocator()); + parquetProperties.getAllocator(), + parquetProperties.isPforDeltaEnabled(path)); } else if (parquetProperties.isByteStreamSplitEnabled(path)) { fallbackWriter = new ByteStreamSplitValuesWriter.IntegerByteStreamSplitValuesWriter( parquetProperties.getInitialSlabSize(), @@ -142,7 +143,8 @@ private ValuesWriter getInt64ValuesWriter(ColumnDescriptor path) { fallbackWriter = new PforValuesWriter.LongPforValuesWriter( parquetProperties.getInitialSlabSize(), parquetProperties.getPageSizeThreshold(), - parquetProperties.getAllocator()); + parquetProperties.getAllocator(), + parquetProperties.isPforDeltaEnabled(path)); } else if (parquetProperties.isByteStreamSplitEnabled(path)) { fallbackWriter = new ByteStreamSplitValuesWriter.LongByteStreamSplitValuesWriter( parquetProperties.getInitialSlabSize(), diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforConstants.java b/parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforConstants.java index a0f0f24222..992536b0b7 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforConstants.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforConstants.java @@ -27,9 +27,15 @@ *

    *
  1. Subtracting the minimum value (Frame of Reference)
    2. *
  2. Choosing an optimal bit width via a cost model
    3. - *
  3. Bit-packing the deltas at the chosen width
    4. + *
  4. Bit-packing the residuals at the chosen width
    5. *
  5. Storing outlier values (exceptions) separately with their positions
    6. *
+ * + *

A writer may first replace the values of a vector with the differences + * between successive values -- the delta mode -- and run all of the above on + * those instead. The choice is recorded per vector in bit 7 of the bit width + * byte, and such a vector carries its own first value so that it still decodes + * without the vector before it. */ public final class PforConstants { @@ -52,9 +58,23 @@ private PforConstants() { // Maximum exceptions per vector (uint16) public static final int MAX_EXCEPTIONS = 65535; - // The bit width occupies bits 0..6 of its byte; bit 7 is reserved and must be - // masked off before the width is used or range-checked. + // The bit width occupies bits 0..6 of its byte and must be masked off before it + // is used or range-checked; bit 7 says the vector holds differences. + // + // The width takes seven bits rather than six because its range is 0..64 + // inclusive, and 64 does not fit in six: masking with six bits would read an + // INT64 vector whose residuals need the full 64 bits as width 0, which has no + // packed bytes and no exceptions, so the misreading looks like a constant + // vector and neither a size mismatch nor an error reveals it. public static final int BIT_WIDTH_MASK = 0x7F; + public static final int DELTA_FLAG = 0x80; + + // A delta vector stores its first value between its info block and its packed + // residuals, so its header is that much longer. + public static int vectorInfoSize(int valueByteWidth, boolean delta) { + int base = valueByteWidth == INT32_VALUE_BYTE_WIDTH ? INT32_VECTOR_INFO_SIZE : INT64_VECTOR_INFO_SIZE; + return delta ? base + valueByteWidth : base; + } // Per-vector metadata sizes in bytes // INT32: frame_of_reference(4) + bit_width(1) + num_exceptions(2) = 7 diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforEncoderDecoder.java b/parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforEncoderDecoder.java index 414ce8f89b..0d9d86acd9 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforEncoderDecoder.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforEncoderDecoder.java @@ -26,7 +26,11 @@ * total_cost(b) = num_elements * b + num_exceptions(b) * (16 + value_bits) * * where {@code value_bits} is 32 for INT32 and 64 for INT64, and - * {@code num_exceptions(b)} is the count of deltas requiring more than {@code b} bits. + * {@code num_exceptions(b)} is the count of residuals requiring more than {@code b} bits. + * + *

The same model decides the delta mode: a vector is costed as it stands and + * again as the differences between its successive values, and the cheaper of the + * two wins. See {@link #chooseVectorPlanForInt}. */ public final class PforEncoderDecoder { @@ -38,13 +42,54 @@ private PforEncoderDecoder() { public static final class BitWidthResult { public final int bitWidth; public final int numExceptions; + /** What the winning width costs, in bits, under the cost model above. */ + public final long costBits; + + BitWidthResult(int bitWidth, int numExceptions, long costBits) { + this.bitWidth = bitWidth; + this.numExceptions = numExceptions; + this.costBits = costBits; + } + } + + /** + * How one vector is to be encoded: whether to difference it first, and the frame, + * width and exception count that follow from that choice. + * + *

The frame and start value are held as longs for both physical types; an INT32 + * writer narrows them back to int, which is lossless because they came from ints. + */ + public static final class VectorPlan { + public final boolean delta; + public final long frameOfReference; + /** The vector's first value, meaningful only when {@link #delta} is set. */ + public final long startValue; + + public final int bitWidth; + public final int numExceptions; + public final long costBits; - BitWidthResult(int bitWidth, int numExceptions) { + VectorPlan( + boolean delta, long frameOfReference, long startValue, int bitWidth, int numExceptions, long costBits) { + this.delta = delta; + this.frameOfReference = frameOfReference; + this.startValue = startValue; this.bitWidth = bitWidth; this.numExceptions = numExceptions; + this.costBits = costBits; } } + // Exception cost: position(16 bits) + one full-width value. + private static final long INT32_EXCEPTION_BITS = 16 + 32; + private static final long INT64_EXCEPTION_BITS = 16 + 64; + + /** + * Enough of a sample to place a distribution across the width bins, and few enough + * that the pass is a fraction of the one it is deciding against. + */ + private static final int DELTA_SAMPLE_TARGET = 128; + /** * Find the optimal bit width for packing INT32 unsigned deltas. * @@ -62,35 +107,7 @@ public static BitWidthResult findOptimalBitWidthForInt(int[] deltas, int numElem bitsHist[bitWidthForInt(deltas[i])]++; } - // Exception cost per exception: position(16 bits) + value(32 bits) = 48 bits - final long exceptionBitsPerValue = 16 + 32; - - long bestCost = Long.MAX_VALUE; - int bestBitWidth = 0; - int bestExceptions = 0; - - // At candidate width b, the deltas needing more than b bits are exceptions. - // A zero delta needs no bits, so at b=0 that is every nonzero delta. - int exceptionsAbove = numElements - bitsHist[0]; - - for (int b = 0; b <= 32; b++) { - long packingCost = (long) numElements * b; - long exceptionCost = (long) exceptionsAbove * exceptionBitsPerValue; - long totalCost = packingCost + exceptionCost; - - if (totalCost < bestCost) { - bestCost = totalCost; - bestBitWidth = b; - bestExceptions = exceptionsAbove; - } - - // Move to next candidate: values requiring exactly (b+1) bits are no longer exceptions - if (b < 32) { - exceptionsAbove -= bitsHist[b + 1]; - } - } - - return new BitWidthResult(bestBitWidth, bestExceptions); + return bestFromHistogram(bitsHist, 32, numElements, INT32_EXCEPTION_BITS); } /** @@ -107,32 +124,246 @@ public static BitWidthResult findOptimalBitWidthForLong(long[] deltas, int numEl bitsHist[bitWidthForLong(deltas[i])]++; } - // Exception cost per exception: position(16 bits) + value(64 bits) = 80 bits - final long exceptionBitsPerValue = 16 + 64; + return bestFromHistogram(bitsHist, 64, numElements, INT64_EXCEPTION_BITS); + } + /** + * Walk the candidate widths over a histogram of required widths and return the + * cheapest. + * + *

{@code bitsHist[b]} counts the residuals needing exactly {@code b} bits, so + * the residuals needing more than {@code b} -- the exceptions at that candidate -- + * are what remains above it, and one subtraction per step keeps that count. Ties + * keep the narrower width, since the first candidate to reach a cost wins. + * + * @param maxBits the physical type's width, the widest candidate + * @param exceptionBitsPerValue what one exception costs: its position and its value + */ + private static BitWidthResult bestFromHistogram( + int[] bitsHist, int maxBits, int numElements, long exceptionBitsPerValue) { long bestCost = Long.MAX_VALUE; int bestBitWidth = 0; int bestExceptions = 0; + // At candidate width b, residuals needing more than b bits are exceptions. + // bitsRequired(0) is 0, so at b = 0 that is everything except the zeros. int exceptionsAbove = numElements - bitsHist[0]; - for (int b = 0; b <= 64; b++) { - long packingCost = (long) numElements * b; - long exceptionCost = (long) exceptionsAbove * exceptionBitsPerValue; - long totalCost = packingCost + exceptionCost; - + for (int b = 0; b <= maxBits; b++) { + long totalCost = (long) numElements * b + (long) exceptionsAbove * exceptionBitsPerValue; if (totalCost < bestCost) { bestCost = totalCost; bestBitWidth = b; bestExceptions = exceptionsAbove; } - - if (b < 64) { + if (b < maxBits) { exceptionsAbove -= bitsHist[b + 1]; } } - return new BitWidthResult(bestBitWidth, bestExceptions); + return new BitWidthResult(bestBitWidth, bestExceptions, bestCost); + } + + /** + * Decide how to encode one INT32 vector: as it stands, or as its differences. + * + *

Both transforms are costed with the same model and the cheaper one wins, so the + * mode is a per-vector decision rather than a per-column one. It has to be: a column + * is rarely all one shape, and differencing costs bits on any stretch whose successive + * values are not close. + * + * @param values the vector + * @param numElements element count, greater than 0 + * @param deltaScratch scratch for numElements differences. On return it holds the + * differences if the plan chose the delta mode, and is clobbered either way. + * @param deltaEnabled whether the delta mode may be chosen at all + */ + public static VectorPlan chooseVectorPlanForInt( + int[] values, int numElements, int[] deltaScratch, boolean deltaEnabled) { + VectorPlan raw = searchForInt(values, numElements); + + // One element has no difference to take, and a vector already packing at width 0 + // cannot be improved on. + if (!deltaEnabled || numElements < 2 || raw.bitWidth == 0) { + return raw; + } + + // A delta vector carries its own first value, so it starts one full-width value + // behind whatever its differences pack to. + final long startValueBits = 32; + + // Decline the mode early when a sampled estimate cannot beat the plain plan. That + // skips the pass that writes the differences out and the search over them. The + // estimate is loose, so it declines only where the two modes are more than a + // sampling error apart, which is where the choice matters least. + if (estimateDeltaCostBitsForInt(values, numElements) + startValueBits >= raw.costBits) { + return raw; + } + + computeDeltasForInt(values, numElements, deltaScratch); + VectorPlan delta = searchForInt(deltaScratch, numElements); + long deltaCost = delta.costBits + startValueBits; + if (deltaCost >= raw.costBits) { + return raw; + } + return new VectorPlan(true, delta.frameOfReference, values[0], delta.bitWidth, delta.numExceptions, deltaCost); + } + + /** Decide how to encode one INT64 vector. See {@link #chooseVectorPlanForInt}. */ + public static VectorPlan chooseVectorPlanForLong( + long[] values, int numElements, long[] deltaScratch, boolean deltaEnabled) { + VectorPlan raw = searchForLong(values, numElements); + + if (!deltaEnabled || numElements < 2 || raw.bitWidth == 0) { + return raw; + } + + final long startValueBits = 64; + if (estimateDeltaCostBitsForLong(values, numElements) + startValueBits >= raw.costBits) { + return raw; + } + + computeDeltasForLong(values, numElements, deltaScratch); + VectorPlan delta = searchForLong(deltaScratch, numElements); + long deltaCost = delta.costBits + startValueBits; + if (deltaCost >= raw.costBits) { + return raw; + } + return new VectorPlan(true, delta.frameOfReference, values[0], delta.bitWidth, delta.numExceptions, deltaCost); + } + + /** + * Take the frame of an INT32 vector and cost the widths over it, without writing the + * residuals out: they are needed once here, for their widths, and again by the caller + * only once the plan is settled. + */ + private static VectorPlan searchForInt(int[] source, int numElements) { + int frame = source[0]; + for (int i = 1; i < numElements; i++) { + if (source[i] < frame) { + frame = source[i]; + } + } + + int[] bitsHist = new int[33]; + for (int i = 0; i < numElements; i++) { + bitsHist[bitWidthForInt(source[i] - frame)]++; + } + + BitWidthResult best = bestFromHistogram(bitsHist, 32, numElements, INT32_EXCEPTION_BITS); + return new VectorPlan(false, frame, 0, best.bitWidth, best.numExceptions, best.costBits); + } + + /** See {@link #searchForInt}. */ + private static VectorPlan searchForLong(long[] source, int numElements) { + long frame = source[0]; + for (int i = 1; i < numElements; i++) { + if (source[i] < frame) { + frame = source[i]; + } + } + + int[] bitsHist = new int[65]; + for (int i = 0; i < numElements; i++) { + bitsHist[bitWidthForLong(source[i] - frame)]++; + } + + BitWidthResult best = bestFromHistogram(bitsHist, 64, numElements, INT64_EXCEPTION_BITS); + return new VectorPlan(false, frame, 0, best.bitWidth, best.numExceptions, best.costBits); + } + + /** + * Fill {@code deltas} with the backward differences of {@code values}. + * + *

{@code deltas[0]} is 0: the first value travels in the plan's start value, and + * giving slot 0 a real difference would mean either a shorter packed run or a value + * that is not a difference sitting in the width histogram. Zero costs the bit width + * and distorts nothing. + * + *

The subtraction wraps, which is what makes the round trip exact for a column + * that spans the type's range; the reader's prefix sum wraps the same way. A vector + * with negative differences needs nothing special: the frame is the minimum of the + * differences, so subtracting it makes every residual non-negative, the same + * mechanism a plain vector uses for negative values. + */ + public static void computeDeltasForInt(int[] values, int numElements, int[] deltas) { + deltas[0] = 0; + for (int i = 1; i < numElements; i++) { + deltas[i] = values[i] - values[i - 1]; + } + } + + /** See {@link #computeDeltasForInt}. */ + public static void computeDeltasForLong(long[] values, int numElements, long[] deltas) { + deltas[0] = 0; + for (int i = 1; i < numElements; i++) { + deltas[i] = values[i] - values[i - 1]; + } + } + + /** + * Estimate what packing the differences of an INT32 vector would cost, in bits. + * + *

The full decision needs the differences written out and searched, which is most + * of what encoding a vector costs. This reaches an answer good enough to decline the + * mode from a strided sample, without writing anything. + * + *

The sample is of widths, not of a span. A gate on the span of the differences + * would decline a sawtooth, which is a tight cluster of small positive differences + * with a handful of large negative ones: its span is as wide as its raw span while + * its cost is a fraction of it. Feeding widths to the same cost model the search + * uses keeps that shape, because the model can trade a wide bin against a patch. + * + *

Zigzagging is what lets a histogram stand in for a frame search that has not + * run. Differences in [-k, k] zigzag into [0, 2k], and a frame at -k maps them onto + * the same [0, 2k], so for a range that straddles zero evenly the estimated width is + * the width the search would find. Where the range leans one way the estimate runs a + * bit or two wide. + */ + static long estimateDeltaCostBitsForInt(int[] values, int numElements) { + int stride = Math.max(1, numElements / DELTA_SAMPLE_TARGET); + int[] bitsHist = new int[33]; + int sampled = 0; + for (int i = stride; i < numElements; i += stride) { + bitsHist[bitWidthForInt(zigZagInt(values[i] - values[i - 1]))]++; + sampled++; + } + if (sampled == 0) { + return 0; + } + + long sampleCost = bestFromHistogram(bitsHist, 32, sampled, INT32_EXCEPTION_BITS).costBits; + // Scale to the whole vector. Both terms of the model are per element -- a width + // costs its bits every element, an exception costs its slot every time it occurs -- + // so the sample cost scales with the count. + return sampleCost * numElements / sampled; + } + + /** See {@link #estimateDeltaCostBitsForInt}. */ + static long estimateDeltaCostBitsForLong(long[] values, int numElements) { + int stride = Math.max(1, numElements / DELTA_SAMPLE_TARGET); + int[] bitsHist = new int[65]; + int sampled = 0; + for (int i = stride; i < numElements; i += stride) { + bitsHist[bitWidthForLong(zigZagLong(values[i] - values[i - 1]))]++; + sampled++; + } + if (sampled == 0) { + return 0; + } + + long sampleCost = bestFromHistogram(bitsHist, 64, sampled, INT64_EXCEPTION_BITS).costBits; + return sampleCost * numElements / sampled; + } + + /** Maps a signed value onto an unsigned one of about the same magnitude. */ + static int zigZagInt(int value) { + return (value << 1) ^ (value >> 31); + } + + /** See {@link #zigZagInt}. */ + static long zigZagLong(long value) { + return (value << 1) ^ (value >> 63); } /** diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforValuesReader.java b/parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforValuesReader.java index 4fabcae56b..c3fa9f0419 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforValuesReader.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforValuesReader.java @@ -139,7 +139,8 @@ protected int getVectorDataPosition(int vectorIdx) { * that remain. All three come off the wire and size every read and write that * follows, including the writes into the fixed-size decode buffers. * - * @param pos position of the packed values, that is, just past the vector info + * @param pos position of the packed values, that is, just past the vector info and, + * in a delta vector, just past the start value that follows it */ protected void checkVectorInfo(int pos, int bitWidth, int numExceptions, int vectorLen) { int maxBitWidth = valueByteWidth * Byte.SIZE; @@ -158,6 +159,23 @@ protected void checkVectorInfo(int pos, int bitWidth, int numExceptions, int vec } } + /** + * Checks that a delta vector's start value is inside the page. + * + *

These bytes need a bound of their own because the header bound was satisfied + * before the delta flag was known. Without it the start value is read from past the + * end of the page, and the residual bound is then checked from an offset that has + * already moved past the end. + * + * @param pos position of the start value, that is, just past the vector info + */ + protected void checkStartValueRoom(int pos) { + if (pos + valueByteWidth > vectorsData.limit()) { + throw new ParquetDecodingException("PFOR delta vector needs " + valueByteWidth + + " bytes for its start value but only " + (vectorsData.limit() - pos) + " remain"); + } + } + /** Exception positions index the decode buffer, so one past the end is a bad write. */ protected static void checkExceptionPosition(int position, int vectorLen) { if (position >= vectorLen) { diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforValuesReaderForInt.java b/parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforValuesReaderForInt.java index d80ef79cc5..ece75d2726 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforValuesReaderForInt.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforValuesReaderForInt.java @@ -34,6 +34,7 @@ *

Per-vector format: * 

  * PforVectorInfo (7B): frame_of_reference(4) + bit_width(1) + num_exceptions(2)
+ * StartValue: 4 bytes, only when bit 7 of bit_width is set
  * PackedValues: ceil(N * bit_width / 8) bytes
  * ExceptionPositions: num_exceptions * 2 bytes
  * ExceptionValues: num_exceptions * 4 bytes
@@ -44,7 +45,7 @@ public class PforValuesReaderForInt extends PforValuesReader {
   private int[] decodedValues;
 
   // Reusable per-vector decode buffers
-  private int[] deltasBuffer;
+  private int[] residualsBuffer;
   private int[] excPositionsBuffer;
   private byte[] unpackPadBuf;
   private int[] unpackTempBuf;
@@ -56,7 +57,7 @@ public PforValuesReaderForInt() {
   @Override
   protected void allocateDecodedBuffer(int capacity) {
     this.decodedValues = new int[capacity];
-    this.deltasBuffer = new int[capacity];
+    this.residualsBuffer = new int[capacity];
     this.excPositionsBuffer = new int[capacity];
     this.unpackPadBuf = new byte[Integer.SIZE]; // max bit width = 32 bytes
     this.unpackTempBuf = new int[8];
@@ -80,26 +81,37 @@ protected void decodeVector(int vectorIdx) {
 
     // Read PforVectorInfo (7 bytes)
     int frameOfReference = getIntLE(vectorsData, pos);
-    int bitWidth = vectorsData.get(pos + 4) & BIT_WIDTH_MASK;
+    int bitWidthByte = vectorsData.get(pos + 4) & 0xFF;
+    int bitWidth = bitWidthByte & BIT_WIDTH_MASK;
+    boolean delta = (bitWidthByte & DELTA_FLAG) != 0;
     int numExceptions = getShortLE(vectorsData, pos + 5) & 0xFFFF;
     pos += INT32_VECTOR_INFO_SIZE;
+
+    // A delta vector stores its own first value, so it decodes without the vector
+    // before it.
+    int startValue = 0;
+    if (delta) {
+      checkStartValueRoom(pos);
+      startValue = getIntLE(vectorsData, pos);
+      pos += INT32_VALUE_BYTE_WIDTH;
+    }
     checkVectorInfo(pos, bitWidth, numExceptions, vectorLen);
 
-    // Unpack bit-packed deltas into reusable buffer
+    // Unpack bit-packed residuals into reusable buffer
     if (bitWidth > 0) {
-      pos = unpackIntsWithBytePacker(vectorsData, pos, deltasBuffer, vectorLen, bitWidth);
+      pos = unpackIntsWithBytePacker(vectorsData, pos, residualsBuffer, vectorLen, bitWidth);
     } else {
       for (int i = 0; i < vectorLen; i++) {
-        deltasBuffer[i] = 0;
+        residualsBuffer[i] = 0;
       }
     }
 
-    // Add frame of reference to reconstruct values
+    // Add frame of reference
     for (int i = 0; i < vectorLen; i++) {
-      decodedValues[i] = deltasBuffer[i] + frameOfReference;
+      decodedValues[i] = residualsBuffer[i] + frameOfReference;
     }
 
-    // Overwrite exception slots with their original values
+    // Overwrite exception slots with their unreduced values
     if (numExceptions > 0) {
       for (int e = 0; e < numExceptions; e++) {
         int position = getShortLE(vectorsData, pos) & 0xFFFF;
@@ -112,6 +124,18 @@ protected void decodeVector(int vectorIdx) {
         pos += Integer.BYTES;
       }
     }
+
+    // Everything above produced differences in a delta vector, so sum them. This has to
+    // come after the patch: an exception there is a difference like any other, and
+    // summing first would carry its zero placeholder into every value that follows it.
+    // The addition wraps, matching how the writer took the differences.
+    if (delta) {
+      int acc = startValue;
+      for (int i = 0; i < vectorLen; i++) {
+        acc += decodedValues[i];
+        decodedValues[i] = acc;
+      }
+    }
   }
 
   private int unpackIntsWithBytePacker(ByteBuffer buf, int pos, int[] output, int count, int bitWidth) {
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforValuesReaderForLong.java b/parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforValuesReaderForLong.java
index 1ce51611a1..5b3efb72c7 100644
--- a/parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforValuesReaderForLong.java
+++ b/parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforValuesReaderForLong.java
@@ -34,6 +34,7 @@
  * 
Per-vector format:
  * 
  * PforVectorInfo (11B): frame_of_reference(8) + bit_width(1) + num_exceptions(2)
+ * StartValue: 8 bytes, only when bit 7 of bit_width is set
  * PackedValues: ceil(N * bit_width / 8) bytes
  * ExceptionPositions: num_exceptions * 2 bytes
  * ExceptionValues: num_exceptions * 8 bytes
@@ -44,7 +45,7 @@ public class PforValuesReaderForLong extends PforValuesReader {
   private long[] decodedValues;
 
   // Reusable per-vector decode buffers
-  private long[] deltasBuffer;
+  private long[] residualsBuffer;
   private int[] excPositionsBuffer;
   private byte[] unpackPadBuf;
   private long[] unpackTempBuf;
@@ -56,7 +57,7 @@ public PforValuesReaderForLong() {
   @Override
   protected void allocateDecodedBuffer(int capacity) {
     this.decodedValues = new long[capacity];
-    this.deltasBuffer = new long[capacity];
+    this.residualsBuffer = new long[capacity];
     this.excPositionsBuffer = new int[capacity];
     this.unpackPadBuf = new byte[Long.SIZE]; // max bit width = 64 bytes
     this.unpackTempBuf = new long[8];
@@ -80,26 +81,37 @@ protected void decodeVector(int vectorIdx) {
 
     // Read PforVectorInfo (11 bytes)
     long frameOfReference = getLongLE(vectorsData, pos);
-    int bitWidth = vectorsData.get(pos + 8) & BIT_WIDTH_MASK;
+    int bitWidthByte = vectorsData.get(pos + 8) & 0xFF;
+    int bitWidth = bitWidthByte & BIT_WIDTH_MASK;
+    boolean delta = (bitWidthByte & DELTA_FLAG) != 0;
     int numExceptions = getShortLE(vectorsData, pos + 9) & 0xFFFF;
     pos += INT64_VECTOR_INFO_SIZE;
+
+    // A delta vector stores its own first value, so it decodes without the vector
+    // before it.
+    long startValue = 0;
+    if (delta) {
+      checkStartValueRoom(pos);
+      startValue = getLongLE(vectorsData, pos);
+      pos += INT64_VALUE_BYTE_WIDTH;
+    }
     checkVectorInfo(pos, bitWidth, numExceptions, vectorLen);
 
-    // Unpack bit-packed deltas into reusable buffer
+    // Unpack bit-packed residuals into reusable buffer
     if (bitWidth > 0) {
-      pos = unpackLongsWithBytePacker(vectorsData, pos, deltasBuffer, vectorLen, bitWidth);
+      pos = unpackLongsWithBytePacker(vectorsData, pos, residualsBuffer, vectorLen, bitWidth);
     } else {
       for (int i = 0; i < vectorLen; i++) {
-        deltasBuffer[i] = 0;
+        residualsBuffer[i] = 0;
       }
     }
 
-    // Add frame of reference to reconstruct values
+    // Add frame of reference
     for (int i = 0; i < vectorLen; i++) {
-      decodedValues[i] = deltasBuffer[i] + frameOfReference;
+      decodedValues[i] = residualsBuffer[i] + frameOfReference;
     }
 
-    // Overwrite exception slots with their original values
+    // Overwrite exception slots with their unreduced values
     if (numExceptions > 0) {
       for (int e = 0; e < numExceptions; e++) {
         int position = getShortLE(vectorsData, pos) & 0xFFFF;
@@ -112,6 +124,18 @@ protected void decodeVector(int vectorIdx) {
         pos += Long.BYTES;
       }
     }
+
+    // Everything above produced differences in a delta vector, so sum them. This has to
+    // come after the patch: an exception there is a difference like any other, and
+    // summing first would carry its zero placeholder into every value that follows it.
+    // The addition wraps, matching how the writer took the differences.
+    if (delta) {
+      long acc = startValue;
+      for (int i = 0; i < vectorLen; i++) {
+        acc += decodedValues[i];
+        decodedValues[i] = acc;
+      }
+    }
   }
 
   private int unpackLongsWithBytePacker(ByteBuffer buf, int pos, long[] output, int count, int bitWidth) {
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforValuesWriter.java b/parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforValuesWriter.java
index cb6524afce..426a844677 100644
--- a/parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforValuesWriter.java
+++ b/parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforValuesWriter.java
@@ -38,7 +38,12 @@
  *
  * 
PFOR compresses integer columns by subtracting the minimum value (FOR),
  * selecting an optimal bit width via a histogram-based cost model, bit-packing
- * the deltas, and storing outlier values (exceptions) separately.
+ * the residuals, and storing outlier values (exceptions) separately.
+ *
+ * 
Per vector, the writer costs the values as they stand and again as the
+ * differences between successive values, and keeps the cheaper of the two -- the
+ * delta mode. A vector in that mode sets bit 7 of its bit width byte and stores
+ * its own first value, so it still decodes without the vector before it.
  *
  * 
Writing is incremental: values are buffered in a fixed-size vector buffer,
  * and each full vector is encoded and flushed to the output stream immediately.
@@ -55,8 +60,8 @@
  *
  * 
A vector carries its own info ahead of its data, so it can be decoded
  * without reading any other vector: PforVectorInfo (7B for INT32, 11B for
- * INT64), then the packed values, then the exception positions, then the
- * exception values.
+ * INT64), then in a delta vector its start value, then the packed values,
+ * then the exception positions, then the exception values.
  */
 public abstract class PforValuesWriter extends ValuesWriter {
 
@@ -65,14 +70,18 @@ public abstract class PforValuesWriter extends ValuesWriter {
   protected final ByteBufferAllocator allocator;
   protected final int vectorSize;
   protected final int logVectorSize;
+  /** Whether a vector may be encoded as differences; see {@link PforEncoderDecoder#chooseVectorPlanForInt}. */
+  protected final boolean deltaEnabled;
 
-  PforValuesWriter(int initialCapacity, int pageSize, ByteBufferAllocator allocator, int vectorSize) {
+  PforValuesWriter(
+      int initialCapacity, int pageSize, ByteBufferAllocator allocator, int vectorSize, boolean deltaEnabled) {
     PforConstants.validateVectorSize(vectorSize);
     this.initialCapacity = initialCapacity;
     this.pageSize = pageSize;
     this.allocator = allocator;
     this.vectorSize = vectorSize;
     this.logVectorSize = Integer.numberOfTrailingZeros(vectorSize);
+    this.deltaEnabled = deltaEnabled;
   }
 
   @Override
@@ -89,7 +98,8 @@ public static class IntPforValuesWriter extends PforValuesWriter {
     private final List vectorByteSizes;
 
     // Reusable per-vector buffers to avoid allocations on every encodeAndFlushVector call
-    private final int[] deltasBuffer;
+    private final int[] residualsBuffer;
+    private final int[] deltaScratch;
     private final short[] excPosBuffer;
     private final int[] excValBuffer;
     private final byte[] metadataBuf;
@@ -97,17 +107,32 @@ public static class IntPforValuesWriter extends PforValuesWriter {
     private final int[] packPadBuf;
 
     public IntPforValuesWriter(int initialCapacity, int pageSize, ByteBufferAllocator allocator) {
-      this(initialCapacity, pageSize, allocator, DEFAULT_VECTOR_SIZE);
+      this(initialCapacity, pageSize, allocator, DEFAULT_VECTOR_SIZE, true);
+    }
+
+    public IntPforValuesWriter(
+        int initialCapacity, int pageSize, ByteBufferAllocator allocator, boolean deltaEnabled) {
+      this(initialCapacity, pageSize, allocator, DEFAULT_VECTOR_SIZE, deltaEnabled);
     }
 
     public IntPforValuesWriter(int initialCapacity, int pageSize, ByteBufferAllocator allocator, int vectorSize) {
-      super(initialCapacity, pageSize, allocator, vectorSize);
+      this(initialCapacity, pageSize, allocator, vectorSize, true);
+    }
+
+    public IntPforValuesWriter(
+        int initialCapacity,
+        int pageSize,
+        ByteBufferAllocator allocator,
+        int vectorSize,
+        boolean deltaEnabled) {
+      super(initialCapacity, pageSize, allocator, vectorSize, deltaEnabled);
       this.vectorBuffer = new int[vectorSize];
       this.bufferCount = 0;
       this.totalCount = 0;
       this.encodedVectors = new CapacityByteArrayOutputStream(initialCapacity, pageSize, allocator);
       this.vectorByteSizes = new ArrayList<>();
-      this.deltasBuffer = new int[vectorSize];
+      this.residualsBuffer = new int[vectorSize];
+      this.deltaScratch = new int[vectorSize];
       this.excPosBuffer = new short[vectorSize];
       this.excValBuffer = new int[vectorSize];
       this.metadataBuf = new byte[INT32_VECTOR_INFO_SIZE];
@@ -126,35 +151,33 @@ public void writeInteger(int v) {
     }
 
     private void encodeAndFlushVector(int vectorLen) {
-      // Find minimum value (frame of reference)
-      int minValue = vectorBuffer[0];
-      for (int i = 1; i < vectorLen; i++) {
-        if (vectorBuffer[i] < minValue) {
-          minValue = vectorBuffer[i];
-        }
-      }
+      PforEncoderDecoder.VectorPlan plan =
+          PforEncoderDecoder.chooseVectorPlanForInt(vectorBuffer, vectorLen, deltaScratch, deltaEnabled);
+
+      // In the delta mode everything below runs on the differences the plan left in the
+      // scratch buffer, and the vector's first value travels in its header instead.
+      int[] source = plan.delta ? deltaScratch : vectorBuffer;
+      int frameOfReference = (int) plan.frameOfReference;
+      int bitWidth = plan.bitWidth;
+      int numExceptions = plan.numExceptions;
 
-      // Compute unsigned deltas into reusable buffer
       for (int i = 0; i < vectorLen; i++) {
-        deltasBuffer[i] = vectorBuffer[i] - minValue;
+        residualsBuffer[i] = source[i] - frameOfReference;
       }
 
-      // Find optimal bit width via cost model
-      PforEncoderDecoder.BitWidthResult result =
-          PforEncoderDecoder.findOptimalBitWidthForInt(deltasBuffer, vectorLen);
-      int bitWidth = result.bitWidth;
-      int numExceptions = result.numExceptions;
-
-      // Collect exceptions: values whose delta doesn't fit in bitWidth bits
+      // Collect exceptions: residuals that don't fit in bitWidth bits
       int excIdx = 0;
       if (numExceptions > 0) {
         int mask = (bitWidth == 32) ? -1 : (1 << bitWidth) - 1;
         for (int i = 0; i < vectorLen; i++) {
-          if (Integer.compareUnsigned(deltasBuffer[i], mask) > 0) {
+          if (Integer.compareUnsigned(residualsBuffer[i], mask) > 0) {
             excPosBuffer[excIdx] = (short) i;
-            excValBuffer[excIdx] = vectorBuffer[i];
+            // Never a residual: what the packed stream would have carried had it fitted,
+            // which in a delta vector is the difference. The reader patches it in before
+            // the prefix sum, so a patched difference is summed like any other.
+            excValBuffer[excIdx] = source[i];
             excIdx++;
-            deltasBuffer[i] = 0;
+            residualsBuffer[i] = 0;
           }
         }
       }
@@ -162,18 +185,28 @@ private void encodeAndFlushVector(int vectorLen) {
       long startSize = encodedVectors.size();
 
       // PforVectorInfo: frame_of_reference(4) + bit_width(1) + num_exceptions(2) = 7B
-      metadataBuf[0] = (byte) (minValue & 0xFF);
-      metadataBuf[1] = (byte) ((minValue >>> 8) & 0xFF);
-      metadataBuf[2] = (byte) ((minValue >>> 16) & 0xFF);
-      metadataBuf[3] = (byte) ((minValue >>> 24) & 0xFF);
-      metadataBuf[4] = (byte) bitWidth;
+      metadataBuf[0] = (byte) (frameOfReference & 0xFF);
+      metadataBuf[1] = (byte) ((frameOfReference >>> 8) & 0xFF);
+      metadataBuf[2] = (byte) ((frameOfReference >>> 16) & 0xFF);
+      metadataBuf[3] = (byte) ((frameOfReference >>> 24) & 0xFF);
+      metadataBuf[4] = (byte) (bitWidth | (plan.delta ? DELTA_FLAG : 0));
       metadataBuf[5] = (byte) (numExceptions & 0xFF);
       metadataBuf[6] = (byte) ((numExceptions >>> 8) & 0xFF);
       encodedVectors.write(metadataBuf, 0, INT32_VECTOR_INFO_SIZE);
 
-      // Pack deltas
+      // The start value sits between the info block and the packed residuals
+      if (plan.delta) {
+        int startValue = (int) plan.startValue;
+        metadataBuf[0] = (byte) (startValue & 0xFF);
+        metadataBuf[1] = (byte) ((startValue >>> 8) & 0xFF);
+        metadataBuf[2] = (byte) ((startValue >>> 16) & 0xFF);
+        metadataBuf[3] = (byte) ((startValue >>> 24) & 0xFF);
+        encodedVectors.write(metadataBuf, 0, INT32_VALUE_BYTE_WIDTH);
+      }
+
+      // Pack residuals
       if (bitWidth > 0) {
-        packIntsWithBytePacker(deltasBuffer, vectorLen, bitWidth);
+        packIntsWithBytePacker(residualsBuffer, vectorLen, bitWidth);
       }
 
       // Exception positions then values
@@ -291,7 +324,8 @@ public static class LongPforValuesWriter extends PforValuesWriter {
     private final List vectorByteSizes;
 
     // Reusable per-vector buffers
-    private final long[] deltasBuffer;
+    private final long[] residualsBuffer;
+    private final long[] deltaScratch;
     private final short[] excPosBuffer;
     private final long[] excValBuffer;
     private final byte[] metadataBuf;
@@ -299,17 +333,32 @@ public static class LongPforValuesWriter extends PforValuesWriter {
     private final long[] packPadBuf;
 
     public LongPforValuesWriter(int initialCapacity, int pageSize, ByteBufferAllocator allocator) {
-      this(initialCapacity, pageSize, allocator, DEFAULT_VECTOR_SIZE);
+      this(initialCapacity, pageSize, allocator, DEFAULT_VECTOR_SIZE, true);
+    }
+
+    public LongPforValuesWriter(
+        int initialCapacity, int pageSize, ByteBufferAllocator allocator, boolean deltaEnabled) {
+      this(initialCapacity, pageSize, allocator, DEFAULT_VECTOR_SIZE, deltaEnabled);
     }
 
     public LongPforValuesWriter(int initialCapacity, int pageSize, ByteBufferAllocator allocator, int vectorSize) {
-      super(initialCapacity, pageSize, allocator, vectorSize);
+      this(initialCapacity, pageSize, allocator, vectorSize, true);
+    }
+
+    public LongPforValuesWriter(
+        int initialCapacity,
+        int pageSize,
+        ByteBufferAllocator allocator,
+        int vectorSize,
+        boolean deltaEnabled) {
+      super(initialCapacity, pageSize, allocator, vectorSize, deltaEnabled);
       this.vectorBuffer = new long[vectorSize];
       this.bufferCount = 0;
       this.totalCount = 0;
       this.encodedVectors = new CapacityByteArrayOutputStream(initialCapacity, pageSize, allocator);
       this.vectorByteSizes = new ArrayList<>();
-      this.deltasBuffer = new long[vectorSize];
+      this.residualsBuffer = new long[vectorSize];
+      this.deltaScratch = new long[vectorSize];
       this.excPosBuffer = new short[vectorSize];
       this.excValBuffer = new long[vectorSize];
       this.metadataBuf = new byte[INT64_VECTOR_INFO_SIZE];
@@ -328,31 +377,27 @@ public void writeLong(long v) {
     }
 
     private void encodeAndFlushVector(int vectorLen) {
-      long minValue = vectorBuffer[0];
-      for (int i = 1; i < vectorLen; i++) {
-        if (vectorBuffer[i] < minValue) {
-          minValue = vectorBuffer[i];
-        }
-      }
+      PforEncoderDecoder.VectorPlan plan =
+          PforEncoderDecoder.chooseVectorPlanForLong(vectorBuffer, vectorLen, deltaScratch, deltaEnabled);
+
+      long[] source = plan.delta ? deltaScratch : vectorBuffer;
+      long frameOfReference = plan.frameOfReference;
+      int bitWidth = plan.bitWidth;
+      int numExceptions = plan.numExceptions;
 
       for (int i = 0; i < vectorLen; i++) {
-        deltasBuffer[i] = vectorBuffer[i] - minValue;
+        residualsBuffer[i] = source[i] - frameOfReference;
       }
 
-      PforEncoderDecoder.BitWidthResult result =
-          PforEncoderDecoder.findOptimalBitWidthForLong(deltasBuffer, vectorLen);
-      int bitWidth = result.bitWidth;
-      int numExceptions = result.numExceptions;
-
       int excIdx = 0;
       if (numExceptions > 0) {
         long mask = (bitWidth == 64) ? -1L : (1L << bitWidth) - 1L;
         for (int i = 0; i < vectorLen; i++) {
-          if (Long.compareUnsigned(deltasBuffer[i], mask) > 0) {
+          if (Long.compareUnsigned(residualsBuffer[i], mask) > 0) {
             excPosBuffer[excIdx] = (short) i;
-            excValBuffer[excIdx] = vectorBuffer[i];
+            excValBuffer[excIdx] = source[i];
             excIdx++;
-            deltasBuffer[i] = 0;
+            residualsBuffer[i] = 0;
           }
         }
       }
@@ -360,21 +405,34 @@ private void encodeAndFlushVector(int vectorLen) {
       long startSize = encodedVectors.size();
 
       // PforVectorInfo: frame_of_reference(8) + bit_width(1) + num_exceptions(2) = 11B
-      metadataBuf[0] = (byte) (minValue & 0xFF);
-      metadataBuf[1] = (byte) ((minValue >>> 8) & 0xFF);
-      metadataBuf[2] = (byte) ((minValue >>> 16) & 0xFF);
-      metadataBuf[3] = (byte) ((minValue >>> 24) & 0xFF);
-      metadataBuf[4] = (byte) ((minValue >>> 32) & 0xFF);
-      metadataBuf[5] = (byte) ((minValue >>> 40) & 0xFF);
-      metadataBuf[6] = (byte) ((minValue >>> 48) & 0xFF);
-      metadataBuf[7] = (byte) ((minValue >>> 56) & 0xFF);
-      metadataBuf[8] = (byte) bitWidth;
+      metadataBuf[0] = (byte) (frameOfReference & 0xFF);
+      metadataBuf[1] = (byte) ((frameOfReference >>> 8) & 0xFF);
+      metadataBuf[2] = (byte) ((frameOfReference >>> 16) & 0xFF);
+      metadataBuf[3] = (byte) ((frameOfReference >>> 24) & 0xFF);
+      metadataBuf[4] = (byte) ((frameOfReference >>> 32) & 0xFF);
+      metadataBuf[5] = (byte) ((frameOfReference >>> 40) & 0xFF);
+      metadataBuf[6] = (byte) ((frameOfReference >>> 48) & 0xFF);
+      metadataBuf[7] = (byte) ((frameOfReference >>> 56) & 0xFF);
+      metadataBuf[8] = (byte) (bitWidth | (plan.delta ? DELTA_FLAG : 0));
       metadataBuf[9] = (byte) (numExceptions & 0xFF);
       metadataBuf[10] = (byte) ((numExceptions >>> 8) & 0xFF);
       encodedVectors.write(metadataBuf, 0, INT64_VECTOR_INFO_SIZE);
 
+      if (plan.delta) {
+        long startValue = plan.startValue;
+        metadataBuf[0] = (byte) (startValue & 0xFF);
+        metadataBuf[1] = (byte) ((startValue >>> 8) & 0xFF);
+        metadataBuf[2] = (byte) ((startValue >>> 16) & 0xFF);
+        metadataBuf[3] = (byte) ((startValue >>> 24) & 0xFF);
+        metadataBuf[4] = (byte) ((startValue >>> 32) & 0xFF);
+        metadataBuf[5] = (byte) ((startValue >>> 40) & 0xFF);
+        metadataBuf[6] = (byte) ((startValue >>> 48) & 0xFF);
+        metadataBuf[7] = (byte) ((startValue >>> 56) & 0xFF);
+        encodedVectors.write(metadataBuf, 0, INT64_VALUE_BYTE_WIDTH);
+      }
+
       if (bitWidth > 0) {
-        packLongsWithBytePacker(deltasBuffer, vectorLen, bitWidth);
+        packLongsWithBytePacker(residualsBuffer, vectorLen, bitWidth);
       }
 
       if (numExceptions > 0) {
diff --git a/parquet-column/src/test/java/org/apache/parquet/column/values/pfor/PforAdversarialTest.java b/parquet-column/src/test/java/org/apache/parquet/column/values/pfor/PforAdversarialTest.java
index 5ff5780c82..8d4f4ce196 100644
--- a/parquet-column/src/test/java/org/apache/parquet/column/values/pfor/PforAdversarialTest.java
+++ b/parquet-column/src/test/java/org/apache/parquet/column/values/pfor/PforAdversarialTest.java
@@ -18,7 +18,6 @@
  */
 package org.apache.parquet.column.values.pfor;
 
-import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertThrows;
 import static org.junit.Assert.assertTrue;
@@ -130,6 +129,85 @@ private static byte[] outlierLongPage() throws Exception {
     }
   }
 
+  // A delta vector has to be worth choosing before the writer will write one: over 64
+  // values a step of 1000 takes the width from 16 bits to 10, which more than pays for
+  // the 32-bit start value.
+  private static final int DELTA_VECTOR_LEN = 64;
+
+  private static byte[] deltaIntPage() throws Exception {
+    PforValuesWriter.IntPforValuesWriter writer = null;
+    try {
+      writer = new PforValuesWriter.IntPforValuesWriter(
+          512, 512, new DirectByteBufferAllocator(), DELTA_VECTOR_LEN);
+      for (int i = 0; i < DELTA_VECTOR_LEN; i++) {
+        writer.writeInteger(1_000_000 + i * 1000);
+      }
+      return requireDeltaVector(toBytes(writer.getBytes()), PforConstants.INT32_VECTOR_INFO_SIZE);
+    } finally {
+      if (writer != null) {
+        writer.reset();
+        writer.close();
+      }
+    }
+  }
+
+  private static byte[] deltaLongPage() throws Exception {
+    PforValuesWriter.LongPforValuesWriter writer = null;
+    try {
+      writer = new PforValuesWriter.LongPforValuesWriter(
+          512, 512, new DirectByteBufferAllocator(), DELTA_VECTOR_LEN);
+      for (int i = 0; i < DELTA_VECTOR_LEN; i++) {
+        writer.writeLong(1_700_000_000_000L + (long) i * 100_000);
+      }
+      return requireDeltaVector(toBytes(writer.getBytes()), PforConstants.INT64_VECTOR_INFO_SIZE);
+    } finally {
+      if (writer != null) {
+        writer.reset();
+        writer.close();
+      }
+    }
+  }
+
+  // A delta vector with one difference far outside the cluster, so it carries an
+  // exception whose position the tests below can corrupt.
+  private static byte[] deltaOutlierIntPage() throws Exception {
+    PforValuesWriter.IntPforValuesWriter writer = null;
+    try {
+      writer = new PforValuesWriter.IntPforValuesWriter(
+          512, 512, new DirectByteBufferAllocator(), DELTA_VECTOR_LEN);
+      for (int i = 0; i < DELTA_VECTOR_LEN; i++) {
+        writer.writeInteger(500 + i * 3 + (i >= 40 ? 5_000_000 : 0));
+      }
+      return requireDeltaVector(toBytes(writer.getBytes()), PforConstants.INT32_VECTOR_INFO_SIZE);
+    } finally {
+      if (writer != null) {
+        writer.reset();
+        writer.close();
+      }
+    }
+  }
+
+  // The tests that corrupt a delta vector only mean something if the writer chose the
+  // mode in the first place.
+  private static byte[] requireDeltaVector(byte[] page, int vectorInfoSize) {
+    int bitWidthByte = page[bitWidthOffset(vectorInfoSize)] & 0xFF;
+    if ((bitWidthByte & PforConstants.DELTA_FLAG) == 0) {
+      fail("expected the writer to choose the delta mode for this vector");
+    }
+    return page;
+  }
+
+  private static int numExceptionsOfFirstVector(byte[] page) {
+    return shortLE(page, numExceptionsOffset(PforConstants.INT32_VECTOR_INFO_SIZE));
+  }
+
+  // Past the vector info, the start value, and the packed residuals.
+  private static int deltaExceptionPositionOffset(byte[] page, int vectorInfoSize) {
+    int bitWidth = page[bitWidthOffset(vectorInfoSize)] & PforConstants.BIT_WIDTH_MASK;
+    int packedBytes = (DELTA_VECTOR_LEN * bitWidth + 7) / 8;
+    return VECTOR_START + vectorInfoSize + Integer.BYTES + packedBytes;
+  }
+
   private static byte[] toBytes(BytesInput bytes) throws Exception {
     ByteBuffer bb = bytes.toByteBuffer();
     byte[] out = new byte[bb.remaining()];
@@ -373,19 +451,57 @@ public void rejectsExceptionValuesTruncated() throws Exception {
     assertThrows(ParquetDecodingException.class, () -> initIntReader(bad, OUTLIER_VECTOR_LEN));
   }
 
-  // Bit 7 of the bit width byte is reserved, so a writer that sets it must not
-  // change what a reader decodes.
+  // Bit 7 of the bit width byte is the delta flag, so a reader cannot ignore it: with
+  // the bit set, the four bytes after the vector info are the start value and the
+  // residuals begin further along. This page has no room for that, and saying so is
+  // the only safe reading -- decoding it as if the bit were absent would silently
+  // return values the writer never wrote.
   @Test
-  public void ignoresReservedBitInBitWidth() throws Exception {
+  public void rejectsDeltaFlagOnAVectorWithoutRoomForIt() throws Exception {
     byte[] page = outlierIntPage();
     int at = bitWidthOffset(PforConstants.INT32_VECTOR_INFO_SIZE);
-    byte[] withReservedBit = mutate(page, at, (byte) (page[at] | 0x80));
+    byte[] withDeltaFlag = mutate(page, at, (byte) (page[at] | PforConstants.DELTA_FLAG));
+    assertThrows(ParquetDecodingException.class, () -> initIntReader(withDeltaFlag, OUTLIER_VECTOR_LEN));
+  }
 
-    PforValuesReaderForInt reader = new PforValuesReaderForInt();
-    reader.initFromPage(OUTLIER_VECTOR_LEN, ByteBufferInputStream.wrap(ByteBuffer.wrap(withReservedBit)));
-    for (int expected : OUTLIER_INTS) {
-      assertEquals(expected, reader.readInteger());
-    }
+  // A delta vector's start value is bounded separately from its residuals, because the
+  // header bound was checked before the flag was known.
+  @Test
+  public void rejectsDeltaVectorWithTruncatedStartValue() throws Exception {
+    byte[] page = deltaIntPage();
+    // Keep the vector info and two of the start value's four bytes.
+    byte[] bad = truncate(page, VECTOR_START + PforConstants.INT32_VECTOR_INFO_SIZE + 2);
+    ParquetDecodingException e =
+        assertThrows(ParquetDecodingException.class, () -> initIntReader(bad, DELTA_VECTOR_LEN));
+    assertTrue(e.getMessage(), e.getMessage().contains("start value"));
+  }
+
+  @Test
+  public void rejectsDeltaVectorWithTruncatedStartValueLong() throws Exception {
+    byte[] page = deltaLongPage();
+    byte[] bad = truncate(page, VECTOR_START + PforConstants.INT64_VECTOR_INFO_SIZE + 3);
+    ParquetDecodingException e =
+        assertThrows(ParquetDecodingException.class, () -> initLongReader(bad, DELTA_VECTOR_LEN));
+    assertTrue(e.getMessage(), e.getMessage().contains("start value"));
+  }
+
+  // Nothing stops a corrupt page from claiming a width the value type cannot hold, and
+  // the flag must not smuggle one past the check.
+  @Test
+  public void rejectsBitWidthAboveValueWidthInADeltaVector() throws Exception {
+    byte[] page = deltaIntPage();
+    int at = bitWidthOffset(PforConstants.INT32_VECTOR_INFO_SIZE);
+    byte[] bad = mutate(page, at, (byte) (PforConstants.DELTA_FLAG | 33));
+    assertThrows(ParquetDecodingException.class, () -> initIntReader(bad, DELTA_VECTOR_LEN));
+  }
+
+  @Test
+  public void rejectsExceptionPositionPastEndOfDeltaVector() throws Exception {
+    byte[] page = deltaOutlierIntPage();
+    assertTrue("the outlier is stored as an exception", numExceptionsOfFirstVector(page) >= 1);
+    int at = deltaExceptionPositionOffset(page, PforConstants.INT32_VECTOR_INFO_SIZE);
+    byte[] bad = putShortLE(page, at, 100);
+    assertThrows(ParquetDecodingException.class, () -> initIntReader(bad, DELTA_VECTOR_LEN));
   }
 
   // Skip/read bounds
diff --git a/parquet-column/src/test/java/org/apache/parquet/column/values/pfor/PforDeltaModeTest.java b/parquet-column/src/test/java/org/apache/parquet/column/values/pfor/PforDeltaModeTest.java
new file mode 100644
index 0000000000..8d14e3bc6a
--- /dev/null
+++ b/parquet-column/src/test/java/org/apache/parquet/column/values/pfor/PforDeltaModeTest.java
@@ -0,0 +1,652 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.column.values.pfor;
+
+import static org.apache.parquet.column.values.pfor.PforConstants.BIT_WIDTH_MASK;
+import static org.apache.parquet.column.values.pfor.PforConstants.DELTA_FLAG;
+import static org.apache.parquet.column.values.pfor.PforConstants.INT32_VECTOR_INFO_SIZE;
+import static org.apache.parquet.column.values.pfor.PforConstants.INT64_VECTOR_INFO_SIZE;
+import static org.apache.parquet.column.values.pfor.PforConstants.PFOR_HEADER_SIZE;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.nio.ByteBuffer;
+import java.util.Random;
+import org.apache.parquet.bytes.ByteBufferInputStream;
+import org.apache.parquet.bytes.BytesInput;
+import org.apache.parquet.bytes.DirectByteBufferAllocator;
+import org.apache.parquet.column.ColumnDescriptor;
+import org.apache.parquet.column.ParquetProperties;
+import org.apache.parquet.column.values.ValuesWriter;
+import org.apache.parquet.schema.MessageType;
+import org.apache.parquet.schema.MessageTypeParser;
+import org.junit.Test;
+
+/**
+ * Tests for the PFOR delta mode: the writer's per-vector choice between packing the values
+ * and packing their differences, and the reader's prefix sum over the latter.
+ *
+ * 
The mode is visible on the wire in bit 7 of a vector's bit width byte and in the start
+ * value that follows the vector info when that bit is set, so the tests here assert on those
+ * bytes as well as on round trips.
+ */
+public class PforDeltaModeTest {
+
+  // Page building and reading
+
+  private static byte[] intPage(int[] values, int vectorSize, boolean deltaEnabled) throws Exception {
+    PforValuesWriter.IntPforValuesWriter writer = null;
+    try {
+      int cap = Math.max(1024, values.length * 8);
+      writer = new PforValuesWriter.IntPforValuesWriter(
+          cap, cap, new DirectByteBufferAllocator(), vectorSize, deltaEnabled);
+      for (int v : values) {
+        writer.writeInteger(v);
+      }
+      return toBytes(writer.getBytes());
+    } finally {
+      if (writer != null) {
+        writer.reset();
+        writer.close();
+      }
+    }
+  }
+
+  private static byte[] longPage(long[] values, int vectorSize, boolean deltaEnabled) throws Exception {
+    PforValuesWriter.LongPforValuesWriter writer = null;
+    try {
+      int cap = Math.max(1024, values.length * 16);
+      writer = new PforValuesWriter.LongPforValuesWriter(
+          cap, cap, new DirectByteBufferAllocator(), vectorSize, deltaEnabled);
+      for (long v : values) {
+        writer.writeLong(v);
+      }
+      return toBytes(writer.getBytes());
+    } finally {
+      if (writer != null) {
+        writer.reset();
+        writer.close();
+      }
+    }
+  }
+
+  private static byte[] toBytes(BytesInput bytes) throws Exception {
+    ByteBuffer bb = bytes.toByteBuffer();
+    byte[] out = new byte[bb.remaining()];
+    bb.duplicate().get(out);
+    return out;
+  }
+
+  private static int[] decodeInts(byte[] page, int count) throws Exception {
+    PforValuesReaderForInt reader = new PforValuesReaderForInt();
+    reader.initFromPage(count, ByteBufferInputStream.wrap(ByteBuffer.wrap(page)));
+    int[] out = new int[count];
+    for (int i = 0; i < count; i++) {
+      out[i] = reader.readInteger();
+    }
+    return out;
+  }
+
+  private static long[] decodeLongs(byte[] page, int count) throws Exception {
+    PforValuesReaderForLong reader = new PforValuesReaderForLong();
+    reader.initFromPage(count, ByteBufferInputStream.wrap(ByteBuffer.wrap(page)));
+    long[] out = new long[count];
+    for (int i = 0; i < count; i++) {
+      out[i] = reader.readLong();
+    }
+    return out;
+  }
+
+  private static void assertIntRoundTrip(int[] values, int vectorSize, boolean deltaEnabled) throws Exception {
+    int[] decoded = decodeInts(intPage(values, vectorSize, deltaEnabled), values.length);
+    for (int i = 0; i < values.length; i++) {
+      assertEquals("value at " + i, values[i], decoded[i]);
+    }
+  }
+
+  private static void assertLongRoundTrip(long[] values, int vectorSize, boolean deltaEnabled) throws Exception {
+    long[] decoded = decodeLongs(longPage(values, vectorSize, deltaEnabled), values.length);
+    for (int i = 0; i < values.length; i++) {
+      assertEquals("value at " + i, values[i], decoded[i]);
+    }
+  }
+
+  // Wire inspection. Offsets in the page are relative to the start of the offset
+  // array, which begins right after the fixed header.
+
+  private static int intLE(byte[] page, int pos) {
+    return (page[pos] & 0xFF)
+        | ((page[pos + 1] & 0xFF) << 8)
+        | ((page[pos + 2] & 0xFF) << 16)
+        | ((page[pos + 3] & 0xFF) << 24);
+  }
+
+  private static long longLE(byte[] page, int pos) {
+    return (intLE(page, pos) & 0xFFFFFFFFL) | ((long) intLE(page, pos + 4) << 32);
+  }
+
+  private static int vectorPos(byte[] page, int vectorIdx) {
+    return PFOR_HEADER_SIZE + intLE(page, PFOR_HEADER_SIZE + vectorIdx * Integer.BYTES);
+  }
+
+  private static int bitWidthByte(byte[] page, int vectorIdx, int vectorInfoSize) {
+    return page[vectorPos(page, vectorIdx) + vectorInfoSize - 3] & 0xFF;
+  }
+
+  private static boolean intDeltaFlag(byte[] page, int vectorIdx) {
+    return (bitWidthByte(page, vectorIdx, INT32_VECTOR_INFO_SIZE) & DELTA_FLAG) != 0;
+  }
+
+  private static boolean longDeltaFlag(byte[] page, int vectorIdx) {
+    return (bitWidthByte(page, vectorIdx, INT64_VECTOR_INFO_SIZE) & DELTA_FLAG) != 0;
+  }
+
+  private static int intNumExceptions(byte[] page, int vectorIdx) {
+    int pos = vectorPos(page, vectorIdx) + INT32_VECTOR_INFO_SIZE - 2;
+    return (page[pos] & 0xFF) | ((page[pos + 1] & 0xFF) << 8);
+  }
+
+  private static int longNumExceptions(byte[] page, int vectorIdx) {
+    int pos = vectorPos(page, vectorIdx) + INT64_VECTOR_INFO_SIZE - 2;
+    return (page[pos] & 0xFF) | ((page[pos + 1] & 0xFF) << 8);
+  }
+
+  // Data shapes
+
+  private static int[] monotoneInts(int count, int base, int step) {
+    int[] values = new int[count];
+    for (int i = 0; i < count; i++) {
+      values[i] = base + i * step;
+    }
+    return values;
+  }
+
+  private static long[] monotoneLongs(int count, long base, long step) {
+    long[] values = new long[count];
+    for (int i = 0; i < count; i++) {
+      values[i] = base + i * step;
+    }
+    return values;
+  }
+
+  // The mode is chosen where it pays, and shows up on the wire
+
+  @Test
+  public void monotoneIntsAreWrittenAsDifferences() throws Exception {
+    int[] values = monotoneInts(1024, 1_000_000, 7);
+    byte[] page = intPage(values, 1024, true);
+
+    assertTrue("delta flag", intDeltaFlag(page, 0));
+    // Bit 7 is the flag, so the width has to be read out from under it.
+    assertEquals(3, bitWidthByte(page, 0, INT32_VECTOR_INFO_SIZE) & BIT_WIDTH_MASK);
+    // The start value sits between the vector info and the packed residuals.
+    assertEquals(values[0], intLE(page, vectorPos(page, 0) + INT32_VECTOR_INFO_SIZE));
+    assertIntRoundTrip(values, 1024, true);
+
+    byte[] plain = intPage(values, 1024, false);
+    assertFalse("delta declined when disabled", intDeltaFlag(plain, 0));
+    assertTrue("differences pack smaller: " + page.length + " vs " + plain.length, page.length < plain.length);
+  }
+
+  @Test
+  public void monotoneLongsAreWrittenAsDifferences() throws Exception {
+    long[] values = monotoneLongs(1024, 1_700_000_000_000L, 7);
+    byte[] page = longPage(values, 1024, true);
+
+    assertTrue("delta flag", longDeltaFlag(page, 0));
+    assertEquals(3, bitWidthByte(page, 0, INT64_VECTOR_INFO_SIZE) & BIT_WIDTH_MASK);
+    assertEquals(values[0], longLE(page, vectorPos(page, 0) + INT64_VECTOR_INFO_SIZE));
+    assertLongRoundTrip(values, 1024, true);
+
+    byte[] plain = longPage(values, 1024, false);
+    assertFalse(longDeltaFlag(plain, 0));
+    assertTrue("differences pack smaller: " + page.length + " vs " + plain.length, page.length < plain.length);
+  }
+
+  @Test
+  public void descendingValuesAreWrittenAsDifferences() throws Exception {
+    // Every difference is negative, so the frame of reference is negative and the
+    // residuals are what the reader adds it back to before summing.
+    int[] values = monotoneInts(1024, 5_000_000, -11);
+    byte[] page = intPage(values, 1024, true);
+    assertTrue(intDeltaFlag(page, 0));
+    assertIntRoundTrip(values, 1024, true);
+  }
+
+  @Test
+  public void nearMonotoneValuesRoundTrip() throws Exception {
+    Random rnd = new Random(1234);
+    int[] values = new int[2000];
+    values[0] = 42;
+    for (int i = 1; i < values.length; i++) {
+      values[i] = values[i - 1] + rnd.nextInt(40) - 5;
+    }
+    assertIntRoundTrip(values, 1024, true);
+  }
+
+  @Test
+  public void modeIsDecidedPerVector() throws Exception {
+    // One monotone vector followed by one that is not: the choice has to differ
+    // between them, which is why the flag lives in the vector info.
+    Random rnd = new Random(99);
+    int[] values = new int[128];
+    for (int i = 0; i < 64; i++) {
+      values[i] = 500_000 + i * 3;
+    }
+    for (int i = 64; i < 128; i++) {
+      values[i] = rnd.nextInt();
+    }
+
+    byte[] page = intPage(values, 64, true);
+    assertTrue("monotone vector uses the mode", intDeltaFlag(page, 0));
+    assertFalse("random vector does not", intDeltaFlag(page, 1));
+    assertIntRoundTrip(values, 64, true);
+  }
+
+  @Test
+  public void randomValuesDeclineTheMode() throws Exception {
+    Random rnd = new Random(7);
+    int[] values = new int[1024];
+    for (int i = 0; i < values.length; i++) {
+      values[i] = rnd.nextInt();
+    }
+    byte[] page = intPage(values, 1024, true);
+    assertFalse(intDeltaFlag(page, 0));
+    assertIntRoundTrip(values, 1024, true);
+  }
+
+  @Test
+  public void constantVectorStaysPlain() throws Exception {
+    // A vector already packing at width 0 cannot be improved on, and differencing it
+    // would only add a start value.
+    int[] values = new int[1024];
+    java.util.Arrays.fill(values, -7);
+    byte[] page = intPage(values, 1024, true);
+    assertFalse(intDeltaFlag(page, 0));
+    assertEquals(0, bitWidthByte(page, 0, INT32_VECTOR_INFO_SIZE) & BIT_WIDTH_MASK);
+    assertIntRoundTrip(values, 1024, true);
+  }
+
+  // Wrapping. Both the differencing and the sum are modular, so values that step
+  // across the ends of the range have to come back unchanged.
+
+  @Test
+  public void intSequenceWrappingPastMaxRoundTrips() throws Exception {
+    int[] values = new int[1024];
+    int v = Integer.MAX_VALUE - 200;
+    for (int i = 0; i < values.length; i++) {
+      values[i] = v++;
+    }
+    assertIntRoundTrip(values, 1024, true);
+  }
+
+  @Test
+  public void longSequenceWrappingPastMaxRoundTrips() throws Exception {
+    long[] values = new long[1024];
+    long v = Long.MAX_VALUE - 200;
+    for (int i = 0; i < values.length; i++) {
+      values[i] = v++;
+    }
+    assertLongRoundTrip(values, 1024, true);
+  }
+
+  @Test
+  public void alternatingExtremesRoundTrip() throws Exception {
+    int[] values = new int[1024];
+    for (int i = 0; i < values.length; i++) {
+      values[i] = (i % 2 == 0) ? Integer.MIN_VALUE : Integer.MAX_VALUE;
+    }
+    assertIntRoundTrip(values, 1024, true);
+  }
+
+  @Test
+  public void alternatingLongExtremesRoundTrip() throws Exception {
+    long[] values = new long[1024];
+    for (int i = 0; i < values.length; i++) {
+      values[i] = (i % 2 == 0) ? Long.MIN_VALUE : Long.MAX_VALUE;
+    }
+    assertLongRoundTrip(values, 1024, true);
+  }
+
+  @Test
+  public void differencesAreTakenUnsigned() throws Exception {
+    int[] deltas = new int[2];
+    PforEncoderDecoder.computeDeltasForInt(new int[] {Integer.MIN_VALUE, Integer.MAX_VALUE}, 2, deltas);
+    assertEquals(0, deltas[0]);
+    assertEquals(-1, deltas[1]); // 0xFFFFFFFF, the difference taken modulo 2^32
+
+    long[] longDeltas = new long[2];
+    PforEncoderDecoder.computeDeltasForLong(new long[] {Long.MIN_VALUE, Long.MAX_VALUE}, 2, longDeltas);
+    assertEquals(0L, longDeltas[0]);
+    assertEquals(-1L, longDeltas[1]);
+  }
+
+  // Exceptions in a delta vector hold differences, so the patch has to land before
+  // the prefix sum. Patching after it would leave every later value short by the
+  // difference between the exception and the placeholder that stood in for it.
+
+  @Test
+  public void deltaVectorWithExceptionsRoundTrips() throws Exception {
+    int[] values = monotoneInts(1024, 100_000, 3);
+    for (int i = 500; i < values.length; i++) {
+      values[i] += 40_000_000; // one difference far outside the cluster
+    }
+
+    byte[] page = intPage(values, 1024, true);
+    assertTrue("delta flag", intDeltaFlag(page, 0));
+    assertTrue("the jump is stored as an exception", intNumExceptions(page, 0) >= 1);
+    assertIntRoundTrip(values, 1024, true);
+  }
+
+  @Test
+  public void deltaVectorWithExceptionsRoundTripsForLongs() throws Exception {
+    long[] values = monotoneLongs(1024, 1_700_000_000_000L, 5);
+    for (int i = 700; i < values.length; i++) {
+      values[i] += 1L << 45;
+    }
+
+    byte[] page = longPage(values, 1024, true);
+    assertTrue(longDeltaFlag(page, 0));
+    assertTrue("the jump is stored as an exception", longNumExceptions(page, 0) >= 1);
+    assertLongRoundTrip(values, 1024, true);
+  }
+
+  @Test
+  public void severalExceptionsInOneDeltaVectorRoundTrip() throws Exception {
+    int[] values = monotoneInts(1024, 0, 4);
+    int bump = 0;
+    for (int i = 0; i < values.length; i++) {
+      if (i == 10 || i == 300 || i == 301 || i == 1023) {
+        bump += 30_000_000;
+      }
+      values[i] += bump;
+    }
+    byte[] page = intPage(values, 1024, true);
+    assertTrue(intDeltaFlag(page, 0));
+    assertTrue(intNumExceptions(page, 0) >= 2);
+    assertIntRoundTrip(values, 1024, true);
+  }
+
+  // Vector sizes, partial vectors, and reads that do not start at a boundary
+
+  @Test
+  public void allVectorSizesRoundTrip() throws Exception {
+    for (int log = 3; log <= 12; log++) {
+      int vectorSize = 1 << log;
+      int[] ints = monotoneInts(1000, 3_000_000, 9);
+      long[] longs = monotoneLongs(1000, 9_000_000_000L, 9);
+      assertIntRoundTrip(ints, vectorSize, true);
+      assertLongRoundTrip(longs, vectorSize, true);
+    }
+  }
+
+  @Test
+  public void partialLastDeltaVectorRoundTrips() throws Exception {
+    // 130 values over vectors of 64 leaves a last vector of 2, which still carries
+    // its own start value.
+    assertIntRoundTrip(monotoneInts(130, 77, 6), 64, true);
+    assertLongRoundTrip(monotoneLongs(130, 77L, 6), 64, true);
+  }
+
+  @Test
+  public void twoValueDeltaVectorRoundTrips() throws Exception {
+    assertIntRoundTrip(new int[] {5, 9}, 8, true);
+    assertLongRoundTrip(new long[] {5L, 9L}, 8, true);
+  }
+
+  @Test
+  public void singleValueVectorHasNoDifferenceToTake() throws Exception {
+    byte[] page = intPage(new int[] {12345}, 8, true);
+    assertFalse(intDeltaFlag(page, 0));
+    assertIntRoundTrip(new int[] {12345}, 8, true);
+  }
+
+  @Test
+  public void skipInsideAndAcrossDeltaVectors() throws Exception {
+    int[] values = monotoneInts(300, 1_000, 4);
+    byte[] page = intPage(values, 64, true);
+    assertTrue(intDeltaFlag(page, 0));
+
+    PforValuesReaderForInt reader = new PforValuesReaderForInt();
+    reader.initFromPage(values.length, ByteBufferInputStream.wrap(ByteBuffer.wrap(page)));
+
+    for (int i = 0; i < 10; i++) {
+      assertEquals(values[i], reader.readInteger());
+    }
+    reader.skip(45); // stops inside vector 0, resumes inside the same vector
+    for (int i = 55; i < 60; i++) {
+      assertEquals(values[i], reader.readInteger());
+    }
+    reader.skip(128); // crosses two whole vectors without decoding them
+    for (int i = 188; i < 300; i++) {
+      assertEquals("value at " + i, values[i], reader.readInteger());
+    }
+  }
+
+  @Test
+  public void skipOneAtATimeThroughDeltaVectors() throws Exception {
+    long[] values = monotoneLongs(200, 50L, 3);
+    byte[] page = longPage(values, 32, true);
+    PforValuesReaderForLong reader = new PforValuesReaderForLong();
+    reader.initFromPage(values.length, ByteBufferInputStream.wrap(ByteBuffer.wrap(page)));
+    for (int i = 0; i < values.length; i++) {
+      if (i % 2 == 0) {
+        reader.skip();
+      } else {
+        assertEquals(values[i], reader.readLong());
+      }
+    }
+  }
+
+  // The decision itself
+
+  @Test
+  public void planTakesDifferencesForMonotoneInts() throws Exception {
+    int[] values = monotoneInts(1024, 1_000_000, 7);
+    PforEncoderDecoder.VectorPlan plan =
+        PforEncoderDecoder.chooseVectorPlanForInt(values, values.length, new int[values.length], true);
+
+    assertTrue(plan.delta);
+    assertEquals(values[0], plan.startValue);
+    // The differences are 0 then 7 repeated, so the frame is 0 and 7 needs 3 bits.
+    assertEquals(0, plan.frameOfReference);
+    assertEquals(3, plan.bitWidth);
+    assertEquals(0, plan.numExceptions);
+  }
+
+  @Test
+  public void planTakesDifferencesForMonotoneLongs() throws Exception {
+    long[] values = monotoneLongs(1024, 1_700_000_000_000L, 7);
+    PforEncoderDecoder.VectorPlan plan =
+        PforEncoderDecoder.chooseVectorPlanForLong(values, values.length, new long[values.length], true);
+
+    assertTrue(plan.delta);
+    assertEquals(values[0], plan.startValue);
+    assertEquals(0, plan.frameOfReference);
+    assertEquals(3, plan.bitWidth);
+  }
+
+  @Test
+  public void planKeepsValuesWhenTheModeIsDisabled() throws Exception {
+    int[] values = monotoneInts(1024, 1_000_000, 7);
+    PforEncoderDecoder.VectorPlan plan =
+        PforEncoderDecoder.chooseVectorPlanForInt(values, values.length, new int[values.length], false);
+    assertFalse(plan.delta);
+    assertEquals(0L, plan.startValue);
+  }
+
+  @Test
+  public void planPaysForTheStartValue() throws Exception {
+    // Eight values stepping by 100: differencing takes the width from 10 bits to 7,
+    // which saves 24 bits over the vector -- less than the 32 the start value costs.
+    int[] shortRun = {0, 100, 200, 300, 400, 500, 600, 700};
+    PforEncoderDecoder.VectorPlan plan =
+        PforEncoderDecoder.chooseVectorPlanForInt(shortRun, shortRun.length, new int[shortRun.length], true);
+    assertFalse("start value is not paid for over 8 values", plan.delta);
+
+    // The same step over a whole vector saves far more than it costs.
+    int[] longRun = monotoneInts(1024, 0, 100);
+    PforEncoderDecoder.VectorPlan longPlan =
+        PforEncoderDecoder.chooseVectorPlanForInt(longRun, longRun.length, new int[longRun.length], true);
+    assertTrue(longPlan.delta);
+  }
+
+  @Test
+  public void planDeclinesDifferencesForRandomValues() throws Exception {
+    Random rnd = new Random(31);
+    int[] values = new int[1024];
+    for (int i = 0; i < values.length; i++) {
+      values[i] = rnd.nextInt();
+    }
+    PforEncoderDecoder.VectorPlan plan =
+        PforEncoderDecoder.chooseVectorPlanForInt(values, values.length, new int[values.length], true);
+    assertFalse(plan.delta);
+  }
+
+  @Test
+  public void planReportsTheCheaperCost() throws Exception {
+    int[] values = monotoneInts(1024, 1_000_000, 7);
+    PforEncoderDecoder.VectorPlan delta =
+        PforEncoderDecoder.chooseVectorPlanForInt(values, values.length, new int[values.length], true);
+    PforEncoderDecoder.VectorPlan plain =
+        PforEncoderDecoder.chooseVectorPlanForInt(values, values.length, new int[values.length], false);
+    assertTrue("delta cost " + delta.costBits + " vs plain " + plain.costBits, delta.costBits < plain.costBits);
+    // The reported cost carries the start value, so it is what the two modes were
+    // compared on rather than the width alone.
+    assertEquals(1024L * 3 + 32, delta.costBits);
+  }
+
+  // Fuzz
+
+  @Test
+  public void randomWalksRoundTrip() throws Exception {
+    for (long seed = 0; seed < 20; seed++) {
+      Random rnd = new Random(seed);
+      int count = 1 + rnd.nextInt(3000);
+      int vectorSize = 1 << (3 + rnd.nextInt(8));
+      int stepRange = 1 << (1 + rnd.nextInt(30));
+
+      int[] ints = new int[count];
+      long[] longs = new long[count];
+      ints[0] = rnd.nextInt();
+      longs[0] = rnd.nextLong();
+      for (int i = 1; i < count; i++) {
+        ints[i] = ints[i - 1] + rnd.nextInt(stepRange) - stepRange / 2;
+        longs[i] = longs[i - 1] + rnd.nextInt(stepRange) - stepRange / 2;
+      }
+
+      assertIntRoundTrip(ints, vectorSize, true);
+      assertLongRoundTrip(longs, vectorSize, true);
+    }
+  }
+
+  @Test
+  public void enablingTheModeNeverChangesWhatIsRead() throws Exception {
+    for (long seed = 100; seed < 110; seed++) {
+      Random rnd = new Random(seed);
+      int count = 1 + rnd.nextInt(500);
+      int[] values = new int[count];
+      values[0] = rnd.nextInt(1000);
+      for (int i = 1; i < count; i++) {
+        values[i] = values[i - 1] + rnd.nextInt(200) - 50;
+      }
+      int[] withDelta = decodeInts(intPage(values, 128, true), count);
+      int[] withoutDelta = decodeInts(intPage(values, 128, false), count);
+      for (int i = 0; i < count; i++) {
+        assertEquals(values[i], withDelta[i]);
+        assertEquals(values[i], withoutDelta[i]);
+      }
+    }
+  }
+
+  // The writer-side property, and the path from it to the bytes
+
+  private static final MessageType SCHEMA = MessageTypeParser.parseMessageType("message m { required int32 c; }");
+
+  private static ColumnDescriptor column() {
+    return SCHEMA.getColumns().get(0);
+  }
+
+  @Test
+  public void theModeIsOnByDefault() {
+    // Leaving it on cannot make a page larger, because the writer keeps whichever mode
+    // costs fewer bits.
+    assertTrue(ParquetProperties.builder().build().isPforDeltaEnabled(column()));
+    assertTrue(ParquetProperties.DEFAULT_IS_PFOR_DELTA_ENABLED);
+  }
+
+  @Test
+  public void theModeCanBeTurnedOffGloballyAndPerColumn() {
+    ParquetProperties off =
+        ParquetProperties.builder().withPforDeltaEncoding(false).build();
+    assertFalse(off.isPforDeltaEnabled(column()));
+
+    ParquetProperties offForOneColumn =
+        ParquetProperties.builder().withPforDeltaEncoding("c", false).build();
+    assertFalse(offForOneColumn.isPforDeltaEnabled(column()));
+  }
+
+  @Test
+  public void copyingThePropertiesKeepsTheSetting() {
+    ParquetProperties off =
+        ParquetProperties.builder().withPforDeltaEncoding(false).build();
+    assertFalse(ParquetProperties.copy(off).build().isPforDeltaEnabled(column()));
+  }
+
+  @Test
+  public void thePropertyReachesTheBytes() throws Exception {
+    int[] values = monotoneInts(1024, 1_000_000, 7);
+
+    byte[] withMode = pforPage(values, true);
+    assertTrue("property on, flag set", intDeltaFlag(withMode, 0));
+
+    byte[] withoutMode = pforPage(values, false);
+    assertFalse("property off, flag clear", intDeltaFlag(withoutMode, 0));
+
+    int[] decoded = decodeInts(withoutMode, values.length);
+    for (int i = 0; i < values.length; i++) {
+      assertEquals(values[i], decoded[i]);
+    }
+  }
+
+  // Goes through the writer factory rather than constructing the PFOR writer directly,
+  // so the property is read where a real writer reads it.
+  private static byte[] pforPage(int[] values, boolean deltaEnabled) throws Exception {
+    ParquetProperties props = ParquetProperties.builder()
+        .withWriterVersion(ParquetProperties.WriterVersion.PARQUET_2_0)
+        .withDictionaryEncoding(false)
+        .withPforEncoding(true)
+        .withPforDeltaEncoding(deltaEnabled)
+        .withAllocator(new DirectByteBufferAllocator())
+        .build();
+
+    ValuesWriter writer = props.newValuesWriter(column());
+    try {
+      for (int v : values) {
+        writer.writeInteger(v);
+      }
+      assertEquals(org.apache.parquet.column.Encoding.PFOR, writer.getEncoding());
+      return toBytes(writer.getBytes());
+    } finally {
+      writer.reset();
+      writer.close();
+    }
+  }
+}

From 6fe7f116f32c986294bde207c8a8974f5cb17a1d Mon Sep 17 00:00:00 2001
From: Prateek Gaur 
Date: Sat, 5 Sep 2026 01:28:16 +0000
Subject: [PATCH 6/7] Add a Hadoop configuration key for PFOR delta

parquet.enable.pfor.delta lets a job configured only through a Hadoop
Configuration keep every PFOR vector on absolute values. It defaults to
on and is consulted only where PFOR itself is enabled.
---
 .../parquet/hadoop/ParquetOutputFormat.java   | 10 +++++++
 .../parquet/hadoop/TestPforConfiguration.java | 26 ++++++++++++++++---
 2 files changed, 32 insertions(+), 4 deletions(-)

diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java
index 41f690f517..26d40d2cdc 100644
--- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java
+++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java
@@ -86,6 +86,10 @@
  * # To enable/disable PFOR encoding for INT32 and INT64 columns
  * parquet.enable.pfor=false # true to enable PFOR encoding
  *
+ * # To allow a PFOR vector to hold the differences between its successive values
+ * # Only consulted where PFOR itself is enabled
+ * parquet.enable.pfor.delta=true # false to keep every PFOR vector on absolute values
+ *
  * # To enable/disable summary metadata aggregation at the end of a MR job
  * # The default is true (enabled)
  * parquet.enable.summary-metadata=true # false to disable summary aggregation
@@ -145,6 +149,7 @@ public static enum JobSummaryLevel {
   public static final String ENABLE_DICTIONARY = "parquet.enable.dictionary";
   public static final String ENABLE_BYTE_STREAM_SPLIT = "parquet.enable.bytestreamsplit";
   public static final String ENABLE_PFOR = "parquet.enable.pfor";
+  public static final String ENABLE_PFOR_DELTA = "parquet.enable.pfor.delta";
   public static final String VALIDATION = "parquet.validation";
   public static final String WRITER_VERSION = "parquet.writer.version";
   public static final String MEMORY_POOL_RATIO = "parquet.memory.pool.ratio";
@@ -287,6 +292,10 @@ public static boolean getPforEnabled(Configuration configuration) {
     return configuration.getBoolean(ENABLE_PFOR, ParquetProperties.DEFAULT_IS_PFOR_ENABLED);
   }
 
+  public static boolean getPforDeltaEnabled(Configuration configuration) {
+    return configuration.getBoolean(ENABLE_PFOR_DELTA, ParquetProperties.DEFAULT_IS_PFOR_DELTA_ENABLED);
+  }
+
   public static int getMinRowCountForPageSizeCheck(Configuration configuration) {
     return configuration.getInt(
         MIN_ROW_COUNT_FOR_PAGE_SIZE_CHECK, ParquetProperties.DEFAULT_MINIMUM_RECORD_COUNT_FOR_CHECK);
@@ -522,6 +531,7 @@ public RecordWriter getRecordWriter(Configuration conf, Path file, Comp
         .withDictionaryEncoding(getEnableDictionary(conf))
         .withByteStreamSplitEncoding(getByteStreamSplitEnabled(conf))
         .withPforEncoding(getPforEnabled(conf))
+        .withPforDeltaEncoding(getPforDeltaEnabled(conf))
         .withWriterVersion(getWriterVersion(conf))
         .estimateRowCountForPageSizeCheck(getEstimatePageSizeCheck(conf))
         .withMinRowCountForPageSizeCheck(getMinRowCountForPageSizeCheck(conf))
diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestPforConfiguration.java b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestPforConfiguration.java
index 3e161e5f54..b5426bbdf3 100644
--- a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestPforConfiguration.java
+++ b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestPforConfiguration.java
@@ -45,40 +45,58 @@ private static ColumnDescriptor doubleColumn() {
   @Test
   public void testDefault() throws Exception {
     Configuration conf = new Configuration();
-    // PFOR is off unless a job asks for it
+    // PFOR is off unless a job asks for it, and the delta mode is on wherever PFOR is
     assertEquals(ParquetProperties.DEFAULT_IS_PFOR_ENABLED, ParquetOutputFormat.getPforEnabled(conf));
+    assertEquals(ParquetProperties.DEFAULT_IS_PFOR_DELTA_ENABLED, ParquetOutputFormat.getPforDeltaEnabled(conf));
   }
 
   @Test
-  public void testTheKeyNameIsTheDocumentedOne() throws Exception {
-    // This string is the public surface; the class javadoc documents it
+  public void testTheKeyNamesAreTheDocumentedOnes() throws Exception {
+    // These strings are the public surface; the class javadoc documents them
     assertEquals("parquet.enable.pfor", ParquetOutputFormat.ENABLE_PFOR);
+    assertEquals("parquet.enable.pfor.delta", ParquetOutputFormat.ENABLE_PFOR_DELTA);
   }
 
   @Test
   public void testSetTrue() throws Exception {
     Configuration conf = new Configuration();
     conf.setBoolean(ParquetOutputFormat.ENABLE_PFOR, true);
+    conf.setBoolean(ParquetOutputFormat.ENABLE_PFOR_DELTA, true);
     assertTrue(ParquetOutputFormat.getPforEnabled(conf));
+    assertTrue(ParquetOutputFormat.getPforDeltaEnabled(conf));
   }
 
   @Test
   public void testSetFalse() throws Exception {
     Configuration conf = new Configuration();
     conf.setBoolean(ParquetOutputFormat.ENABLE_PFOR, false);
+    conf.setBoolean(ParquetOutputFormat.ENABLE_PFOR_DELTA, false);
     assertFalse(ParquetOutputFormat.getPforEnabled(conf));
+    assertFalse(ParquetOutputFormat.getPforDeltaEnabled(conf));
   }
 
   @Test
-  public void testTheKeyReachesTheWriterProperties() throws Exception {
+  public void testTheDeltaModeCanBeTurnedOffWhilePforStaysOn() throws Exception {
     Configuration conf = new Configuration();
     conf.setBoolean(ParquetOutputFormat.ENABLE_PFOR, true);
+    conf.setBoolean(ParquetOutputFormat.ENABLE_PFOR_DELTA, false);
+    assertTrue(ParquetOutputFormat.getPforEnabled(conf));
+    assertFalse(ParquetOutputFormat.getPforDeltaEnabled(conf));
+  }
+
+  @Test
+  public void testTheKeysReachTheWriterProperties() throws Exception {
+    Configuration conf = new Configuration();
+    conf.setBoolean(ParquetOutputFormat.ENABLE_PFOR, true);
+    conf.setBoolean(ParquetOutputFormat.ENABLE_PFOR_DELTA, false);
 
     ParquetProperties props = ParquetProperties.builder()
         .withPforEncoding(ParquetOutputFormat.getPforEnabled(conf))
+        .withPforDeltaEncoding(ParquetOutputFormat.getPforDeltaEnabled(conf))
         .build();
 
     assertTrue(props.isPforEnabled(intColumn()));
+    assertFalse(props.isPforDeltaEnabled(intColumn()));
     // PFOR encodes INT32 and INT64 only, whatever the configuration says
     assertFalse(props.isPforEnabled(doubleColumn()));
   }

From 5444a16c39889debe61f482289a98ee4dadc2905 Mon Sep 17 00:00:00 2001
From: Prateek Gaur 
Date: Sat, 5 Sep 2026 05:38:48 +0000
Subject: [PATCH 7/7] Search for the PFOR frame of reference

With the vector minimum as the frame, one value far below a tight
cluster forces a bit width wide enough to reach it, and no exception can
help. The writer now searches 256 buckets of the vector's range for a
frame that may sit above the minimum. A value below the frame wraps to a
residual too wide for the packed width and becomes an ordinary
exception. The minimum is costed first, so the chosen frame is never
more expensive. The wire format and the reader are unchanged, and the
delta mode searches the frame of its differences the same way.
---
 .../column/values/pfor/PforConstants.java     |   3 +-
 .../values/pfor/PforEncoderDecoder.java       | 275 ++++++++-
 .../column/values/pfor/PforValuesWriter.java  |   6 +-
 .../column/values/pfor/PforDeltaModeTest.java |  35 +-
 .../values/pfor/PforFrameSearchTest.java      | 535 ++++++++++++++++++
 5 files changed, 823 insertions(+), 31 deletions(-)
 create mode 100644 parquet-column/src/test/java/org/apache/parquet/column/values/pfor/PforFrameSearchTest.java

diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforConstants.java b/parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforConstants.java
index 992536b0b7..2e12188ba2 100644
--- a/parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforConstants.java
+++ b/parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforConstants.java
@@ -25,7 +25,8 @@
  *
  * 
PFOR encoding compresses integer columns (INT32/INT64) by:
  * 
    
- *   
  1. Subtracting the minimum value (Frame of Reference)
    2. 
+ *   
  2. Subtracting a frame of reference: any lower bound on the vector, chosen so the
+ *       residuals pack narrowly, and not necessarily the minimum
    3. 
  *   
  3. Choosing an optimal bit width via a cost model
    4. 
  *   
  4. Bit-packing the residuals at the chosen width
    5. 
  *   
  5. Storing outlier values (exceptions) separately with their positions
    6. 
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforEncoderDecoder.java b/parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforEncoderDecoder.java
index 0d9d86acd9..c1d35ebbad 100644
--- a/parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforEncoderDecoder.java
+++ b/parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforEncoderDecoder.java
@@ -31,6 +31,11 @@
  * 
    The same model decides the delta mode: a vector is costed as it stands and
  * again as the differences between its successive values, and the cheaper of the
  * two wins. See {@link #chooseVectorPlanForInt}.
+ *
+ * 
    It also decides the frame of reference, which is any lower bound on the vector
+ * rather than its minimum: a window placed where the values cluster can be narrower
+ * than one anchored at an outlier below them, and the values it leaves out are patched
+ * like any other exception. See {@link #searchForInt}.
  */
 public final class PforEncoderDecoder {
 
@@ -233,43 +238,283 @@ public static VectorPlan chooseVectorPlanForLong(
   }
 
   /**
-   * Take the frame of an INT32 vector and cost the widths over it, without writing the
-   * residuals out: they are needed once here, for their widths, and again by the caller
-   * only once the plan is settled.
+   * Bucket count the frame search works at, as a shift and as a count. 256 buckets keep
+   * the window scan in {@link #scanFrameWindow} at about two passes' worth of work over a
+   * 1024-value vector.
+   */
+  private static final int FRAME_SEARCH_BITS = 8;
+
+  private static final int FRAME_SEARCH_BUCKETS = 1 << FRAME_SEARCH_BITS;
+
+  /** A run of frame search buckets: the offsets in {@code [start << shift, end << shift)}. */
+  private static final class FrameWindow {
+    final int start;
+    final int end;
+
+    FrameWindow(int start, int end) {
+      this.start = start;
+      this.end = end;
+    }
+  }
+
+  /**
+   * Choose a frame of reference for an INT32 vector and the width that suits it.
+   *
+   * 
    With the minimum as the frame every exception is an overshoot: one value far below
+   * the cluster drags the packed window down with it and nothing can patch it back. Any
+   * lower bound is a valid frame, so the window can sit where the values are and patch on
+   * both sides. A value below the frame wraps, in the modular subtraction the writer
+   * already does, to a huge offset that fails the same unsigned width test as a value
+   * above the window, so it becomes an exception and is patched back with its unreduced
+   * value. There is no sign or direction to track.
+   *
+   * 
    The wire format does not change: the frame field holds a full-width value and a
+   * reader only adds it, so any PFOR reader reads these pages.
+   *
+   * 
    The search is approximate. An exact answer needs the values sorted; instead the
+   * range is bucketed with a shift and for each candidate width a window is slid over
+   * the bucket counts. Only whole buckets count as covered, so the exception estimate is
+   * an upper bound, never optimistic. The minimum as a frame is always among the
+   * candidates and it alone is costed from a real histogram, so the search can never do
+   * worse than the width search alone would have.
    */
   private static VectorPlan searchForInt(int[] source, int numElements) {
-    int frame = source[0];
+    int min = source[0];
+    int max = source[0];
     for (int i = 1; i < numElements; i++) {
-      if (source[i] < frame) {
-        frame = source[i];
+      if (source[i] < min) {
+        min = source[i];
+      } else if (source[i] > max) {
+        max = source[i];
       }
     }
 
+    // The range is an unsigned quantity even though its ends are signed: it spans the
+    // whole type when they sit at the extremes, and the subtraction wraps to say so.
+    int range = max - min;
+    if (range == 0) {
+      // A constant vector needs no search. The early exit also avoids the histogram
+      // pass, where a run of equal values sends every element to one bin and the
+      // read-modify-write serializes.
+      return new VectorPlan(false, min, 0, 0, 0, 0);
+    }
+
+    int rangeBits = bitWidthForInt(range);
+    int shift = rangeBits > FRAME_SEARCH_BITS ? rangeBits - FRAME_SEARCH_BITS : 0;
+
+    // One walk serves both halves of the search: the width histogram costs the minimum
+    // as a frame, the bucket counts cost every other frame. They are gathered together
+    // because each needs the same offset.
     int[] bitsHist = new int[33];
+    int[] counts = new int[FRAME_SEARCH_BUCKETS + 1];
     for (int i = 0; i < numElements; i++) {
-      bitsHist[bitWidthForInt(source[i] - frame)]++;
+      int offset = source[i] - min;
+      bitsHist[bitWidthForInt(offset)]++;
+      counts[offset >>> shift]++;
     }
 
+    // Candidate 0 is the minimum, costed from an exact histogram, so the search never
+    // does worse than taking the minimum.
     BitWidthResult best = bestFromHistogram(bitsHist, 32, numElements, INT32_EXCEPTION_BITS);
-    return new VectorPlan(false, frame, 0, best.bitWidth, best.numExceptions, best.costBits);
+    VectorPlan minPlan = new VectorPlan(false, min, 0, best.bitWidth, best.numExceptions, best.costBits);
+
+    // Width 0 with a few patches cannot be improved by moving the frame. Having no
+    // exceptions is different: a frame above the minimum trades a narrower width for a
+    // few patches, so an exception-free choice is where the search starts.
+    if (best.bitWidth == 0) {
+      return minPlan;
+    }
+
+    int numBuckets = (range >>> shift) + 1;
+    FrameWindow window =
+        scanFrameWindow(counts, numBuckets, shift, 32, numElements, INT32_EXCEPTION_BITS, best.costBits);
+    if (window == null) {
+      return minPlan;
+    }
+
+    // Lower the frame from the boundary of the winning window onto the smallest value the
+    // window covers. Bucket boundaries stand 2^shift apart, which on a wide column is
+    // thousands, and a cluster sitting just above one would otherwise pay those bits for
+    // nothing.
+    //
+    // A walk of its own, rather than per-bucket minima kept by the pass above: tracking
+    // them there costs every vector a compare and a store per element, including the
+    // vectors where the scan finds nothing and the minima are thrown away. Here only a
+    // vector whose search has already won pays, and it pays one traversal.
+    int windowLo = window.start << shift;
+    // A window reaching the last bucket has no upper edge to test against: that edge
+    // would be numBuckets << shift, one past the range whenever the offsets span the
+    // whole type.
+    boolean boundedAbove = window.end < numBuckets;
+    int windowHi = boundedAbove ? window.end << shift : 0;
+
+    int frameOffset = 0;
+    boolean coversAnything = false;
+    for (int i = 0; i < numElements; i++) {
+      int offset = source[i] - min;
+      if (Integer.compareUnsigned(offset, windowLo) < 0
+          || (boundedAbove && Integer.compareUnsigned(offset, windowHi) >= 0)) {
+        continue;
+      }
+      if (!coversAnything || Integer.compareUnsigned(offset, frameOffset) < 0) {
+        frameOffset = offset;
+        coversAnything = true;
+      }
+    }
+    if (!coversAnything || frameOffset == 0) {
+      return minPlan;
+    }
+
+    // Cost the winning frame exactly; this pass decides the width and the exception
+    // count. The scan works at bucket granularity and so cannot see a window narrower
+    // than one bucket, which is often where the best answer is: a sawtooth spanning 12
+    // bits has buckets 16 wide, and no scan over them resolves the 0-bit window its few
+    // patches leave behind.
+    int scanFrame = min + frameOffset;
+    int[] exactHist = new int[33];
+    for (int i = 0; i < numElements; i++) {
+      exactHist[bitWidthForInt(source[i] - scanFrame)]++;
+    }
+    BitWidthResult exact = bestFromHistogram(exactHist, 32, numElements, INT32_EXCEPTION_BITS);
+    if (exact.costBits >= best.costBits) {
+      return minPlan;
+    }
+    return new VectorPlan(false, scanFrame, 0, exact.bitWidth, exact.numExceptions, exact.costBits);
   }
 
   /** See {@link #searchForInt}. */
   private static VectorPlan searchForLong(long[] source, int numElements) {
-    long frame = source[0];
+    long min = source[0];
+    long max = source[0];
     for (int i = 1; i < numElements; i++) {
-      if (source[i] < frame) {
-        frame = source[i];
+      if (source[i] < min) {
+        min = source[i];
+      } else if (source[i] > max) {
+        max = source[i];
       }
     }
 
+    long range = max - min;
+    if (range == 0) {
+      return new VectorPlan(false, min, 0, 0, 0, 0);
+    }
+
+    int rangeBits = bitWidthForLong(range);
+    int shift = rangeBits > FRAME_SEARCH_BITS ? rangeBits - FRAME_SEARCH_BITS : 0;
+
     int[] bitsHist = new int[65];
+    int[] counts = new int[FRAME_SEARCH_BUCKETS + 1];
     for (int i = 0; i < numElements; i++) {
-      bitsHist[bitWidthForLong(source[i] - frame)]++;
+      long offset = source[i] - min;
+      bitsHist[bitWidthForLong(offset)]++;
+      counts[(int) (offset >>> shift)]++;
     }
 
     BitWidthResult best = bestFromHistogram(bitsHist, 64, numElements, INT64_EXCEPTION_BITS);
-    return new VectorPlan(false, frame, 0, best.bitWidth, best.numExceptions, best.costBits);
+    VectorPlan minPlan = new VectorPlan(false, min, 0, best.bitWidth, best.numExceptions, best.costBits);
+    if (best.bitWidth == 0) {
+      return minPlan;
+    }
+
+    int numBuckets = (int) (range >>> shift) + 1;
+    FrameWindow window =
+        scanFrameWindow(counts, numBuckets, shift, 64, numElements, INT64_EXCEPTION_BITS, best.costBits);
+    if (window == null) {
+      return minPlan;
+    }
+
+    long windowLo = (long) window.start << shift;
+    boolean boundedAbove = window.end < numBuckets;
+    long windowHi = boundedAbove ? (long) window.end << shift : 0;
+
+    long frameOffset = 0;
+    boolean coversAnything = false;
+    for (int i = 0; i < numElements; i++) {
+      long offset = source[i] - min;
+      if (Long.compareUnsigned(offset, windowLo) < 0
+          || (boundedAbove && Long.compareUnsigned(offset, windowHi) >= 0)) {
+        continue;
+      }
+      if (!coversAnything || Long.compareUnsigned(offset, frameOffset) < 0) {
+        frameOffset = offset;
+        coversAnything = true;
+      }
+    }
+    if (!coversAnything || frameOffset == 0) {
+      return minPlan;
+    }
+
+    long scanFrame = min + frameOffset;
+    int[] exactHist = new int[65];
+    for (int i = 0; i < numElements; i++) {
+      exactHist[bitWidthForLong(source[i] - scanFrame)]++;
+    }
+    BitWidthResult exact = bestFromHistogram(exactHist, 64, numElements, INT64_EXCEPTION_BITS);
+    if (exact.costBits >= best.costBits) {
+      return minPlan;
+    }
+    return new VectorPlan(false, scanFrame, 0, exact.bitWidth, exact.numExceptions, exact.costBits);
+  }
+
+  /**
+   * Slide a window of {@code 2^w} offsets over the bucket counts, for each candidate
+   * width in turn, and return where it costs least.
+   *
+   * 
    Widths below the bucket size cannot be resolved at this granularity, and once one
+   * window spans every bucket there are no exceptions left to remove, so only the
+   * {@link #FRAME_SEARCH_BITS} or so widths in between are scanned: fixed work, and none
+   * of it touching the data again.
+   *
+   * 
    What comes out is a frame, not a width. Only whole buckets count as covered, so
+   * {@code w} here is an upper bound on the width the frame really needs and the
+   * exception count an upper bound too; the caller's exact pass is what turns the frame
+   * into a plan.
+   *
+   * @param incumbentCost the cost to beat, so a window only registers if it beats the
+   *     minimum as a frame. That skips the rest of the search entirely on a column the
+   *     frame cannot help, which is most of them, and it errs in the conservative
+   *     direction: the scan over-counts exceptions, so it can decline a frame whose exact
+   *     cost would have won, but it cannot accept one that loses.
+   * @return the winning window, or null if none beat {@code incumbentCost}
+   */
+  private static FrameWindow scanFrameWindow(
+      int[] counts,
+      int numBuckets,
+      int shift,
+      int maxBits,
+      int numElements,
+      long exceptionBitsPerValue,
+      long incumbentCost) {
+    int[] prefix = new int[numBuckets + 1];
+    for (int b = 0; b < numBuckets; b++) {
+      prefix[b + 1] = prefix[b] + counts[b];
+    }
+
+    int bestStart = -1;
+    int bestEnd = 0;
+    long bestCost = incumbentCost;
+    for (int w = shift; w <= maxBits; w++) {
+      // Buckets that fit under a width of w. The exponent stays small -- there are at
+      // most FRAME_SEARCH_BUCKETS buckets, so the loop leaves as soon as w - shift
+      // reaches FRAME_SEARCH_BITS -- but it is clamped rather than shifted, because a
+      // shift count of 64 or more is not a shift at all in Java.
+      int k = (w - shift) >= FRAME_SEARCH_BITS ? numBuckets : (int) Math.min(1L << (w - shift), numBuckets);
+      for (int s = 0; s < numBuckets; s++) {
+        int end = Math.min(s + k, numBuckets);
+        long exceptions = numElements - (prefix[end] - prefix[s]);
+        long cost = (long) numElements * w + exceptions * exceptionBitsPerValue;
+        if (cost < bestCost) {
+          bestCost = cost;
+          bestStart = s;
+          bestEnd = end;
+        }
+      }
+      if (k >= numBuckets) {
+        break; // one window already spans the data
+      }
+    }
+
+    return bestStart < 0 ? null : new FrameWindow(bestStart, bestEnd);
   }
 
   /**
@@ -282,8 +527,8 @@ private static VectorPlan searchForLong(long[] source, int numElements) {
    *
    * 
    The subtraction wraps, which is what makes the round trip exact for a column
    * that spans the type's range; the reader's prefix sum wraps the same way. A vector
-   * with negative differences needs nothing special: the frame is the minimum of the
-   * differences, so subtracting it makes every residual non-negative, the same
+   * with negative differences needs nothing special: the frame is a lower bound on the
+   * differences, so subtracting it leaves residuals the packed width can hold, the same
    * mechanism a plain vector uses for negative values.
    */
   public static void computeDeltasForInt(int[] values, int numElements, int[] deltas) {
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforValuesWriter.java b/parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforValuesWriter.java
index 426a844677..181c250b92 100644
--- a/parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforValuesWriter.java
+++ b/parquet-column/src/main/java/org/apache/parquet/column/values/pfor/PforValuesWriter.java
@@ -36,9 +36,11 @@
 /**
  * PFOR (Patched Frame of Reference) values writer for INT32 and INT64 columns.
  *
- * 
    PFOR compresses integer columns by subtracting the minimum value (FOR),
+ * 
    PFOR compresses integer columns by subtracting a frame of reference (FOR),
  * selecting an optimal bit width via a histogram-based cost model, bit-packing
- * the residuals, and storing outlier values (exceptions) separately.
+ * the residuals, and storing outlier values (exceptions) separately. The frame is
+ * searched for rather than taken to be the vector minimum, so a value below it is
+ * an exception just as a value above the packed window is.
  *
  * 
    Per vector, the writer costs the values as they stand and again as the
  * differences between successive values, and keeps the cheaper of the two -- the
diff --git a/parquet-column/src/test/java/org/apache/parquet/column/values/pfor/PforDeltaModeTest.java b/parquet-column/src/test/java/org/apache/parquet/column/values/pfor/PforDeltaModeTest.java
index 8d14e3bc6a..6a81628150 100644
--- a/parquet-column/src/test/java/org/apache/parquet/column/values/pfor/PforDeltaModeTest.java
+++ b/parquet-column/src/test/java/org/apache/parquet/column/values/pfor/PforDeltaModeTest.java
@@ -194,8 +194,11 @@ public void monotoneIntsAreWrittenAsDifferences() throws Exception {
     byte[] page = intPage(values, 1024, true);
 
     assertTrue("delta flag", intDeltaFlag(page, 0));
-    // Bit 7 is the flag, so the width has to be read out from under it.
-    assertEquals(3, bitWidthByte(page, 0, INT32_VECTOR_INFO_SIZE) & BIT_WIDTH_MASK);
+    // Bit 7 is the flag, so the width has to be read out from under it. Every difference
+    // here is 7 except the leading 0, and the frame search sits the frame on 7 and patches
+    // that one, which leaves nothing to pack.
+    assertEquals(0, bitWidthByte(page, 0, INT32_VECTOR_INFO_SIZE) & BIT_WIDTH_MASK);
+    assertEquals(1, intNumExceptions(page, 0));
     // The start value sits between the vector info and the packed residuals.
     assertEquals(values[0], intLE(page, vectorPos(page, 0) + INT32_VECTOR_INFO_SIZE));
     assertIntRoundTrip(values, 1024, true);
@@ -211,7 +214,8 @@ public void monotoneLongsAreWrittenAsDifferences() throws Exception {
     byte[] page = longPage(values, 1024, true);
 
     assertTrue("delta flag", longDeltaFlag(page, 0));
-    assertEquals(3, bitWidthByte(page, 0, INT64_VECTOR_INFO_SIZE) & BIT_WIDTH_MASK);
+    assertEquals(0, bitWidthByte(page, 0, INT64_VECTOR_INFO_SIZE) & BIT_WIDTH_MASK);
+    assertEquals(1, longNumExceptions(page, 0));
     assertEquals(values[0], longLE(page, vectorPos(page, 0) + INT64_VECTOR_INFO_SIZE));
     assertLongRoundTrip(values, 1024, true);
 
@@ -465,10 +469,12 @@ public void planTakesDifferencesForMonotoneInts() throws Exception {
 
     assertTrue(plan.delta);
     assertEquals(values[0], plan.startValue);
-    // The differences are 0 then 7 repeated, so the frame is 0 and 7 needs 3 bits.
-    assertEquals(0, plan.frameOfReference);
-    assertEquals(3, plan.bitWidth);
-    assertEquals(0, plan.numExceptions);
+    // The differences are 0 then 7 repeated. A frame of 7 packs the run at width 0 and
+    // leaves the leading 0 as the one exception, which costs less than charging every
+    // value the 3 bits that a frame at the minimum would need.
+    assertEquals(7, plan.frameOfReference);
+    assertEquals(0, plan.bitWidth);
+    assertEquals(1, plan.numExceptions);
   }
 
   @Test
@@ -479,8 +485,9 @@ public void planTakesDifferencesForMonotoneLongs() throws Exception {
 
     assertTrue(plan.delta);
     assertEquals(values[0], plan.startValue);
-    assertEquals(0, plan.frameOfReference);
-    assertEquals(3, plan.bitWidth);
+    assertEquals(7, plan.frameOfReference);
+    assertEquals(0, plan.bitWidth);
+    assertEquals(1, plan.numExceptions);
   }
 
   @Test
@@ -494,8 +501,9 @@ public void planKeepsValuesWhenTheModeIsDisabled() throws Exception {
 
   @Test
   public void planPaysForTheStartValue() throws Exception {
-    // Eight values stepping by 100: differencing takes the width from 10 bits to 7,
-    // which saves 24 bits over the vector -- less than the 32 the start value costs.
+    // Eight values stepping by 100. Differencing packs them at width 0 with the leading
+    // difference patched, 48 bits in all, and the start value costs 32 more -- exactly
+    // what the values cost as they stand, so the tie keeps the values.
     int[] shortRun = {0, 100, 200, 300, 400, 500, 600, 700};
     PforEncoderDecoder.VectorPlan plan =
         PforEncoderDecoder.chooseVectorPlanForInt(shortRun, shortRun.length, new int[shortRun.length], true);
@@ -529,8 +537,9 @@ public void planReportsTheCheaperCost() throws Exception {
         PforEncoderDecoder.chooseVectorPlanForInt(values, values.length, new int[values.length], false);
     assertTrue("delta cost " + delta.costBits + " vs plain " + plain.costBits, delta.costBits < plain.costBits);
     // The reported cost carries the start value, so it is what the two modes were
-    // compared on rather than the width alone.
-    assertEquals(1024L * 3 + 32, delta.costBits);
+    // compared on rather than the width alone: nothing packed, one exception at 16 bits
+    // of position and 32 of value, and the 32-bit start value.
+    assertEquals((16 + 32) + 32L, delta.costBits);
   }
 
   // Fuzz
diff --git a/parquet-column/src/test/java/org/apache/parquet/column/values/pfor/PforFrameSearchTest.java b/parquet-column/src/test/java/org/apache/parquet/column/values/pfor/PforFrameSearchTest.java
new file mode 100644
index 0000000000..a22e67d40a
--- /dev/null
+++ b/parquet-column/src/test/java/org/apache/parquet/column/values/pfor/PforFrameSearchTest.java
@@ -0,0 +1,535 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.column.values.pfor;
+
+import static org.apache.parquet.column.values.pfor.PforConstants.BIT_WIDTH_MASK;
+import static org.apache.parquet.column.values.pfor.PforConstants.INT32_VECTOR_INFO_SIZE;
+import static org.apache.parquet.column.values.pfor.PforConstants.INT64_VECTOR_INFO_SIZE;
+import static org.apache.parquet.column.values.pfor.PforConstants.PFOR_HEADER_SIZE;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.nio.ByteBuffer;
+import java.util.Random;
+import org.apache.parquet.bytes.ByteBufferInputStream;
+import org.apache.parquet.bytes.BytesInput;
+import org.apache.parquet.bytes.DirectByteBufferAllocator;
+import org.junit.Test;
+
+/**
+ * Tests for the searched frame of reference: the frame a vector carries is any lower bound
+ * on its values, not necessarily their minimum.
+ *
+ * 
    What that buys is a packed window sitting where the values cluster, with the values
+ * outside it patched. A value below the frame needs no special handling on either side: the
+ * writer's subtraction is modular, so it wraps to an offset the packed width cannot hold and
+ * fails the same unsigned test as a value above the window, and the exception carries the
+ * unreduced value. So the tests here assert both halves -- that the frame does move off the
+ * minimum where that is cheaper, and that values on both sides of the window round trip.
+ *
+ * 
    The wire format is unchanged: the frame field holds a full-width value and a reader
+ * only adds it, so these pages are readable by any reader that could read a minimum-framed
+ * one.
+ */
+public class PforFrameSearchTest {
+
+  private static final long INT32_EXCEPTION_BITS = 16 + 32;
+  private static final long INT64_EXCEPTION_BITS = 16 + 64;
+
+  // Pages, round trips and wire inspection
+
+  private static byte[] intPage(int[] values, int vectorSize) throws Exception {
+    PforValuesWriter.IntPforValuesWriter writer = null;
+    try {
+      int cap = Math.max(1024, values.length * 8);
+      writer = new PforValuesWriter.IntPforValuesWriter(
+          cap, cap, new DirectByteBufferAllocator(), vectorSize, true);
+      for (int v : values) {
+        writer.writeInteger(v);
+      }
+      return toBytes(writer.getBytes());
+    } finally {
+      if (writer != null) {
+        writer.reset();
+        writer.close();
+      }
+    }
+  }
+
+  private static byte[] longPage(long[] values, int vectorSize) throws Exception {
+    PforValuesWriter.LongPforValuesWriter writer = null;
+    try {
+      int cap = Math.max(1024, values.length * 16);
+      writer = new PforValuesWriter.LongPforValuesWriter(
+          cap, cap, new DirectByteBufferAllocator(), vectorSize, true);
+      for (long v : values) {
+        writer.writeLong(v);
+      }
+      return toBytes(writer.getBytes());
+    } finally {
+      if (writer != null) {
+        writer.reset();
+        writer.close();
+      }
+    }
+  }
+
+  private static byte[] toBytes(BytesInput bytes) throws Exception {
+    ByteBuffer bb = bytes.toByteBuffer();
+    byte[] out = new byte[bb.remaining()];
+    bb.duplicate().get(out);
+    return out;
+  }
+
+  private static void assertIntRoundTrip(int[] values, int vectorSize) throws Exception {
+    byte[] page = intPage(values, vectorSize);
+    PforValuesReaderForInt reader = new PforValuesReaderForInt();
+    reader.initFromPage(values.length, ByteBufferInputStream.wrap(ByteBuffer.wrap(page)));
+    for (int i = 0; i < values.length; i++) {
+      assertEquals("value at " + i, values[i], reader.readInteger());
+    }
+  }
+
+  private static void assertLongRoundTrip(long[] values, int vectorSize) throws Exception {
+    byte[] page = longPage(values, vectorSize);
+    PforValuesReaderForLong reader = new PforValuesReaderForLong();
+    reader.initFromPage(values.length, ByteBufferInputStream.wrap(ByteBuffer.wrap(page)));
+    for (int i = 0; i < values.length; i++) {
+      assertEquals("value at " + i, values[i], reader.readLong());
+    }
+  }
+
+  private static int intLE(byte[] page, int pos) {
+    return (page[pos] & 0xFF)
+        | ((page[pos + 1] & 0xFF) << 8)
+        | ((page[pos + 2] & 0xFF) << 16)
+        | ((page[pos + 3] & 0xFF) << 24);
+  }
+
+  private static long longLE(byte[] page, int pos) {
+    return (intLE(page, pos) & 0xFFFFFFFFL) | ((long) intLE(page, pos + 4) << 32);
+  }
+
+  private static int vectorPos(byte[] page, int vectorIdx) {
+    return PFOR_HEADER_SIZE + intLE(page, PFOR_HEADER_SIZE + vectorIdx * Integer.BYTES);
+  }
+
+  /** The frame is the first field of a vector's info block. */
+  private static int intFrame(byte[] page, int vectorIdx) {
+    return intLE(page, vectorPos(page, vectorIdx));
+  }
+
+  private static long longFrame(byte[] page, int vectorIdx) {
+    return longLE(page, vectorPos(page, vectorIdx));
+  }
+
+  private static int intBitWidth(byte[] page, int vectorIdx) {
+    return page[vectorPos(page, vectorIdx) + INT32_VECTOR_INFO_SIZE - 3] & BIT_WIDTH_MASK;
+  }
+
+  private static int intNumExceptions(byte[] page, int vectorIdx) {
+    int pos = vectorPos(page, vectorIdx) + INT32_VECTOR_INFO_SIZE - 2;
+    return (page[pos] & 0xFF) | ((page[pos + 1] & 0xFF) << 8);
+  }
+
+  private static int longNumExceptions(byte[] page, int vectorIdx) {
+    int pos = vectorPos(page, vectorIdx) + INT64_VECTOR_INFO_SIZE - 2;
+    return (page[pos] & 0xFF) | ((page[pos + 1] & 0xFF) << 8);
+  }
+
+  // What the minimum as a frame would have cost, computed independently of the
+  // code under test, so the search can be held to never losing against it.
+
+  private static long minFrameCostInt(int[] values) {
+    int min = values[0];
+    for (int v : values) {
+      if (v < min) {
+        min = v;
+      }
+    }
+    int[] hist = new int[33];
+    for (int v : values) {
+      hist[PforEncoderDecoder.bitWidthForInt(v - min)]++;
+    }
+    return bestCost(hist, 32, values.length, INT32_EXCEPTION_BITS);
+  }
+
+  private static long minFrameCostLong(long[] values) {
+    long min = values[0];
+    for (long v : values) {
+      if (v < min) {
+        min = v;
+      }
+    }
+    int[] hist = new int[65];
+    for (long v : values) {
+      hist[PforEncoderDecoder.bitWidthForLong(v - min)]++;
+    }
+    return bestCost(hist, 64, values.length, INT64_EXCEPTION_BITS);
+  }
+
+  private static long bestCost(int[] hist, int maxBits, int numElements, long exceptionBits) {
+    long best = Long.MAX_VALUE;
+    long exceptionsAbove = numElements - hist[0];
+    for (int b = 0; b <= maxBits; b++) {
+      best = Math.min(best, (long) numElements * b + exceptionsAbove * exceptionBits);
+      if (b < maxBits) {
+        exceptionsAbove -= hist[b + 1];
+      }
+    }
+    return best;
+  }
+
+  // Data shapes
+
+  /** A tight cluster with one value far below it: the case the minimum cannot handle. */
+  private static int[] clusterWithLowOutlier(int count, int base, int spread, int outlier) {
+    Random rnd = new Random(7);
+    int[] values = new int[count];
+    for (int i = 0; i < count; i++) {
+      values[i] = base + rnd.nextInt(spread);
+    }
+    values[count / 2] = outlier;
+    return values;
+  }
+
+  private static long[] clusterWithLowOutlierLongs(int count, long base, int spread, long outlier) {
+    Random rnd = new Random(7);
+    long[] values = new long[count];
+    for (int i = 0; i < count; i++) {
+      values[i] = base + rnd.nextInt(spread);
+    }
+    values[count / 2] = outlier;
+    return values;
+  }
+
+  // The frame moves off the minimum where that is cheaper
+
+  @Test
+  public void frameSitsOnTheClusterAndNotOnTheOutlier() {
+    int[] values = clusterWithLowOutlier(1024, 1_000_000, 16, 0);
+    PforEncoderDecoder.VectorPlan plan =
+        PforEncoderDecoder.chooseVectorPlanForInt(values, values.length, new int[values.length], false);
+
+    // The minimum is 0, so a frame there would charge every value the 20 bits that
+    // 1,000,015 needs. The frame lands on the cluster instead: four bits of spread, and
+    // the one value below the window becomes an exception.
+    assertTrue("frame " + plan.frameOfReference + " is above the minimum", plan.frameOfReference >= 1_000_000);
+    assertEquals(4, plan.bitWidth);
+    assertEquals(1, plan.numExceptions);
+    assertTrue(
+        "searched cost " + plan.costBits + " beats the minimum's " + minFrameCostInt(values),
+        plan.costBits < minFrameCostInt(values));
+  }
+
+  @Test
+  public void frameSitsOnTheClusterForLongs() {
+    long[] values = clusterWithLowOutlierLongs(1024, 1_700_000_000_000L, 16, 0);
+    PforEncoderDecoder.VectorPlan plan =
+        PforEncoderDecoder.chooseVectorPlanForLong(values, values.length, new long[values.length], false);
+
+    assertTrue("frame " + plan.frameOfReference, plan.frameOfReference >= 1_700_000_000_000L);
+    assertEquals(4, plan.bitWidth);
+    assertEquals(1, plan.numExceptions);
+    assertTrue(plan.costBits < minFrameCostLong(values));
+  }
+
+  @Test
+  public void oneRepeatedValueAndAnOutlierPacksAtWidthZero() {
+    // The window has no width at all: every value but one is the same, so the frame is that
+    // value and the outlier is patched. A frame at the minimum would pay 23 bits a value.
+    int[] values = new int[1024];
+    for (int i = 0; i < values.length; i++) {
+      values[i] = 5_000_000;
+    }
+    values[100] = 0;
+
+    PforEncoderDecoder.VectorPlan plan =
+        PforEncoderDecoder.chooseVectorPlanForInt(values, values.length, new int[values.length], false);
+    assertEquals(5_000_000, plan.frameOfReference);
+    assertEquals(0, plan.bitWidth);
+    assertEquals(1, plan.numExceptions);
+    assertEquals(INT32_EXCEPTION_BITS, plan.costBits);
+  }
+
+  @Test
+  public void aFrameOnTheClusterIsWrittenToThePage() throws Exception {
+    int[] values = clusterWithLowOutlier(1024, 1_000_000, 16, 0);
+    byte[] page = intPage(values, 1024);
+
+    // The searched frame is what the page carries, in the frame of reference field.
+    assertTrue("wire frame " + intFrame(page, 0), intFrame(page, 0) >= 1_000_000);
+    assertEquals(4, intBitWidth(page, 0));
+    assertEquals(1, intNumExceptions(page, 0));
+    assertIntRoundTrip(values, 1024);
+  }
+
+  @Test
+  public void theSearchDeclinesOnAUniformColumn() {
+    // Nothing clusters, so no window can pay for the values it leaves out and the frame
+    // stays at the minimum.
+    Random rnd = new Random(11);
+    int[] values = new int[1024];
+    int min = Integer.MAX_VALUE;
+    for (int i = 0; i < values.length; i++) {
+      values[i] = rnd.nextInt();
+      min = Math.min(min, values[i]);
+    }
+
+    PforEncoderDecoder.VectorPlan plan =
+        PforEncoderDecoder.chooseVectorPlanForInt(values, values.length, new int[values.length], false);
+    assertEquals(min, plan.frameOfReference);
+    assertEquals(minFrameCostInt(values), plan.costBits);
+  }
+
+  @Test
+  public void aConstantVectorNeedsNoWidthAndNoPatches() throws Exception {
+    int[] values = new int[1024];
+    for (int i = 0; i < values.length; i++) {
+      values[i] = -42;
+    }
+    PforEncoderDecoder.VectorPlan plan =
+        PforEncoderDecoder.chooseVectorPlanForInt(values, values.length, new int[values.length], false);
+    assertEquals(-42, plan.frameOfReference);
+    assertEquals(0, plan.bitWidth);
+    assertEquals(0, plan.numExceptions);
+    assertEquals(0, plan.costBits);
+    assertIntRoundTrip(values, 1024);
+  }
+
+  // Patching on both sides of the window
+
+  @Test
+  public void valuesBelowAndAboveTheWindowAreBothPatched() throws Exception {
+    int[] values = clusterWithLowOutlier(1024, 1_000_000, 16, 0);
+    values[900] = 1_100_000; // and one above
+
+    PforEncoderDecoder.VectorPlan plan =
+        PforEncoderDecoder.chooseVectorPlanForInt(values, values.length, new int[values.length], false);
+    assertTrue("frame " + plan.frameOfReference, plan.frameOfReference >= 1_000_000);
+    assertEquals(4, plan.bitWidth);
+    assertEquals("one below the window and one above it", 2, plan.numExceptions);
+    assertIntRoundTrip(values, 1024);
+  }
+
+  @Test
+  public void valuesBelowAndAboveTheWindowAreBothPatchedForLongs() throws Exception {
+    long[] values = clusterWithLowOutlierLongs(1024, 1_700_000_000_000L, 16, 0);
+    values[900] = 1_800_000_000_000L;
+
+    PforEncoderDecoder.VectorPlan plan =
+        PforEncoderDecoder.chooseVectorPlanForLong(values, values.length, new long[values.length], false);
+    assertEquals(2, plan.numExceptions);
+    assertLongRoundTrip(values, 1024);
+  }
+
+  @Test
+  public void manyValuesBelowTheFrameRoundTrip() throws Exception {
+    // A tenth of the vector sits below the cluster, spread out, so patching has to handle
+    // a crowd of below-frame values and not just one.
+    Random rnd = new Random(19);
+    int[] values = new int[1024];
+    for (int i = 0; i < values.length; i++) {
+      values[i] = (i % 10 == 0) ? rnd.nextInt(1_000_000) : 8_000_000 + rnd.nextInt(8);
+    }
+    assertIntRoundTrip(values, 1024);
+  }
+
+  @Test
+  public void theTypeExtremesRoundTripWithASearchedFrame() throws Exception {
+    int[] ints = new int[1024];
+    for (int i = 0; i < ints.length; i++) {
+      ints[i] = (i % 3 == 0) ? Integer.MIN_VALUE : (i % 3 == 1) ? Integer.MAX_VALUE : 0;
+    }
+    assertIntRoundTrip(ints, 1024);
+
+    long[] longs = new long[1024];
+    for (int i = 0; i < longs.length; i++) {
+      longs[i] = (i % 3 == 0) ? Long.MIN_VALUE : (i % 3 == 1) ? Long.MAX_VALUE : 0;
+    }
+    assertLongRoundTrip(longs, 1024);
+
+    // A cluster at the top of the range, which is where the window has no upper edge to
+    // test against and the search has to say so rather than compute one.
+    int[] atTheTop = new int[1024];
+    for (int i = 0; i < atTheTop.length; i++) {
+      atTheTop[i] = Integer.MAX_VALUE - (i & 7);
+    }
+    atTheTop[500] = Integer.MIN_VALUE;
+    assertIntRoundTrip(atTheTop, 1024);
+  }
+
+  @Test
+  public void everyVectorSizeRoundTrips() throws Exception {
+    for (int log = 3; log <= 12; log++) {
+      int size = 1 << log;
+      int[] values = clusterWithLowOutlier(3 * size + 5, 1_000_000, 16, 0);
+      assertIntRoundTrip(values, size);
+    }
+  }
+
+  @Test
+  public void eachVectorSearchesItsOwnFrame() throws Exception {
+    // Two vectors with unrelated clusters: the frames are per vector, so neither drags the
+    // other's window.
+    int[] values = new int[2048];
+    for (int i = 0; i < 1024; i++) {
+      values[i] = 1_000_000 + (i & 15);
+    }
+    for (int i = 1024; i < 2048; i++) {
+      values[i] = -5_000_000 + (i & 15);
+    }
+    values[10] = 0;
+    values[2000] = 0;
+
+    byte[] page = intPage(values, 1024);
+    assertTrue(intFrame(page, 0) >= 1_000_000);
+    assertTrue(intFrame(page, 1) <= -5_000_000);
+    assertIntRoundTrip(values, 1024);
+  }
+
+  // The search can never lose to the minimum
+
+  @Test
+  public void theSearchNeverCostsMoreThanTheMinimumWould() {
+    for (long seed = 0; seed < 60; seed++) {
+      Random rnd = new Random(seed);
+      int count = 1 + rnd.nextInt(2000);
+      int shape = (int) (seed % 6);
+      int[] values = new int[count];
+      for (int i = 0; i < count; i++) {
+        switch (shape) {
+          case 0:
+            values[i] = rnd.nextInt();
+            break;
+          case 1:
+            values[i] = 1_000_000 + rnd.nextInt(64);
+            break;
+          case 2:
+            values[i] = rnd.nextInt(4) == 0 ? rnd.nextInt() : 500_000 + rnd.nextInt(8);
+            break;
+          case 3:
+            values[i] = -rnd.nextInt(1 << 20);
+            break;
+          case 4:
+            values[i] = rnd.nextBoolean() ? Integer.MIN_VALUE : Integer.MAX_VALUE;
+            break;
+          default:
+            values[i] = 77;
+            break;
+        }
+      }
+
+      PforEncoderDecoder.VectorPlan plan =
+          PforEncoderDecoder.chooseVectorPlanForInt(values, count, new int[count], false);
+      assertTrue(
+          "seed " + seed + ": searched " + plan.costBits + " vs minimum " + minFrameCostInt(values),
+          plan.costBits <= minFrameCostInt(values));
+    }
+  }
+
+  @Test
+  public void theSearchNeverCostsMoreThanTheMinimumWouldForLongs() {
+    for (long seed = 0; seed < 60; seed++) {
+      Random rnd = new Random(seed);
+      int count = 1 + rnd.nextInt(2000);
+      int shape = (int) (seed % 4);
+      long[] values = new long[count];
+      for (int i = 0; i < count; i++) {
+        switch (shape) {
+          case 0:
+            values[i] = rnd.nextLong();
+            break;
+          case 1:
+            values[i] = 1_700_000_000_000L + rnd.nextInt(64);
+            break;
+          case 2:
+            values[i] = rnd.nextInt(4) == 0 ? rnd.nextLong() : -900_000_000_000L + rnd.nextInt(8);
+            break;
+          default:
+            values[i] = rnd.nextBoolean() ? Long.MIN_VALUE : Long.MAX_VALUE;
+            break;
+        }
+      }
+
+      PforEncoderDecoder.VectorPlan plan =
+          PforEncoderDecoder.chooseVectorPlanForLong(values, count, new long[count], false);
+      assertTrue("seed " + seed, plan.costBits <= minFrameCostLong(values));
+    }
+  }
+
+  @Test
+  public void randomShapesRoundTripWhateverTheFrame() throws Exception {
+    for (long seed = 0; seed < 40; seed++) {
+      Random rnd = new Random(seed);
+      int count = 1 + rnd.nextInt(2500);
+      int[] ints = new int[count];
+      long[] longs = new long[count];
+      for (int i = 0; i < count; i++) {
+        boolean outlier = rnd.nextInt(20) == 0;
+        ints[i] = outlier ? rnd.nextInt() : 3_000_000 + rnd.nextInt(32);
+        longs[i] = outlier ? rnd.nextLong() : 3_000_000_000_000L + rnd.nextInt(32);
+      }
+      assertIntRoundTrip(ints, 1024);
+      assertLongRoundTrip(longs, 1024);
+    }
+  }
+
+  // The frame search and the delta mode compose
+
+  @Test
+  public void differencesGetASearchedFrameToo() throws Exception {
+    // A run of equal steps with one jump in it. The differences are one value repeated
+    // plus the jump and the leading zero, so their frame is the step and both of those are
+    // patched -- which is only reachable because the frame may sit above the minimum.
+    int[] values = new int[1024];
+    for (int i = 1; i < values.length; i++) {
+      values[i] = values[i - 1] + 7;
+    }
+    for (int i = 500; i < values.length; i++) {
+      values[i] += 1000;
+    }
+
+    PforEncoderDecoder.VectorPlan plan =
+        PforEncoderDecoder.chooseVectorPlanForInt(values, values.length, new int[values.length], true);
+    assertTrue("delta mode", plan.delta);
+    assertEquals(7, plan.frameOfReference);
+    assertEquals(0, plan.bitWidth);
+    assertEquals("the leading zero and the jump", 2, plan.numExceptions);
+    assertIntRoundTrip(values, 1024);
+  }
+
+  @Test
+  public void theSearchDeclinesWhenBucketsAreTooCoarse() throws Exception {
+    // The buckets divide the vector's range, so a value far above the cluster coarsens
+    // them: here the range is 2,000,000,000, which puts the bucket width at 8,388,608 and
+    // sweeps the low outlier into the same bucket as the cluster a million above it. No
+    // window can then separate them and the search declines, leaving the frame at the
+    // minimum, which is the floor this search guarantees. The C++ and Rust encoders make
+    // the same approximation, so a file written by any of them holds the same frame.
+    int[] values = clusterWithLowOutlier(1024, 1_000_000, 16, 0);
+    values[900] = 2_000_000_000;
+
+    PforEncoderDecoder.VectorPlan plan =
+        PforEncoderDecoder.chooseVectorPlanForInt(values, values.length, new int[values.length], false);
+    assertEquals(0, plan.frameOfReference);
+    assertEquals(minFrameCostInt(values), plan.costBits);
+    assertIntRoundTrip(values, 1024);
+  }
+}