Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -449,7 +449,8 @@ public void serializeArray(Object array, int rowSize, DataOutputStream stream)
public void serializeArray(Object array, int length, ByteBuffer buffer) {
double[] values = (double[]) array;
for (int i = 0; i < length; i++) {
buffer.putDouble(values[i]);
// Match the canonical NaN representation of the stream serialization.
buffer.putLong(Double.doubleToLongBits(values[i]));
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -447,7 +447,8 @@ public void serializeArray(Object array, int rowSize, DataOutputStream stream)
public void serializeArray(Object array, int length, ByteBuffer buffer) {
float[] values = (float[]) array;
for (int i = 0; i < length; i++) {
buffer.putFloat(values[i]);
// Match the canonical NaN representation of the stream serialization.
buffer.putInt(Float.floatToIntBits(values[i]));
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,10 @@ public abstract class AbstractAlignedPageReader implements IPageReader {
public BatchData getAllSatisfiedPageData(boolean ascending) throws IOException {
BatchData pageData = BatchDataFactory.createBatchData(TSDataType.VECTOR, ascending, false);
int timeIndex = -1;
Object[] rowValues = new Object[valueCount];
// Without record filters, only the presence of each value is needed. Avoid boxing every
// primitive just to check whether the row is all null.
Object[] rowValues =
globalTimeFilter == null && pushDownFilter == null ? null : new Object[valueCount];
while (timePageReader.hasNextTime()) {
long timestamp = timePageReader.nextTime();
timeIndex++;
Expand All @@ -141,13 +144,14 @@ public BatchData getAllSatisfiedPageData(boolean ascending) throws IOException {
ValuePageReader pageReader = valuePageReaderList.get(i);
if (pageReader != null) {
v[i] = pageReader.nextValue(timestamp, timeIndex);
rowValues[i] = (v[i] == null) ? null : v[i].getValue();
} else {
v[i] = null;
rowValues[i] = null;
}
if (rowValues[i] != null) {
hasNotNullValues = true;
if (rowValues == null) {
hasNotNullValues |= v[i] != null;
} else {
rowValues[i] = v[i] == null ? null : v[i].getValue();
hasNotNullValues |= rowValues[i] != null;
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -257,6 +257,11 @@ public static int write(byte n, OutputStream outputStream) throws IOException {
return Byte.BYTES;
}

public static int write(byte n, DataOutputStream outputStream) throws IOException {
outputStream.writeByte(n);
return Byte.BYTES;
}

/**
* write a short n.
*
Expand All @@ -268,6 +273,11 @@ public static int write(short n, OutputStream outputStream) throws IOException {
return bytes.length;
}

public static int write(short n, DataOutputStream outputStream) throws IOException {
outputStream.writeShort(n);
return SHORT_LEN;
}

/**
* write a byte n to byteBuffer.
*
Expand Down Expand Up @@ -311,6 +321,11 @@ public static int write(int n, OutputStream outputStream) throws IOException {
return INT_LEN;
}

public static int write(int n, DataOutputStream outputStream) throws IOException {
outputStream.writeInt(n);
return INT_LEN;
}

/** write the size (int) of the binary and then the bytes in binary */
public static int write(Binary binary, OutputStream outputStream) throws IOException {
byte[] size = BytesUtils.intToBytes(binary.getValues().length);
Expand All @@ -319,6 +334,13 @@ public static int write(Binary binary, OutputStream outputStream) throws IOExcep
return size.length + binary.getValues().length;
}

public static int write(Binary binary, DataOutputStream outputStream) throws IOException {
byte[] values = binary.getValues();
outputStream.writeInt(values.length);
outputStream.write(values);
return INT_LEN + values.length;
}

/**
* write a int n to byteBuffer.
*
Expand All @@ -340,6 +362,11 @@ public static int write(float n, OutputStream outputStream) throws IOException {
return FLOAT_LEN;
}

public static int write(float n, DataOutputStream outputStream) throws IOException {
outputStream.writeFloat(n);
return FLOAT_LEN;
}

/**
* Write a double n to outputStream.
*
Expand All @@ -351,6 +378,11 @@ public static int write(double n, OutputStream outputStream) throws IOException
return DOUBLE_LEN;
}

public static int write(double n, DataOutputStream outputStream) throws IOException {
outputStream.writeDouble(n);
return DOUBLE_LEN;
}

/**
* write a long n to outputStream.
*
Expand All @@ -362,6 +394,11 @@ public static int write(long n, OutputStream outputStream) throws IOException {
return LONG_LEN;
}

public static int write(long n, DataOutputStream outputStream) throws IOException {
outputStream.writeLong(n);
return LONG_LEN;
}

/** write a long n to byteBuffer. */
public static int write(long n, ByteBuffer buffer) {
buffer.putLong(n);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@
import org.apache.tsfile.utils.Binary;
import org.apache.tsfile.utils.BitMap;
import org.apache.tsfile.utils.BytesUtils;
import org.apache.tsfile.utils.PublicBAOS;
import org.apache.tsfile.utils.RamUsageEstimator;
import org.apache.tsfile.utils.ReadWriteIOUtils;
import org.apache.tsfile.write.schema.IMeasurementSchema;
Expand All @@ -42,6 +41,7 @@
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.time.LocalDate;
import java.util.ArrayList;
import java.util.Arrays;
Expand Down Expand Up @@ -601,11 +601,29 @@ private Object createValueColumnOfDataType(TSDataType dataType, int capacity) {

/** Serialize {@link Tablet} */
public ByteBuffer serialize() throws IOException {
final int serializedSize = serializedSize();
try (PublicBAOS byteArrayOutputStream = new PublicBAOS(serializedSize);
DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
serialize(outputStream);
return ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size());
final ByteBuffer buffer = ByteBuffer.allocate(serializedSize());
serialize(buffer);
buffer.flip();
return buffer;
}

/**
* Serialize this tablet at the buffer's current position using the same big-endian format as
* {@link #serialize(DataOutputStream)}. The buffer's position advances and its byte order is
* preserved. The caller must provide at least {@link #serializedSize()} remaining bytes.
*/
public void serialize(ByteBuffer buffer) {
final ByteOrder previousOrder = buffer.order();
buffer.order(ByteOrder.BIG_ENDIAN);
try {
ReadWriteIOUtils.write(insertTargetName, buffer);
ReadWriteIOUtils.write(rowSize, buffer);
writeMeasurementSchemas(buffer);
writeTimes(buffer);
writeBitMaps(buffer);
writeValues(buffer);
} finally {
buffer.order(previousOrder);
}
}

Expand Down Expand Up @@ -713,6 +731,21 @@ private void writeMeasurementSchemas(DataOutputStream stream) throws IOException
}
}

private void writeMeasurementSchemas(ByteBuffer buffer) {
ReadWriteIOUtils.write(BytesUtils.boolToByte(schemas != null), buffer);
if (schemas != null) {
ReadWriteIOUtils.write(schemas.size(), buffer);
for (int i = 0; i < schemas.size(); i++) {
IMeasurementSchema schema = schemas.get(i);
ReadWriteIOUtils.write(BytesUtils.boolToByte(schema != null), buffer);
if (schema != null) {
schema.serializeTo(buffer);
ReadWriteIOUtils.write((byte) columnCategories.get(i).ordinal(), buffer);
}
}
}
}

private void writeTimes(DataOutputStream stream) throws IOException {
ReadWriteIOUtils.write(BytesUtils.boolToByte(timestamps != null), stream);
if (timestamps != null) {
Expand All @@ -722,6 +755,15 @@ private void writeTimes(DataOutputStream stream) throws IOException {
}
}

private void writeTimes(ByteBuffer buffer) {
ReadWriteIOUtils.write(BytesUtils.boolToByte(timestamps != null), buffer);
if (timestamps != null) {
for (int i = 0; i < rowSize; i++) {
buffer.putLong(timestamps[i]);
}
}
}

/** Serialize {@link BitMap}s */
private void writeBitMaps(DataOutputStream stream) throws IOException {
ReadWriteIOUtils.write(BytesUtils.boolToByte(bitMaps != null), stream);
Expand All @@ -739,6 +781,23 @@ private void writeBitMaps(DataOutputStream stream) throws IOException {
}
}

private void writeBitMaps(ByteBuffer buffer) {
ReadWriteIOUtils.write(BytesUtils.boolToByte(bitMaps != null), buffer);
if (bitMaps != null) {
int size = schemas == null ? 0 : schemas.size();
for (int i = 0; i < size; i++) {
final boolean hasNull = bitMaps[i] != null && !bitMaps[i].isAllUnmarked(rowSize);
ReadWriteIOUtils.write(BytesUtils.boolToByte(hasNull), buffer);
if (hasNull) {
ReadWriteIOUtils.write(rowSize, buffer);
final byte[] bytes = bitMaps[i].getTruncatedByteArray(rowSize);
ReadWriteIOUtils.write(bytes.length, buffer);
buffer.put(bytes);
}
}
}
}

/** Serialize values */
private void writeValues(DataOutputStream stream) throws IOException {
ReadWriteIOUtils.write(BytesUtils.boolToByte(values != null), stream);
Expand All @@ -750,6 +809,20 @@ private void writeValues(DataOutputStream stream) throws IOException {
}
}

private void writeValues(ByteBuffer buffer) {
ReadWriteIOUtils.write(BytesUtils.boolToByte(values != null), buffer);
if (values != null) {
int size = schemas == null ? 0 : schemas.size();
for (int i = 0; i < size; i++) {
final Object column = values[i];
ReadWriteIOUtils.write(BytesUtils.boolToByte(column != null), buffer);
if (column != null) {
Type.fromTsDataType(schemas.get(i).getType()).serializeArray(column, rowSize, buffer);
}
}
}
}

private void serializeColumn(Type type, Object column, DataOutputStream stream)
throws IOException {
ReadWriteIOUtils.write(BytesUtils.boolToByte(column != null), stream);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,13 +29,15 @@
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.file.header.PageHeader;
import org.apache.tsfile.file.metadata.enums.CompressionType;
import org.apache.tsfile.read.common.BatchData;
import org.apache.tsfile.read.common.TimeRange;
import org.apache.tsfile.read.common.block.TsBlock;
import org.apache.tsfile.read.filter.basic.Filter;
import org.apache.tsfile.read.filter.factory.TimeFilterApi;
import org.apache.tsfile.read.filter.factory.ValueFilterApi;
import org.apache.tsfile.read.reader.page.AlignedPageReader;
import org.apache.tsfile.read.reader.page.LazyLoadPageData;
import org.apache.tsfile.read.reader.page.TablePageReader;
import org.apache.tsfile.read.reader.series.PaginationController;
import org.apache.tsfile.write.page.TimePageWriter;
import org.apache.tsfile.write.page.ValuePageWriter;
Expand All @@ -62,6 +64,110 @@ public class AlignedPageReaderPushDownTest {
private static final PageHeader testValuePageHeader2;
private static ByteBuffer testValuePageData2;

@Test
public void testBatchDataWithoutRecordFilters() throws IOException {
for (boolean lazy : new boolean[] {false, true}) {
AlignedPageReader reader =
lazy ? generateAlignedPageReaderUsingLazyLoad(null) : generateAlignedPageReader(null);
BatchData data = reader.getAllSatisfiedPageData(true);
for (int time = 0; time < 100; time++, data.next()) {
Assert.assertTrue(data.hasCurrent());
Assert.assertEquals(time, data.currentTime());
Assert.assertEquals(time, data.getVector()[0].getInt());
if (time >= 10 && time < 90) {
Assert.assertEquals(time, data.getVector()[1].getInt());
} else {
Assert.assertNull(data.getVector()[1]);
}
}
Assert.assertFalse(data.hasCurrent());

reader =
lazy
? generateSingleColumnAlignedPageReaderUsingLazyLoad(null, false)
: generateSingleColumnAlignedPageReader(null, false);
data = reader.getAllSatisfiedPageData(true);
for (int time = 10; time < 90; time++, data.next()) {
Assert.assertTrue(data.hasCurrent());
Assert.assertEquals(time, data.currentTime());
Assert.assertEquals(time, data.getVector()[0].getInt());
}
Assert.assertFalse(data.hasCurrent());
}
}

@Test
public void testBatchDataWithoutRecordFiltersHonorsDeletes() throws IOException {
AlignedPageReader reader = generateAlignedPageReader(null, Arrays.asList(true, true));
reader.setDeleteIntervalList(
Arrays.asList(
Collections.singletonList(new TimeRange(0, 29)),
Collections.singletonList(new TimeRange(20, 39))));
BatchData data = reader.getAllSatisfiedPageData(true);
for (int time = 0; time < 100; time++) {
if (time < 10 || (time >= 20 && time < 30)) {
continue;
}
Assert.assertTrue(data.hasCurrent());
Assert.assertEquals(time, data.currentTime());
if (time < 30) {
Assert.assertNull(data.getVector()[0]);
} else {
Assert.assertEquals(time, data.getVector()[0].getInt());
}
if ((time >= 10 && time < 20) || (time >= 40 && time < 90)) {
Assert.assertEquals(time, data.getVector()[1].getInt());
} else {
Assert.assertNull(data.getVector()[1]);
}
data.next();
}
Assert.assertFalse(data.hasCurrent());
}

@Test
public void testBatchDataWithRecordFilters() throws IOException {
for (Filter timeFilter : new Filter[] {null, TimeFilterApi.between(30, 69)}) {
AlignedPageReader reader = generateAlignedPageReader(timeFilter);
reader.addRecordFilter(ValueFilterApi.gtEq(DEFAULT_MEASUREMENT_INDEX, 50, TSDataType.INT32));
BatchData data = reader.getAllSatisfiedPageData(true);
int end = timeFilter == null ? 100 : 70;
for (int time = 50; time < end; time++, data.next()) {
Assert.assertTrue(data.hasCurrent());
Assert.assertEquals(time, data.currentTime());
Assert.assertEquals(time, data.getVector()[0].getInt());
}
Assert.assertFalse(data.hasCurrent());
}
}

@Test
public void testTableBatchDataKeepsAllNullRowsWithoutRecordFilters() throws IOException {
resetDataBuffer();
TablePageReader reader =
new TablePageReader(
testTimePageHeader,
testTimePageData,
new DeltaBinaryDecoder.LongDeltaDecoder(),
Arrays.asList(null, testValuePageHeader2),
Arrays.asList(null, testValuePageData2),
Arrays.asList(TSDataType.INT32, TSDataType.INT32),
Arrays.asList(new IntRleDecoder(), new IntRleDecoder()),
null);
BatchData data = reader.getAllSatisfiedPageData(true);
for (int time = 0; time < 100; time++, data.next()) {
Assert.assertTrue(data.hasCurrent());
Assert.assertEquals(time, data.currentTime());
Assert.assertNull(data.getVector()[0]);
if (time >= 10 && time < 90) {
Assert.assertEquals(time, data.getVector()[1].getInt());
} else {
Assert.assertNull(data.getVector()[1]);
}
}
Assert.assertFalse(data.hasCurrent());
}

static {
TimePageWriter timePageWriter =
new TimePageWriter(
Expand Down
Loading
Loading