Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -24,13 +24,15 @@

import org.apache.tsfile.common.conf.TSFileConfig;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.read.common.Field;
import org.apache.tsfile.read.common.type.Type;
import org.apache.tsfile.utils.Binary;
import org.apache.tsfile.utils.BitMap;
import org.apache.tsfile.write.record.Tablet;
import org.apache.tsfile.write.schema.IMeasurementSchema;
import org.apache.tsfile.write.schema.MeasurementSchema;

import java.time.LocalDate;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
Expand Down Expand Up @@ -171,6 +173,57 @@ public static void putValue(
unmarkNullValue(tablet, rowIndex, columnIndex);
}

/** Copy a query field without boxing primitive values on the way to the tablet. */
public static void putFieldValue(
final Tablet tablet,
final int rowIndex,
final int columnIndex,
final TSDataType dataType,
final Field field) {
if (field.getDataType() == null) {
putValue(tablet, rowIndex, columnIndex, dataType, null);
return;
}

final Object column = tablet.getValues()[columnIndex];
switch (dataType) {
case BOOLEAN:
((boolean[]) column)[rowIndex] = field.getBoolV();
break;
case INT32:
((int[]) column)[rowIndex] = field.getIntV();
break;
case INT64:
case TIMESTAMP:
((long[]) column)[rowIndex] = field.getLongV();
break;
case FLOAT:
((float[]) column)[rowIndex] = field.getFloatV();
break;
case DOUBLE:
((double[]) column)[rowIndex] = field.getDoubleV();
break;
case DATE:
if (!(column instanceof LocalDate[])) {
putValue(tablet, rowIndex, columnIndex, dataType, field.getObjectValue(dataType));
return;
}
((LocalDate[]) column)[rowIndex] = field.getDateV();
break;
case TEXT:
case STRING:
case BLOB:
final Binary value = field.getBinaryV();
((Binary[]) column)[rowIndex] = value == null ? Binary.EMPTY_VALUE : value;
break;
default:
putValue(tablet, rowIndex, columnIndex, dataType, field.getObjectValue(dataType));
return;
}
// A row discarded after applying mods may have marked this slot before it is reused.
unmarkNullValue(tablet, rowIndex, columnIndex);
}

public static void unmarkNullValue(
final Tablet tablet, final int rowIndex, final int columnIndex) {
final BitMap[] bitMaps = tablet.getBitMaps();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,6 @@ private Tablet buildNextTablet() throws IOException {
final int fieldSize = fields.size();
for (int i = 0; i < fieldSize; i++) {
final Field field = fields.get(i);
final String measurement = measurements.get(i);
final TSDataType dataType = schemas.get(i).getType();
// Check if this value is deleted by mods
if (field == null
Expand All @@ -194,8 +193,7 @@ private Tablet buildNextTablet() throws IOException {
}
PipeTabletUtils.markNullValue(tablet, rowIndex, i);
} else {
PipeTabletUtils.putValue(
tablet, rowIndex, i, dataType, field.getObjectValue(schemas.get(i).getType()));
PipeTabletUtils.putFieldValue(tablet, rowIndex, i, dataType, field);
isNeedFillTime = true;
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,12 +32,10 @@
import org.apache.tsfile.read.common.type.Type;
import org.apache.tsfile.utils.BitMap;
import org.apache.tsfile.utils.Pair;
import org.apache.tsfile.utils.PublicBAOS;
import org.apache.tsfile.utils.RamUsageEstimator;
import org.apache.tsfile.utils.ReadWriteIOUtils;
import org.apache.tsfile.write.record.Tablet;

import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
Expand Down Expand Up @@ -200,14 +198,7 @@ public PipeTransferTabletBatchReqV2 toTPipeTransferReq() throws IOException {
}
}
for (final Pair<Boolean, Tablet> tabletPair : batchTablets) {
try (final PublicBAOS byteArrayOutputStream =
new PublicBAOS(calculateTabletSerializedSize(tabletPair.getRight()));
final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
tabletPair.getRight().serialize(outputStream);
ReadWriteIOUtils.write(true, outputStream);
tabletBuffers.add(
ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size()));
}
tabletBuffers.add(serializeTabletWithAlignment(tabletPair.getRight(), true));
tabletDataBases.add(databaseName);
}
}
Expand Down Expand Up @@ -268,13 +259,7 @@ private long buildTabletInsertionBuffer(final TabletInsertionEvent event) throws
pipeRawTabletInsertionEvent.getTableModelDatabaseName());
} else {
final Tablet tablet = pipeRawTabletInsertionEvent.convertToTablet();
try (final PublicBAOS byteArrayOutputStream =
new PublicBAOS(calculateTabletSerializedSize(tablet));
final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
tablet.serialize(outputStream);
ReadWriteIOUtils.write(pipeRawTabletInsertionEvent.isAligned(), outputStream);
buffer = ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size());
}
buffer = serializeTabletWithAlignment(tablet, pipeRawTabletInsertionEvent.isAligned());
final String databaseName = pipeRawTabletInsertionEvent.getTreeModelDatabaseName();
estimateSize = RamUsageEstimator.sizeOf(databaseName) + buffer.limit();
increaseTotalBufferSizeAndUpdateMemoryBlock(estimateSize);
Expand Down Expand Up @@ -307,8 +292,14 @@ private static long calculateTabletSizeInBytes(final Tablet tablet) {
return PipeMemoryWeightUtil.calculateTabletSizeInBytes(tablet) + 4;
}

private static int calculateTabletSerializedSize(final Tablet tablet) {
return tablet.serializedSize() + Byte.BYTES;
private static ByteBuffer serializeTabletWithAlignment(
final Tablet tablet, final boolean isAligned) {
final ByteBuffer buffer =
ByteBuffer.allocate(Math.addExact(tablet.serializedSize(), Byte.BYTES));
tablet.serialize(buffer);
ReadWriteIOUtils.write(isAligned, buffer);
buffer.flip();
return buffer;
}

static boolean mayAppendTablet(final Tablet target, final Tablet source) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,19 @@
package org.apache.iotdb.db.pipe.event.common.tablet;

import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.read.common.Field;
import org.apache.tsfile.read.common.type.Type;
import org.apache.tsfile.utils.Binary;
import org.apache.tsfile.utils.DateUtils;
import org.apache.tsfile.write.record.Tablet;
import org.apache.tsfile.write.schema.IMeasurementSchema;
import org.apache.tsfile.write.schema.MeasurementSchema;
import org.junit.Assert;
import org.junit.Test;

import java.time.LocalDate;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;

public class PipeTabletUtilsTest {
Expand All @@ -49,4 +55,66 @@ public void testPutValueUnmarksReusedNullRow() {
Assert.assertFalse(tablet.isNull(0, 0));
Assert.assertTrue(tablet.isNull(0, 1));
}

@Test
public void testPutFieldValueMatchesObjectConversion() {
final TSDataType[] types = {
TSDataType.BOOLEAN,
TSDataType.INT32,
TSDataType.INT64,
TSDataType.TIMESTAMP,
TSDataType.FLOAT,
TSDataType.DOUBLE,
TSDataType.DATE,
TSDataType.TEXT,
TSDataType.STRING,
TSDataType.BLOB,
TSDataType.OBJECT
};
final Object[] values = {
true,
123456789,
9876543210L,
Long.MIN_VALUE,
-0.0f,
Double.longBitsToDouble(0x7ff0000000000001L),
DateUtils.parseDateExpressionToInt(LocalDate.of(2024, 2, 29)),
new Binary(new byte[] {1, 2, 3}),
new Binary(new byte[] {4, 5}),
new Binary(new byte[0]),
new Binary(new byte[16])
};
for (int i = 0; i < types.length; i++) {
final List<IMeasurementSchema> schemas =
Collections.singletonList(new MeasurementSchema("s1", types[i]));
final Tablet expected = new Tablet("root.sg.d1", schemas, 1);
final Tablet actual = new Tablet("root.sg.d1", schemas, 1);
final Field field = Type.fromTsDataType(types[i]).getField(values[i]);
PipeTabletUtils.markNullValue(actual, 0, 0);

PipeTabletUtils.putValue(expected, 0, 0, types[i], field.getObjectValue(types[i]));
PipeTabletUtils.putFieldValue(actual, 0, 0, types[i], field);

Assert.assertFalse(actual.isNull(0, 0));
Assert.assertEquals(expected.getValue(0, 0), actual.getValue(0, 0));
}
}

@Test
public void testPutFieldValuePreservesNullFieldDefaults() {
for (final TSDataType type :
Arrays.asList(TSDataType.INT32, TSDataType.DATE, TSDataType.STRING, TSDataType.BLOB)) {
final List<IMeasurementSchema> schemas =
Collections.singletonList(new MeasurementSchema("s1", type));
final Tablet expected = new Tablet("root.sg.d1", schemas, 1);
final Tablet actual = new Tablet("root.sg.d1", schemas, 1);
PipeTabletUtils.markNullValue(actual, 0, 0);

PipeTabletUtils.putValue(expected, 0, 0, type, null);
PipeTabletUtils.putFieldValue(actual, 0, 0, type, new Field(null));

Assert.assertFalse(actual.isNull(0, 0));
Assert.assertEquals(expected.getValue(0, 0), actual.getValue(0, 0));
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.iotdb.db.pipe.event.common.tsfile.parser.query;

import org.apache.iotdb.commons.path.MeasurementPath;
import org.apache.iotdb.commons.path.PatternTreeMap;
import org.apache.iotdb.db.pipe.event.common.tablet.PipeTabletUtils.TabletStringInternPool;
import org.apache.iotdb.db.pipe.event.common.tsfile.parser.TsFileInsertionEventParserMemoryBlock;
import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry;
import org.apache.iotdb.db.storageengine.dataregion.modification.TreeDeletionEntry;
import org.apache.iotdb.db.utils.datastructure.PatternTreeMapFactory;

import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.file.metadata.IDeviceID;
import org.apache.tsfile.read.TsFileReader;
import org.apache.tsfile.read.common.Field;
import org.apache.tsfile.read.common.RowRecord;
import org.apache.tsfile.read.expression.QueryExpression;
import org.apache.tsfile.read.query.dataset.QueryDataSet;
import org.apache.tsfile.utils.Binary;
import org.apache.tsfile.write.record.Tablet;
import org.junit.Assert;
import org.junit.Test;

import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;

import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

public class TsFileInsertionEventQueryParserTabletIteratorTest {

@Test
public void testDeletedRowSlotIsReusedWithCorrectNulls() throws Exception {
final RowRecord deleted = row(0, 10L, new Binary(new byte[] {1}));
final RowRecord sparse = row(1, 11L, null);
final RowRecord dense = row(2, 12L, new Binary(new byte[] {2, 3}));
final List<RowRecord> rows = Arrays.asList(deleted, sparse, dense);
final AtomicInteger cursor = new AtomicInteger();
final QueryDataSet dataSet = mock(QueryDataSet.class);
when(dataSet.hasNext()).thenAnswer(invocation -> cursor.get() < rows.size());
when(dataSet.next()).thenAnswer(invocation -> rows.get(cursor.getAndIncrement()));
final TsFileReader reader = mock(TsFileReader.class);
when(reader.query(any(QueryExpression.class))).thenReturn(dataSet);
final TsFileInsertionEventParserMemoryBlock memory =
mock(TsFileInsertionEventParserMemoryBlock.class);
when(memory.getMemoryUsageInBytes()).thenReturn(Long.MAX_VALUE);
final PatternTreeMap<ModEntry, PatternTreeMapFactory.ModsSerializer> modifications =
PatternTreeMapFactory.getModsPatternTreeMap();
for (final TreeDeletionEntry deletion :
Arrays.asList(
new TreeDeletionEntry(new MeasurementPath("root.sg.d.s1"), 0, 0),
new TreeDeletionEntry(new MeasurementPath("root.sg.d.s2"), 0, 0))) {
modifications.append(deletion.keyOfPatternTree(), deletion);
}

final TsFileInsertionEventQueryParserTabletIterator iterator =
new TsFileInsertionEventQueryParserTabletIterator(
reader,
Map.of("root.sg.d.s1", TSDataType.INT64, "root.sg.d.s2", TSDataType.STRING),
IDeviceID.Factory.DEFAULT_FACTORY.create("root.sg.d"),
Arrays.asList("s2", "s1"),
null,
memory,
modifications,
new TabletStringInternPool());

final Tablet tablet = iterator.next();
Assert.assertEquals(2, tablet.getRowSize());
Assert.assertEquals(1, tablet.getTimestamp(0));
Assert.assertEquals(11L, tablet.getValue(0, 0));
Assert.assertFalse(tablet.isNull(0, 0));
Assert.assertTrue(tablet.isNull(0, 1));
Assert.assertEquals(2, tablet.getTimestamp(1));
Assert.assertEquals(12L, tablet.getValue(1, 0));
Assert.assertEquals(dense.getFields().get(1).getBinaryV(), tablet.getValue(1, 1));
Assert.assertFalse(tablet.isNull(1, 1));
Assert.assertFalse(iterator.hasNext());
}

private static RowRecord row(final long time, final long value, final Binary binary) {
final RowRecord row = new RowRecord(time);
final Field number = new Field(TSDataType.INT64);
number.setLongV(value);
row.addField(number);
if (binary == null) {
row.addField(null);
} else {
final Field text = new Field(TSDataType.STRING);
text.setBinaryV(binary);
row.addField(text);
}
return row;
}
}
Loading
Loading