From 960f564758a540cde1304f944bbcf32d87fb9b7a Mon Sep 17 00:00:00 2001 From: 761417898 <761417898@qq.com> Date: Thu, 12 Nov 2026 16:58:28 +0800 Subject: [PATCH 1/5] Support log payload compression in IoTConsensus V1 --- iotdb-core/consensus/pom.xml | 4 + .../consensus/i18n/IoTConsensusMessages.java | 4 + .../consensus/i18n/IoTConsensusMessages.java | 4 + .../consensus/config/IoTConsensusConfig.java | 37 ++ .../iot/IoTConsensusServerMetrics.java | 20 + .../iot/client/DispatchLogHandler.java | 13 +- .../IoTConsensusMemoryManager.java | 9 + .../iot/logdispatcher/LogCompression.java | 332 ++++++++++++++++ .../iot/logdispatcher/LogDispatcher.java | 45 ++- .../LogDispatcherThreadMetrics.java | 75 ++++ .../IoTConsensusRPCServiceProcessor.java | 14 + .../iot/CompressedReplicationTest.java | 178 +++++++++ .../iot/client/DispatchLogHandlerTest.java | 89 +++++ .../iot/logdispatcher/LogCompressionTest.java | 364 ++++++++++++++++++ .../LogDispatcherCompressionTest.java | 178 +++++++++ .../LogDispatcherThreadMetricsTest.java | 41 ++ .../iotdb/db/i18n/DataNodeMiscMessages.java | 1 + .../iotdb/db/i18n/DataNodeMiscMessages.java | 1 + .../org/apache/iotdb/db/conf/IoTDBConfig.java | 19 + .../apache/iotdb/db/conf/IoTDBDescriptor.java | 21 + .../db/consensus/DataRegionConsensusImpl.java | 2 + .../conf/iotdb-system.properties.template | 14 + .../commons/service/metric/enums/Metric.java | 2 + .../src/main/thrift/iotconsensus.thrift | 18 + 24 files changed, 1482 insertions(+), 3 deletions(-) create mode 100644 iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompression.java create mode 100644 iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/CompressedReplicationTest.java create mode 100644 iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompressionTest.java create mode 100644 iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcherCompressionTest.java diff --git a/iotdb-core/consensus/pom.xml b/iotdb-core/consensus/pom.xml index 7130af748289f..922b68801f2fc 100644 --- a/iotdb-core/consensus/pom.xml +++ b/iotdb-core/consensus/pom.xml @@ -114,6 +114,10 @@ com.google.code.findbugs jsr305 + + com.github.luben + zstd-jni + org.apache.tsfile tsfile diff --git a/iotdb-core/consensus/src/main/i18n/en/org/apache/iotdb/consensus/i18n/IoTConsensusMessages.java b/iotdb-core/consensus/src/main/i18n/en/org/apache/iotdb/consensus/i18n/IoTConsensusMessages.java index d07373015a83e..3ebcd8f93e839 100644 --- a/iotdb-core/consensus/src/main/i18n/en/org/apache/iotdb/consensus/i18n/IoTConsensusMessages.java +++ b/iotdb-core/consensus/src/main/i18n/en/org/apache/iotdb/consensus/i18n/IoTConsensusMessages.java @@ -25,6 +25,10 @@ */ public final class IoTConsensusMessages { + public static final String EXCEPTION_INVALID_IOTCONSENSUS_LOG_COMPRESSION_MINIMUM_SIZE_ARG_ACEEA255 = "Invalid IoTConsensus log compression minimum size: %d"; + + public static final String EXCEPTION_INVALID_IOTCONSENSUS_COMPRESSED_LOG_ARG_EXPECTED_ARG_ACTUAL_ARG_4986175E = "Invalid IoTConsensus compressed log (%s, expected %d, actual %d)"; + private IoTConsensusMessages() {} // ===================== IoTConsensus lifecycle ===================== diff --git a/iotdb-core/consensus/src/main/i18n/zh/org/apache/iotdb/consensus/i18n/IoTConsensusMessages.java b/iotdb-core/consensus/src/main/i18n/zh/org/apache/iotdb/consensus/i18n/IoTConsensusMessages.java index 0d908aa8613de..5b2ddc42f860d 100644 --- a/iotdb-core/consensus/src/main/i18n/zh/org/apache/iotdb/consensus/i18n/IoTConsensusMessages.java +++ b/iotdb-core/consensus/src/main/i18n/zh/org/apache/iotdb/consensus/i18n/IoTConsensusMessages.java @@ -24,6 +24,10 @@ */ public final class IoTConsensusMessages { + public static final String EXCEPTION_INVALID_IOTCONSENSUS_LOG_COMPRESSION_MINIMUM_SIZE_ARG_ACEEA255 = "无效的 IoTConsensus 日志压缩最小大小:%d"; + + public static final String EXCEPTION_INVALID_IOTCONSENSUS_COMPRESSED_LOG_ARG_EXPECTED_ARG_ACTUAL_ARG_4986175E = "无效的 IoTConsensus 压缩日志(%s,预期 %d,实际 %d)"; + private IoTConsensusMessages() {} // ===================== IoTConsensus 生命周期 ===================== diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/config/IoTConsensusConfig.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/config/IoTConsensusConfig.java index c6ca9a7e0dd5c..c31197f0334eb 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/config/IoTConsensusConfig.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/config/IoTConsensusConfig.java @@ -22,6 +22,8 @@ import org.apache.iotdb.commons.client.property.ClientPoolProperty.DefaultProperty; import org.apache.iotdb.commons.memory.AtomicLongMemoryBlock; import org.apache.iotdb.commons.memory.IMemoryBlock; +import org.apache.iotdb.consensus.i18n.IoTConsensusMessages; +import org.apache.iotdb.consensus.iot.thrift.TLogCompressionType; import java.util.Optional; import java.util.concurrent.TimeUnit; @@ -310,6 +312,8 @@ public static class Replication { private final int maxLogEntriesNumPerBatch; private final int maxSizePerBatch; + private final TLogCompressionType logCompressionType; + private final int logCompressionMinSize; private final int maxPendingBatchesNum; private final int maxQueueLength; @@ -330,6 +334,8 @@ public static class Replication { private Replication( int maxLogEntriesNumPerBatch, int maxSizePerBatch, + TLogCompressionType logCompressionType, + int logCompressionMinSize, int maxPendingBatchesNum, int maxQueueLength, long maxWaitingTimeForWaitBatchInMs, @@ -347,6 +353,8 @@ private Replication( long snapshotTransmissionProgressLogIntervalMs) { this.maxLogEntriesNumPerBatch = maxLogEntriesNumPerBatch; this.maxSizePerBatch = maxSizePerBatch; + this.logCompressionType = logCompressionType; + this.logCompressionMinSize = logCompressionMinSize; this.maxPendingBatchesNum = maxPendingBatchesNum; this.maxQueueLength = maxQueueLength; this.maxWaitingTimeForWaitBatchInMs = maxWaitingTimeForWaitBatchInMs; @@ -372,6 +380,14 @@ public int getMaxSizePerBatch() { return maxSizePerBatch; } + public TLogCompressionType getLogCompressionType() { + return logCompressionType; + } + + public int getLogCompressionMinSize() { + return logCompressionMinSize; + } + public int getMaxPendingBatchesNum() { return maxPendingBatchesNum; } @@ -440,6 +456,8 @@ public static class Builder { private int maxLogEntriesNumPerBatch = 1024; private int maxSizePerBatch = 16 * 1024 * 1024; + private TLogCompressionType logCompressionType = TLogCompressionType.NONE; + private int logCompressionMinSize = 4096; private int maxPendingBatchesNum = 5; private int maxQueueLength = 4096; private long maxWaitingTimeForWaitBatchInMs = 10 * 1000L; @@ -473,6 +491,23 @@ public Builder setMaxSizePerBatch(int maxSizePerBatch) { return this; } + public Builder setLogCompressionType(TLogCompressionType type) { + this.logCompressionType = type; + return this; + } + + public Builder setLogCompressionMinSize(int size) { + if (size < 0) { + throw new IllegalArgumentException( + String.format( + IoTConsensusMessages + .EXCEPTION_INVALID_IOTCONSENSUS_LOG_COMPRESSION_MINIMUM_SIZE_ARG_ACEEA255, + size)); + } + this.logCompressionMinSize = size; + return this; + } + public Replication.Builder setMaxPendingBatchesNum(int maxPendingBatchesNum) { this.maxPendingBatchesNum = maxPendingBatchesNum; return this; @@ -557,6 +592,8 @@ public Replication build() { return new Replication( maxLogEntriesNumPerBatch, maxSizePerBatch, + logCompressionType, + logCompressionMinSize, maxPendingBatchesNum, maxQueueLength, maxWaitingTimeForWaitBatchInMs, diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerMetrics.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerMetrics.java index 117f971263572..08714ff2ee40a 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerMetrics.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerMetrics.java @@ -37,6 +37,7 @@ public class IoTConsensusServerMetrics implements IMetricSet { private Timer offerRequestToQueueTimer = DoNothingMetricManager.DO_NOTHING_TIMER; private Timer consensusWriteTimer = DoNothingMetricManager.DO_NOTHING_TIMER; private static final String IOT_RECEIVE_LOG = Metric.IOT_RECEIVE_LOG.toString(); + private Timer decompressTimer = DoNothingMetricManager.DO_NOTHING_TIMER; private Timer deserializeTimer = DoNothingMetricManager.DO_NOTHING_TIMER; private Timer sortTimer = DoNothingMetricManager.DO_NOTHING_TIMER; private Timer applyTimer = DoNothingMetricManager.DO_NOTHING_TIMER; @@ -50,6 +51,10 @@ public IoTConsensusServerMetrics(IoTConsensusServerImpl impl) { private static final String IMPL = "ioTConsensusServerImpl"; + public void recordDecompressCost(long nanos) { + decompressTimer.updateNanos(nanos); + } + public void recordDeserializeCost(long costTimeInNanos) { deserializeTimer.updateNanos(costTimeInNanos); } @@ -190,6 +195,14 @@ private void bindStageTimer(AbstractMetricService metricService) { private void bindSyncLogTimer(AbstractMetricService metricService) { // bind sync log timers + decompressTimer = + metricService.getOrCreateTimer( + IOT_RECEIVE_LOG, + MetricLevel.IMPORTANT, + Tag.STAGE.toString(), + "decompress", + Tag.REGION.toString(), + impl.getConsensusGroupId()); deserializeTimer = metricService.getOrCreateTimer( IOT_RECEIVE_LOG, @@ -319,6 +332,13 @@ private void unbindStageTimer(AbstractMetricService metricService) { private void unbindSyncLogTimer(AbstractMetricService metricService) { // unbind sync log timers + metricService.remove( + MetricType.TIMER, + IOT_RECEIVE_LOG, + Tag.STAGE.toString(), + "decompress", + Tag.REGION.toString(), + impl.getConsensusGroupId()); metricService.remove( MetricType.TIMER, IOT_RECEIVE_LOG, diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/client/DispatchLogHandler.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/client/DispatchLogHandler.java index 615ecd6b07c46..fe264ef681f54 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/client/DispatchLogHandler.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/client/DispatchLogHandler.java @@ -39,6 +39,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; import java.util.List; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -53,6 +54,7 @@ public class DispatchLogHandler implements AsyncMethodCallback RetryUtils.needRetryForWrite(status.getCode()))) { @@ -138,7 +149,7 @@ public void onError(Exception exception) { actualCause.toString()); } // skip TApplicationException caused by follower - if (actualCause instanceof TApplicationException) { + if (actualCause instanceof TApplicationException && !hasCompressedAttempt) { completeBatch(batch); LOGGER.warn(IoTConsensusMessages.SKIP_RETRY_TAPPLICATION_EXCEPTION, batch); logDispatcherThreadMetrics.recordSyncLogTimePerRequest(System.nanoTime() - createTime); diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/IoTConsensusMemoryManager.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/IoTConsensusMemoryManager.java index 4d57b32e50699..c1a1d4277d3ff 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/IoTConsensusMemoryManager.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/IoTConsensusMemoryManager.java @@ -129,6 +129,15 @@ public void free(IndexedConsensusRequest request) { } } + /** Non-blocking reservation for the additional arrays held by an encoded RPC attempt. */ + public boolean reserveLogCompression(long size) { + return reserve(size, false); + } + + public void freeLogCompression(long size) { + free(size, false); + } + public void free(Batch batch) { free(batch.getMemorySize(), false); if (logger.isDebugEnabled()) { diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompression.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompression.java new file mode 100644 index 0000000000000..1162873f2c4de --- /dev/null +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompression.java @@ -0,0 +1,332 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.consensus.iot.logdispatcher; + +import org.apache.iotdb.consensus.i18n.IoTConsensusMessages; +import org.apache.iotdb.consensus.iot.thrift.TLogBufferCompression; +import org.apache.iotdb.consensus.iot.thrift.TLogCompressionType; +import org.apache.iotdb.consensus.iot.thrift.TLogEntry; +import org.apache.iotdb.consensus.iot.thrift.TSyncLogEntriesReq; + +import com.github.luben.zstd.Zstd; +import com.github.luben.zstd.ZstdException; +import org.apache.tsfile.compress.ICompressor; +import org.apache.tsfile.compress.IUnCompressor; +import org.apache.tsfile.file.metadata.enums.CompressionType; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; +import java.util.zip.CRC32C; + +/** Wire-only encoding: the original batch and its buffer positions remain available for retries. */ +public final class LogCompression { + // Conservative allowance for the additional Thrift descriptor and list field on the wire. + private static final int DESCRIPTOR_BYTES = 32; + private static final ICompressor LZ4 = ICompressor.getCompressor(CompressionType.LZ4); + private static final ICompressor SNAPPY = ICompressor.getCompressor(CompressionType.SNAPPY); + private static final IUnCompressor LZ4_DECODER = + IUnCompressor.getUnCompressor(CompressionType.LZ4); + private static final IUnCompressor SNAPPY_DECODER = + IUnCompressor.getUnCompressor(CompressionType.SNAPPY); + private static final IUnCompressor ZSTD_DECODER = + IUnCompressor.getUnCompressor(CompressionType.ZSTD); + + private LogCompression() {} + + /** Accounts for arrays throughout the async attempt, including synchronous send failure. */ + public static final class EncodedRequest implements AutoCloseable { + private TSyncLogEntriesReq request; + private final AtomicLong reserved; + private final long rawBytes; + private long encodedBytes; + private boolean memoryLimited; + + private EncodedRequest( + TSyncLogEntriesReq request, long reserved, long rawBytes, boolean memoryLimited) { + this.request = request; + this.reserved = new AtomicLong(reserved); + this.rawBytes = rawBytes; + this.encodedBytes = rawBytes; + this.memoryLimited = memoryLimited; + } + + public long getRawBytes() { + return rawBytes; + } + + public long getEncodedBytes() { + return encodedBytes; + } + + public boolean isMemoryLimited() { + return memoryLimited; + } + + public TSyncLogEntriesReq getRequest() { + return request; + } + + private void retain(long bytes) { + long released = reserved.getAndSet(bytes) - bytes; + if (released > 0) IoTConsensusMemoryManager.getInstance().freeLogCompression(released); + } + + @Override + public void close() { + long bytes = reserved.getAndSet(0); + if (bytes > 0) IoTConsensusMemoryManager.getInstance().freeLogCompression(bytes); + } + } + + public static EncodedRequest compress( + TSyncLogEntriesReq original, TLogCompressionType codec, int minSize, int maxDecodedBytes) + throws IOException { + long buffers = 0, rawSize = 0; + boolean hasEligibleBuffer = false; + for (TLogEntry entry : original.getLogEntries()) { + for (ByteBuffer buffer : entry.getData()) { + int size = buffer.remaining(); + rawSize += size; + if (size >= minSize && size > 0) { + hasEligibleBuffer = true; + } + buffers++; + } + } + // Preserve the original path for oversized legal entries and when scratch memory is scarce. + if (codec == TLogCompressionType.NONE || !hasEligibleBuffer || rawSize > maxDecodedBytes) + return new EncodedRequest(original, 0, rawSize, false); + // Charge descriptor/list/entry overhead first, then one block's working arrays at a time. + // Retain only actual compact outputs between blocks, not a whole-batch worst-case estimate. + long budget = buffers * 256 + original.getLogEntriesSize() * 128L; + if (!IoTConsensusMemoryManager.getInstance().reserveLogCompression(budget)) { + return new EncodedRequest(original, 0, rawSize, true); + } + EncodedRequest result = new EncodedRequest(original, budget, rawSize, false); + try { + List entries = new ArrayList<>(original.getLogEntriesSize()); + long retained = 0; + boolean changed = false; + for (TLogEntry entry : original.getLogEntries()) { + List data = new ArrayList<>(entry.getDataSize()); + List info = new ArrayList<>(entry.getDataSize()); + long raw = 0, wire = 0, arrays = 0; + boolean compressed = false; + for (ByteBuffer buffer : entry.getData()) { + int size = buffer.remaining(); + raw += size; + ByteBuffer encoded = buffer; + TLogCompressionType actualCodec = TLogCompressionType.NONE; + if (size >= minSize && size > 0) { + // Includes an optional input copy, codec-bound output and compact output, with + // allowance for array headers and codec bounds on very small buffers. + long workspace = 4L * size + 256; + long before = result.reserved.get(); + if (IoTConsensusMemoryManager.getInstance().reserveLogCompression(workspace)) { + result.reserved.addAndGet(workspace); + byte[] input = buffer.hasArray() ? buffer.array() : bytes(buffer); + int offset = buffer.hasArray() ? buffer.arrayOffset() + buffer.position() : 0; + int bound = + codec == TLogCompressionType.ZSTD + ? (int) Zstd.compressBound(size) + : compressor(codec).getMaxBytesForCompression(size); + byte[] output = new byte[bound]; + int length = + codec == TLogCompressionType.ZSTD + ? (int) + Zstd.compressByteArray(output, 0, output.length, input, offset, size, 1) + : compressor(codec).compress(input, offset, size, output); + if (length < 0) throw invalid(Failure.CODEC, size, length); + if ((long) length + DESCRIPTOR_BYTES < size) { + encoded = ByteBuffer.wrap(Arrays.copyOf(output, length)); + actualCodec = codec; + arrays += length; + compressed = true; + } + result.retain( + before + (actualCodec == TLogCompressionType.NONE ? 0 : encoded.remaining())); + } else { + // Keep this block raw; other smaller blocks in the entry may still fit. + result.memoryLimited = true; + } + } + data.add(encoded); + info.add(new TLogBufferCompression(actualCodec, size, 0)); + wire += encoded.remaining(); + } + // Raw descriptors in a mixed WAL entry cost space too. + if (compressed && wire + DESCRIPTOR_BYTES * (long) data.size() + 8 < raw) { + for (int i = 0; i < info.size(); i++) + info.get(i).setChecksum(checksum(entry.getData().get(i))); + entries.add(copy(entry, data).setCompressionInfo(info)); + retained += arrays + data.size() * 256L + 128; + result.encodedBytes += wire - raw; + changed = true; + } else { + entries.add(entry); + result.retain(result.reserved.get() - arrays); + } + } + if (changed) + result.request = + new TSyncLogEntriesReq(original.getPeerId(), original.getConsensusGroupId(), entries); + result.retain(retained); + return result; + } catch (IOException | RuntimeException e) { + result.close(); + throw e; + } + } + + /** Validate every declared size before allocating; apply nothing until all buffers pass CRC. */ + public static void decompress(TSyncLogEntriesReq request, int maxDecodedBytes) + throws IOException { + boolean encoded = false; + long total = 0; + for (TLogEntry entry : request.getLogEntries()) { + if (entry.isSetCompressionInfo()) { + encoded = true; + if (entry.getCompressionInfoSize() != entry.getDataSize()) + throw invalid(Failure.DESCRIPTORS, entry.getDataSize(), entry.getCompressionInfoSize()); + for (int i = 0; i < entry.getDataSize(); i++) { + TLogBufferCompression info = entry.getCompressionInfo().get(i); + int size = info.getUncompressedSize(); + if (size < 0) throw invalid(Failure.LENGTH, 0, size); + if (info.getCodec() == null) throw invalid(Failure.CODEC, 0, -1); + if (info.getCodec() == TLogCompressionType.NONE + && size != entry.getData().get(i).remaining()) + throw invalid(Failure.LENGTH, size, entry.getData().get(i).remaining()); + total += size; + if (total > maxDecodedBytes) throw invalid(Failure.LIMIT, maxDecodedBytes, total); + } + } else { + for (ByteBuffer buffer : entry.getData()) total += buffer.remaining(); + } + } + if (!encoded) return; + if (total > maxDecodedBytes) throw invalid(Failure.LIMIT, maxDecodedBytes, total); + List decoded = new ArrayList<>(request.getLogEntriesSize()); + for (TLogEntry entry : request.getLogEntries()) { + if (!entry.isSetCompressionInfo()) { + decoded.add(entry); + continue; + } + List data = new ArrayList<>(entry.getDataSize()); + for (int i = 0; i < entry.getDataSize(); i++) { + TLogBufferCompression info = entry.getCompressionInfo().get(i); + ByteBuffer buffer = entry.getData().get(i); + if (info.getCodec() != TLogCompressionType.NONE) { + byte[] input = buffer.hasArray() ? buffer.array() : bytes(buffer); + int offset = buffer.hasArray() ? buffer.arrayOffset() + buffer.position() : 0; + byte[] output = new byte[info.getUncompressedSize()]; + int actual; + try { + actual = + decoder(info.getCodec()).uncompress(input, offset, buffer.remaining(), output, 0); + } catch (ZstdException e) { + IOException failure = invalid(Failure.CODEC, output.length, buffer.remaining()); + failure.initCause(e); + throw failure; + } + if (actual != output.length) throw invalid(Failure.LENGTH, output.length, actual); + buffer = ByteBuffer.wrap(output); + } + int crc = checksum(buffer); + if (crc != info.getChecksum()) + throw invalid( + Failure.CHECKSUM, + Integer.toUnsignedLong(info.getChecksum()), + Integer.toUnsignedLong(crc)); + data.add(buffer); + } + decoded.add(copy(entry, data)); + } + // The RPC exclusively owns this request. Dropping the encoded entries also drops their arrays. + // No partially decoded request is exposed on failure. + request.setLogEntries(decoded); + } + + private static TLogEntry copy(TLogEntry source, List data) { + TLogEntry entry = + new TLogEntry(data, source.getSearchIndex(), source.isFromWAL(), source.getMemorySize()); + if (source.isSetRoutingEpoch()) entry.setRoutingEpoch(source.getRoutingEpoch()); + if (source.isSetPhysicalTime()) entry.setPhysicalTime(source.getPhysicalTime()); + return entry; + } + + private static byte[] bytes(ByteBuffer buffer) { + ByteBuffer copy = buffer.duplicate(); + byte[] bytes = new byte[copy.remaining()]; + copy.get(bytes); + return bytes; + } + + private static int checksum(ByteBuffer buffer) { + CRC32C crc = new CRC32C(); + crc.update(buffer.duplicate()); + return (int) crc.getValue(); + } + + private static ICompressor compressor(TLogCompressionType codec) throws IOException { + switch (codec) { + case LZ4: + return LZ4; + case SNAPPY: + return SNAPPY; + default: + throw invalid(Failure.CODEC, 0, codec.getValue()); + } + } + + private static IUnCompressor decoder(TLogCompressionType codec) throws IOException { + switch (codec) { + case LZ4: + return LZ4_DECODER; + case SNAPPY: + return SNAPPY_DECODER; + case ZSTD: + return ZSTD_DECODER; + default: + throw invalid(Failure.CODEC, 0, codec.getValue()); + } + } + + private enum Failure { + CODEC, + LENGTH, + LIMIT, + DESCRIPTORS, + CHECKSUM + } + + private static IOException invalid(Failure failure, long expected, long actual) { + return new IOException( + String.format( + IoTConsensusMessages + .EXCEPTION_INVALID_IOTCONSENSUS_COMPRESSED_LOG_ARG_EXPECTED_ARG_ACTUAL_ARG_4986175E, + failure.name(), + expected, + actual)); + } +} diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcher.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcher.java index 0e4f41b9d8412..7f622c000da5e 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcher.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcher.java @@ -35,9 +35,12 @@ import org.apache.iotdb.consensus.iot.client.DispatchLogHandler; import org.apache.iotdb.consensus.iot.log.ConsensusReqReader; import org.apache.iotdb.consensus.iot.log.GetConsensusReqReaderPlan; +import org.apache.iotdb.consensus.iot.thrift.TLogCompressionType; import org.apache.iotdb.consensus.iot.thrift.TLogEntry; import org.apache.iotdb.consensus.iot.thrift.TSyncLogEntriesReq; +import org.apache.iotdb.consensus.iot.thrift.TSyncLogEntriesRes; +import org.apache.thrift.async.AsyncMethodCallback; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -587,8 +590,8 @@ private boolean shouldSendIdleWriterSafeTimeBarrier() { } public void sendBatchAsync(Batch batch, DispatchLogHandler handler) { + LogCompression.EncodedRequest encoded = null; try { - AsyncIoTConsensusServiceClient client = clientManager.borrowClient(peer.getEndpoint()); TSyncLogEntriesReq req = new TSyncLogEntriesReq( selfPeerId, peer.getGroupId().convertToTConsensusGroupId(), batch.getLogEntries()); @@ -597,8 +600,46 @@ public void sendBatchAsync(Batch batch, DispatchLogHandler handler) { batch.getStartIndex(), batch.getEndIndex(), peer.getGroupId().convertToTConsensusGroupId()); - client.syncLogEntries(req, handler); + if (config.getReplication().getLogCompressionType() == TLogCompressionType.NONE) { + clientManager.borrowClient(peer.getEndpoint()).syncLogEntries(req, handler); + return; + } + long compressionStart = System.nanoTime(); + encoded = + LogCompression.compress( + req, + config.getReplication().getLogCompressionType(), + config.getReplication().getLogCompressionMinSize(), + config.getRpc().getThriftMaxFrameSize()); + logDispatcherThreadMetrics.recordCompressTime(System.nanoTime() - compressionStart); + logDispatcherThreadMetrics.recordCompressionBytes( + encoded.getRawBytes(), encoded.getEncodedBytes(), encoded.isMemoryLimited()); + if (encoded.getRequest() == req) { + encoded.close(); + clientManager.borrowClient(peer.getEndpoint()).syncLogEntries(req, handler); + return; + } + handler.markCompressedAttempt(); + final LogCompression.EncodedRequest attempt = encoded; + clientManager + .borrowClient(peer.getEndpoint()) + .syncLogEntries( + attempt.getRequest(), + new AsyncMethodCallback() { + @Override + public void onComplete(TSyncLogEntriesRes response) { + attempt.close(); + handler.onComplete(response); + } + + @Override + public void onError(Exception exception) { + attempt.close(); + handler.onError(exception); + } + }); } catch (Exception e) { + if (encoded != null) encoded.close(); logger.error(IoTConsensusMessages.CANNOT_SYNC_LOGS_TO_PEER, peer, e); handler.onError(e); } diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcherThreadMetrics.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcherThreadMetrics.java index 9a5dd6c4b28ec..501979fbeb7e8 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcherThreadMetrics.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcherThreadMetrics.java @@ -24,6 +24,7 @@ import org.apache.iotdb.metrics.AbstractMetricService; import org.apache.iotdb.metrics.impl.DoNothingMetricManager; import org.apache.iotdb.metrics.metricsets.IMetricSet; +import org.apache.iotdb.metrics.type.Counter; import org.apache.iotdb.metrics.type.Timer; import org.apache.iotdb.metrics.utils.MetricLevel; import org.apache.iotdb.metrics.utils.MetricType; @@ -45,6 +46,10 @@ public class LogDispatcherThreadMetrics implements IMetricSet { private final LogDispatcher.LogDispatcherThread logDispatcherThread; private final String peerGroupId; + private Counter rawBytes = DoNothingMetricManager.DO_NOTHING_COUNTER; + private Counter encodedBytes = DoNothingMetricManager.DO_NOTHING_COUNTER; + private Counter memoryFallbacks = DoNothingMetricManager.DO_NOTHING_COUNTER; + private Timer compressTimer = DoNothingMetricManager.DO_NOTHING_TIMER; private Timer constructBatchTimer = DoNothingMetricManager.DO_NOTHING_TIMER; private Timer syncLogTimePerRequestTimer = DoNothingMetricManager.DO_NOTHING_TIMER; @@ -53,6 +58,16 @@ public LogDispatcherThreadMetrics(LogDispatcher.LogDispatcherThread logDispatche this.peerGroupId = logDispatcherThread.getPeer().getGroupId().toString(); } + public void recordCompressionBytes(long raw, long encoded, boolean memoryLimited) { + rawBytes.inc(raw); + encodedBytes.inc(encoded); + if (memoryLimited) memoryFallbacks.inc(); + } + + public void recordCompressTime(long nanos) { + compressTimer.updateNanos(nanos); + } + public void recordConstructBatchTime(long costTimeInNanos) { constructBatchTimer.updateNanos(costTimeInNanos); } @@ -108,6 +123,38 @@ private void bindStageTimer(AbstractMetricService metricService) { REGION_TO_STAGE_TIMER_OWNERS .computeIfAbsent(peerGroupId, region -> new HashSet<>()) .add(this); + rawBytes = + metricService.getOrCreateCounter( + Metric.IOT_LOG_COMPRESSION_BYTES.toString(), + MetricLevel.IMPORTANT, + Tag.REGION.toString(), + peerGroupId, + Tag.TYPE.toString(), + "raw"); + encodedBytes = + metricService.getOrCreateCounter( + Metric.IOT_LOG_COMPRESSION_BYTES.toString(), + MetricLevel.IMPORTANT, + Tag.REGION.toString(), + peerGroupId, + Tag.TYPE.toString(), + "encoded"); + memoryFallbacks = + metricService.getOrCreateCounter( + Metric.IOT_LOG_COMPRESSION_MEMORY_FALLBACKS.toString(), + MetricLevel.IMPORTANT, + Tag.REGION.toString(), + peerGroupId); + compressTimer = + metricService.getOrCreateTimer( + Metric.IOT_SEND_LOG.toString(), + MetricLevel.IMPORTANT, + Tag.NAME.toString(), + Metric.IOT_CONSENSUS.toString(), + Tag.STAGE.toString(), + "compress", + Tag.REGION.toString(), + peerGroupId); constructBatchTimer = metricService.getOrCreateTimer( Metric.IOT_SEND_LOG.toString(), @@ -140,6 +187,34 @@ private void unbindStageTimer(AbstractMetricService metricService) { return; } REGION_TO_STAGE_TIMER_OWNERS.remove(peerGroupId); + metricService.remove( + MetricType.COUNTER, + Metric.IOT_LOG_COMPRESSION_BYTES.toString(), + Tag.REGION.toString(), + peerGroupId, + Tag.TYPE.toString(), + "raw"); + metricService.remove( + MetricType.COUNTER, + Metric.IOT_LOG_COMPRESSION_BYTES.toString(), + Tag.REGION.toString(), + peerGroupId, + Tag.TYPE.toString(), + "encoded"); + metricService.remove( + MetricType.COUNTER, + Metric.IOT_LOG_COMPRESSION_MEMORY_FALLBACKS.toString(), + Tag.REGION.toString(), + peerGroupId); + metricService.remove( + MetricType.TIMER, + Metric.IOT_SEND_LOG.toString(), + Tag.NAME.toString(), + Metric.IOT_CONSENSUS.toString(), + Tag.STAGE.toString(), + "compress", + Tag.REGION.toString(), + peerGroupId); } metricService.remove( MetricType.TIMER, diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/service/IoTConsensusRPCServiceProcessor.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/service/IoTConsensusRPCServiceProcessor.java index 6db087f7fe283..503e5e6ab7175 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/service/IoTConsensusRPCServiceProcessor.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/service/IoTConsensusRPCServiceProcessor.java @@ -35,6 +35,7 @@ import org.apache.iotdb.consensus.i18n.IoTConsensusV2Messages; import org.apache.iotdb.consensus.iot.IoTConsensus; import org.apache.iotdb.consensus.iot.IoTConsensusServerImpl; +import org.apache.iotdb.consensus.iot.logdispatcher.LogCompression; import org.apache.iotdb.consensus.iot.thrift.IoTConsensusIService; import org.apache.iotdb.consensus.iot.thrift.TActivatePeerReq; import org.apache.iotdb.consensus.iot.thrift.TActivatePeerRes; @@ -65,6 +66,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; import java.util.Collections; import java.util.stream.Collectors; @@ -110,6 +112,18 @@ public TSyncLogEntriesRes syncLogEntries(TSyncLogEntriesReq req) { status.setMessage(message); return new TSyncLogEntriesRes(Collections.singletonList(status)); } + long decompressStart = System.nanoTime(); + try { + // Every block is checked before constructing requests or entering the sync/apply path. + LogCompression.decompress(req, impl.getConfig().getRpc().getThriftMaxFrameSize()); + } catch (IOException | IllegalArgumentException e) { + TSStatus status = new TSStatus(TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode()); + status.setMessage(e.getMessage()); + return new TSyncLogEntriesRes(Collections.singletonList(status)) + .setLogCompressionError(status); + } finally { + impl.getIoTConsensusServerMetrics().recordDecompressCost(System.nanoTime() - decompressStart); + } BatchIndexedConsensusRequest logEntriesInThisBatch = new BatchIndexedConsensusRequest(req.peerId); final int sourceNodeId = req.peerId; diff --git a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/CompressedReplicationTest.java b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/CompressedReplicationTest.java new file mode 100644 index 0000000000000..f01178d24776d --- /dev/null +++ b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/CompressedReplicationTest.java @@ -0,0 +1,178 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.consensus.iot; + +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.consensus.DataRegionId; +import org.apache.iotdb.commons.request.IConsensusRequest; +import org.apache.iotdb.consensus.common.Peer; +import org.apache.iotdb.consensus.common.request.BatchIndexedConsensusRequest; +import org.apache.iotdb.consensus.common.request.IndexedConsensusRequest; +import org.apache.iotdb.consensus.config.ConsensusConfig; +import org.apache.iotdb.consensus.config.IoTConsensusConfig; +import org.apache.iotdb.consensus.iot.thrift.TLogCompressionType; +import org.apache.iotdb.consensus.iot.util.TestEntry; +import org.apache.iotdb.consensus.iot.util.TestStateMachine; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.net.ServerSocket; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +/** Real async Thrift in both directions, using the existing state machine/reader fixture. */ +@RunWith(Parameterized.class) +public class CompressedReplicationTest { + @Rule public TemporaryFolder directory = new TemporaryFolder(); + + @Parameterized.Parameters(name = "{0}") + public static Object[] codecs() { + return new Object[] { + TLogCompressionType.LZ4, TLogCompressionType.SNAPPY, TLogCompressionType.ZSTD + }; + } + + @Parameterized.Parameter public TLogCompressionType codec; + private static final int COUNT = 32; + private static final int BYTES = 65536; + + @Test + public void testBidirectionalReplicationPreservesFullPayloadAndMetadata() throws Exception { + DataRegionId group = new DataRegionId(1); + int firstPort, secondPort; + try (ServerSocket first = new ServerSocket(0); + ServerSocket second = new ServerSocket(0)) { + firstPort = first.getLocalPort(); + secondPort = second.getLocalPort(); + } + List peers = + Arrays.asList( + new Peer(group, 1, new TEndPoint("127.0.0.1", firstPort)), + new Peer(group, 2, new TEndPoint("127.0.0.1", secondPort))); + IoTConsensus[] servers = new IoTConsensus[2]; + ObservedStateMachine[] machines = {new ObservedStateMachine(), new ObservedStateMachine()}; + try { + for (int i = 0; i < 2; i++) { + final int node = i; + ConsensusConfig config = + ConsensusConfig.newBuilder() + .setThisNodeId(i + 1) + .setThisNode(peers.get(i).getEndpoint()) + .setConsensusGroupType(TConsensusGroupType.DataRegion) + .setStorageDir(directory.newFolder().getAbsolutePath()) + .setRecvSnapshotDirs( + Collections.singletonList(directory.newFolder().getAbsolutePath())) + .setIoTConsensusConfig( + IoTConsensusConfig.newBuilder() + .setReplication( + IoTConsensusConfig.Replication.newBuilder() + .setLogCompressionType(codec) + .setMaxWaitingTimeForWaitBatchInMs(100) + .build()) + .build()) + .build(); + servers[i] = new IoTConsensus(config, ignored -> machines[node]); + servers[i].recordCorrectPeerListBeforeStarting(Collections.singletonMap(group, peers)); + servers[i].start(); + } + for (IoTConsensus server : servers) server.createLocalPeer(group, peers); + for (int j = 0; j < COUNT; j++) { + for (int i = 0; i < 2; i++) { + final TestEntry entry = + new TestEntry(j, peers.get(i)) { + @Override + public ByteBuffer serializeToByteBuffer() { + ByteBuffer result = ByteBuffer.allocate(BYTES); + result.put(super.serializeToByteBuffer()); + // A repeated nonzero tail makes corruption visible beyond the parsed TestEntry. + while (result.hasRemaining()) result.put((byte) 42); + result.flip(); + return result; + } + }; + assertEquals(200, servers[i].write(group, entry).getCode()); + } + } + for (int i = 0; i < 2; i++) { + assertTrue( + "remote requests must all be applied", machines[i].applied.await(30, TimeUnit.SECONDS)); + assertNull(machines[i].failure.get()); + assertEquals(COUNT * 2, machines[i].getData().size()); + for (Peer peer : peers) + for (int j = 0; j < COUNT; j++) + assertTrue(machines[i].getData().contains(new TestEntry(j, peer))); + } + } finally { + for (IoTConsensus server : servers) if (server != null) server.stop(); + } + } + + private static class ObservedStateMachine extends TestStateMachine { + private final CountDownLatch applied = new CountDownLatch(COUNT); + private final AtomicReference failure = new AtomicReference<>(); + + @Override + public IConsensusRequest deserializeRequest(IConsensusRequest request) { + if (request instanceof BatchIndexedConsensusRequest) { + try { + for (IndexedConsensusRequest indexed : + ((BatchIndexedConsensusRequest) request).getRequests()) { + assertTrue(indexed.getPhysicalTime() > 0); + for (IConsensusRequest inner : indexed.getRequests()) { + ByteBuffer buffer = inner.serializeToByteBuffer().duplicate(); + assertEquals(BYTES, buffer.remaining()); + buffer.getInt(); + Peer.deserialize(buffer); + while (buffer.hasRemaining()) assertEquals(42, buffer.get()); + } + } + } catch (AssertionError e) { + failure.compareAndSet(null, e); + } + } + return super.deserializeRequest(request); + } + + @Override + public TSStatus write(IConsensusRequest request) { + TSStatus result = super.write(request); + if (request instanceof IndexedConsensusRequest + && ((IndexedConsensusRequest) request).getSearchIndex() == -1) { + applied.countDown(); + } + return result; + } + } +} diff --git a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/client/DispatchLogHandlerTest.java b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/client/DispatchLogHandlerTest.java index 13736903902a9..224d5dfd92fab 100644 --- a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/client/DispatchLogHandlerTest.java +++ b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/client/DispatchLogHandlerTest.java @@ -24,21 +24,42 @@ import org.apache.iotdb.commons.audit.UserDataTransferAuditEvent; import org.apache.iotdb.commons.audit.UserDataTransferAuditHandler; import org.apache.iotdb.commons.audit.UserDataTransferProtectionMethod; +import org.apache.iotdb.commons.consensus.DataRegionId; +import org.apache.iotdb.consensus.common.Peer; import org.apache.iotdb.consensus.config.IoTConsensusConfig; +import org.apache.iotdb.consensus.iot.IoTConsensusServerImpl; import org.apache.iotdb.consensus.iot.logdispatcher.Batch; +import org.apache.iotdb.consensus.iot.logdispatcher.IndexController; +import org.apache.iotdb.consensus.iot.logdispatcher.LogDispatcher.LogDispatcherThread; +import org.apache.iotdb.consensus.iot.logdispatcher.LogDispatcherThreadMetrics; +import org.apache.iotdb.consensus.iot.logdispatcher.SyncStatus; import org.apache.iotdb.consensus.iot.thrift.TLogEntry; import org.apache.iotdb.consensus.iot.thrift.TSyncLogEntriesRes; +import org.apache.thrift.TApplicationException; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.mockito.ArgumentCaptor; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.same; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; public class DispatchLogHandlerTest { @@ -164,6 +185,74 @@ public boolean isEnabled() { null); } + @Rule public TemporaryFolder directory = new TemporaryFolder(); + + @Test + public void testCompressedProtocolFailureRetriesWithoutAdvancingIndex() throws Exception { + verifyCompressionFailure(true); + } + + @Test + public void testExplicitDecodeErrorOverridesNonRetryStatus() throws Exception { + verifyCompressionFailure(false); + } + + private void verifyCompressionFailure(boolean protocolError) throws Exception { + IoTConsensusConfig config = IoTConsensusConfig.newBuilder().build(); + Peer peer = new Peer(new DataRegionId(1), 2, TARGET); + IndexController controller = + new IndexController(directory.newFolder().getAbsolutePath(), peer, 0, 500); + SyncStatus status = new SyncStatus(controller, config); + Batch batch = createBatch(false); + LogDispatcherThread thread = mock(LogDispatcherThread.class); + IoTConsensusServerImpl impl = mock(IoTConsensusServerImpl.class); + ScheduledExecutorService scheduler = mock(ScheduledExecutorService.class); + when(thread.getConfig()).thenReturn(config); + when(thread.getPeer()).thenReturn(peer); + when(thread.getSyncStatus()).thenReturn(status); + when(thread.getImpl()).thenReturn(impl); + when(impl.getBackgroundTaskService()).thenReturn(scheduler); + when(impl.getUserDataTransferAuditHandler()).thenReturn(UserDataTransferAuditHandler.NO_OP); + status.addNextBatch(batch); + try { + DispatchLogHandler handler = + new DispatchLogHandler(thread, mock(LogDispatcherThreadMetrics.class), batch); + if (protocolError) { + handler.markCompressedAttempt(); + handler.onError(new TApplicationException(TApplicationException.PROTOCOL_ERROR)); + } else { + TSStatus error = new TSStatus(200).setMessage("decode failed"); + handler.onComplete( + new TSyncLogEntriesRes(Collections.singletonList(error)).setLogCompressionError(error)); + } + assertEquals(0, controller.getCurrentIndex()); + assertFalse(batch.isSynced()); + assertTrue(status.hasPendingBatches()); + // A later in-flight batch may ACK first. Its success must not advance past the rejected + // batch or release either raw batch before the missing prefix has succeeded. + Batch later = new Batch(config); + later.addTLogEntry(new TLogEntry().setSearchIndex(2).setMemorySize(1)); + later.buildIndex(); + status.addNextBatch(later); + DispatchLogHandler laterHandler = + new DispatchLogHandler(thread, mock(LogDispatcherThreadMetrics.class), later); + laterHandler.onComplete(new TSyncLogEntriesRes(Collections.singletonList(new TSStatus(200)))); + assertTrue(later.isSynced()); + assertEquals(0, controller.getCurrentIndex()); + assertEquals(2, status.getPendingBatches().size()); + ArgumentCaptor retry = ArgumentCaptor.forClass(Runnable.class); + verify(scheduler).schedule(retry.capture(), anyLong(), eq(TimeUnit.MILLISECONDS)); + retry.getValue().run(); + verify(thread).sendBatchAsync(same(batch), same(handler)); + handler.onComplete(new TSyncLogEntriesRes(Collections.singletonList(new TSStatus(200)))); + assertEquals(2, controller.getCurrentIndex()); + assertFalse(status.hasPendingBatches()); + verify(thread, times(2)).updateSafelyDeletedSearchIndex(); + } finally { + status.free(); + } + } + private static Batch createBatch(boolean containsUserData) { final Batch batch = new Batch(IoTConsensusConfig.newBuilder().build()); batch.addTLogEntry(new TLogEntry().setSearchIndex(1).setMemorySize(1), containsUserData); diff --git a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompressionTest.java b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompressionTest.java new file mode 100644 index 0000000000000..09eabafb68198 --- /dev/null +++ b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompressionTest.java @@ -0,0 +1,364 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.consensus.iot.logdispatcher; + +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.consensus.DataRegionId; +import org.apache.iotdb.commons.memory.AtomicLongMemoryBlock; +import org.apache.iotdb.commons.memory.IMemoryBlock; +import org.apache.iotdb.consensus.IStateMachine; +import org.apache.iotdb.consensus.common.request.ByteBufferConsensusRequest; +import org.apache.iotdb.consensus.common.request.IndexedConsensusRequest; +import org.apache.iotdb.consensus.config.IoTConsensusConfig; +import org.apache.iotdb.consensus.iot.IoTConsensus; +import org.apache.iotdb.consensus.iot.IoTConsensusServerImpl; +import org.apache.iotdb.consensus.iot.IoTConsensusServerMetrics; +import org.apache.iotdb.consensus.iot.service.IoTConsensusRPCServiceProcessor; +import org.apache.iotdb.consensus.iot.thrift.TLogCompressionType; +import org.apache.iotdb.consensus.iot.thrift.TLogEntry; +import org.apache.iotdb.consensus.iot.thrift.TSyncLogEntriesReq; +import org.apache.iotdb.consensus.iot.thrift.TSyncLogEntriesRes; + +import org.apache.thrift.TDeserializer; +import org.apache.thrift.TSerializer; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Random; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@RunWith(Parameterized.class) +public class LogCompressionTest { + @Parameterized.Parameters(name = "{0}") + public static Object[] codecs() { + return new Object[] { + TLogCompressionType.LZ4, TLogCompressionType.SNAPPY, TLogCompressionType.ZSTD + }; + } + + @Parameterized.Parameter public TLogCompressionType codec; + private IMemoryBlock previous; + private final IoTConsensusMemoryManager memory = IoTConsensusMemoryManager.getInstance(); + private static final int LIMIT = 8 * 1024 * 1024; + + @Before + public void setUp() { + previous = memory.getMemoryBlock(); + memory.setMemoryBlock(new AtomicLongMemoryBlock("compression-test", null, 64L * 1024 * 1024)); + memory.reset(); + } + + @After + public void tearDown() { + assertEquals(0, memory.getMemoryBlock().getUsedMemoryInBytes()); + memory.reset(); + memory.setMemoryBlock(previous); + } + + private TSyncLogEntriesReq request() { + byte[] compressible = new byte[16384]; + Arrays.fill(compressible, (byte) 42); + for (int i = 0; i < 17; i++) compressible[i] = (byte) i; + ByteBuffer sliced = ByteBuffer.wrap(compressible); + sliced.position(17); + sliced.limit(16000); + ByteBuffer direct = ByteBuffer.allocateDirect(8192); + while (direct.hasRemaining()) direct.put((byte) 7); + direct.flip(); + byte[] random = new byte[8192]; + new Random(123).nextBytes(random); + TLogEntry wal = + new TLogEntry( + Arrays.asList( + sliced, + direct.asReadOnlyBuffer(), + ByteBuffer.wrap(new byte[] {1, 2, 3}), + ByteBuffer.wrap(random)), + 1, + true, + 50000) + .setRoutingEpoch(9) + .setPhysicalTime(12345); + TLogEntry queue = + new TLogEntry( + Collections.singletonList(ByteBuffer.wrap(compressible).asReadOnlyBuffer()), + 2, + false, + 16384); + return new TSyncLogEntriesReq( + 3, new TConsensusGroupId(TConsensusGroupType.DataRegion, 5), Arrays.asList(wal, queue)); + } + + @Test + public void testRoundTripThriftAndRetryOwnership() throws Exception { + TSyncLogEntriesReq raw = request(); + TSyncLogEntriesReq snapshot = raw.deepCopy(); + for (int attempt = 0; attempt < 2; attempt++) { + LogCompression.EncodedRequest encoded = LogCompression.compress(raw, codec, 4096, LIMIT); + try { + assertNotSame(raw, encoded.getRequest()); + assertEquals( + encoded.getRequest().getLogEntries().stream() + .flatMap(e -> e.getData().stream()) + .mapToLong(ByteBuffer::remaining) + .sum(), + encoded.getEncodedBytes()); + assertTrue(encoded.getEncodedBytes() < encoded.getRawBytes()); + assertTrue(memory.getMemoryBlock().getUsedMemoryInBytes() > 0); + assertEquals( + codec, + encoded.getRequest().getLogEntries().get(0).getCompressionInfo().get(0).getCodec()); + assertEquals( + TLogCompressionType.NONE, + encoded.getRequest().getLogEntries().get(0).getCompressionInfo().get(2).getCodec()); + TSyncLogEntriesReq received = new TSyncLogEntriesReq(); + new TDeserializer() + .deserialize(received, new TSerializer().serialize(encoded.getRequest())); + LogCompression.decompress(received, LIMIT); + assertEquals(snapshot, received); + assertEquals(snapshot, raw); + assertEquals(17, raw.getLogEntries().get(0).getData().get(0).position()); + assertEquals(16000, raw.getLogEntries().get(0).getData().get(0).limit()); + } finally { + encoded.close(); + encoded.close(); + } + } + } + + @Test + public void testAdaptiveAndMemoryFallback() throws Exception { + TSyncLogEntriesReq raw = request(); + try (LogCompression.EncodedRequest result = LogCompression.compress(raw, codec, LIMIT, LIMIT)) { + assertSame(raw, result.getRequest()); + } + byte[] random = new byte[16384]; + new Random(1).nextBytes(random); + TSyncLogEntriesReq noise = raw.deepCopy(); + noise.setLogEntries( + Collections.singletonList( + new TLogEntry( + Collections.singletonList(ByteBuffer.wrap(random)), 1, false, random.length))); + try (LogCompression.EncodedRequest result = LogCompression.compress(noise, codec, 0, LIMIT)) { + assertSame(noise, result.getRequest()); + } + long capacity = memory.getMemoryBlock().getTotalMemorySizeInBytes(); + assertTrue(memory.reserveLogCompression(capacity)); + try (LogCompression.EncodedRequest result = LogCompression.compress(raw, codec, 0, LIMIT)) { + assertSame(raw, result.getRequest()); + assertTrue(result.isMemoryLimited()); + assertEquals(result.getRawBytes(), result.getEncodedBytes()); + assertEquals(capacity, memory.getMemoryBlock().getUsedMemoryInBytes()); + } finally { + memory.freeLogCompression(capacity); + } + } + + @Test + public void testRejectsEntireRequestBeforePublishingAnyDecodedEntry() throws Exception { + try (LogCompression.EncodedRequest encoded = + LogCompression.compress(request(), codec, 4096, LIMIT)) { + TSyncLogEntriesReq bad = encoded.getRequest().deepCopy(); + bad.getLogEntries().get(1).getCompressionInfo().get(0).setChecksum(0); + TSyncLogEntriesReq snapshot = bad.deepCopy(); + assertThrows(IOException.class, () -> LogCompression.decompress(bad, LIMIT)); + assertEquals(snapshot, bad); + // Also cover corruption of an uncompressed block in a mixed WAL entry. + TSyncLogEntriesReq secondBad = encoded.getRequest().deepCopy(); + secondBad.getLogEntries().get(0).getCompressionInfo().get(2).setChecksum(0); + final TSyncLogEntriesReq mixed = secondBad; + assertThrows(IOException.class, () -> LogCompression.decompress(mixed, LIMIT)); + } + } + + @Test + public void testMalformedMetadataAndCorruptedCodecInput() throws Exception { + try (LogCompression.EncodedRequest encoded = + LogCompression.compress(request(), codec, 4096, LIMIT)) { + TSyncLogEntriesReq count = encoded.getRequest().deepCopy(); + count.getLogEntries().get(0).getCompressionInfo().remove(0); + assertThrows(IOException.class, () -> LogCompression.decompress(count, LIMIT)); + TSyncLogEntriesReq huge = encoded.getRequest().deepCopy(); + huge.getLogEntries() + .get(0) + .getCompressionInfo() + .get(0) + .setUncompressedSize(Integer.MAX_VALUE); + assertThrows(IOException.class, () -> LogCompression.decompress(huge, LIMIT)); + TSyncLogEntriesReq longOutput = encoded.getRequest().deepCopy(); + longOutput.getLogEntries().get(0).getCompressionInfo().get(0).setUncompressedSize(16000); + assertThrows(IOException.class, () -> LogCompression.decompress(longOutput, LIMIT)); + TSyncLogEntriesReq rawLength = encoded.getRequest().deepCopy(); + rawLength.getLogEntries().get(0).getCompressionInfo().get(2).setUncompressedSize(4); + assertThrows(IOException.class, () -> LogCompression.decompress(rawLength, LIMIT)); + TSyncLogEntriesReq negative = encoded.getRequest().deepCopy(); + negative.getLogEntries().get(0).getCompressionInfo().get(0).setUncompressedSize(-1); + assertThrows(IOException.class, () -> LogCompression.decompress(negative, LIMIT)); + TSyncLogEntriesReq unknown = encoded.getRequest().deepCopy(); + unknown.getLogEntries().get(0).getCompressionInfo().get(0).setCodec(null); + assertThrows(IOException.class, () -> LogCompression.decompress(unknown, LIMIT)); + TSyncLogEntriesReq corrupt = encoded.getRequest().deepCopy(); + corrupt.getLogEntries().get(0).getData().set(0, ByteBuffer.wrap(new byte[] {0, 1, 2})); + assertThrows(IOException.class, () -> LogCompression.decompress(corrupt, LIMIT)); + } + } + + @Test + public void testSingleLogAboveSoftBatchLimit() throws Exception { + TSyncLogEntriesReq raw = request(); + raw.setLogEntries( + Collections.singletonList( + new TLogEntry( + Collections.singletonList(ByteBuffer.allocate(2 * 1024 * 1024)), + 1, + false, + 2 * 1024 * 1024))); + try (LogCompression.EncodedRequest encoded = LogCompression.compress(raw, codec, 4096, LIMIT)) { + TSyncLogEntriesReq received = encoded.getRequest().deepCopy(); + LogCompression.decompress(received, LIMIT); + assertEquals(raw, received); + } + } + + @Test + public void testProcessorRejectsBeforeDeserializeAndDecodesWithSendingDisabled() + throws Exception { + IoTConsensus consensus = mock(IoTConsensus.class); + IoTConsensusServerImpl impl = mock(IoTConsensusServerImpl.class); + IStateMachine stateMachine = mock(IStateMachine.class); + when(consensus.getImpl(new DataRegionId(5))).thenReturn(impl); + when(impl.isActive()).thenReturn(true); + // Decoder support must not depend on the sending codec (default NONE). + when(impl.getConfig()).thenReturn(IoTConsensusConfig.newBuilder().build()); + when(impl.getIoTConsensusServerMetrics()).thenReturn(mock(IoTConsensusServerMetrics.class)); + when(impl.getStateMachine()).thenReturn(stateMachine); + IoTConsensusRPCServiceProcessor processor = new IoTConsensusRPCServiceProcessor(consensus); + try (LogCompression.EncodedRequest encoded = + LogCompression.compress(request(), codec, 4096, LIMIT)) { + TSyncLogEntriesReq corrupt = encoded.getRequest().deepCopy(); + corrupt.getLogEntries().get(1).getCompressionInfo().get(0).setChecksum(0); + assertTrue(processor.syncLogEntries(corrupt).isSetLogCompressionError()); + // Snappy reports a short output buffer as IllegalArgumentException; the RPC boundary must + // reject it in the same explicit, pre-application error path as codec I/O failures. + TSyncLogEntriesReq shortOutput = encoded.getRequest().deepCopy(); + shortOutput.getLogEntries().get(0).getCompressionInfo().get(0).setUncompressedSize(1); + assertTrue(processor.syncLogEntries(shortOutput).isSetLogCompressionError()); + verify(impl, never()).getStateMachine(); + verify(impl, never()).syncLog(anyInt(), any()); + verify(impl, never()) + .buildIndexedConsensusRequestForRemoteRequest( + anyLong(), anyLong(), anyLong(), anyInt(), anyList()); + when(impl.buildIndexedConsensusRequestForRemoteRequest( + anyLong(), anyLong(), anyLong(), anyInt(), anyList())) + .thenAnswer( + call -> new IndexedConsensusRequest(-1, call.getArgument(0), call.getArgument(4))); + ByteBufferConsensusRequest applied = new ByteBufferConsensusRequest(ByteBuffer.allocate(1)); + when(stateMachine.deserializeRequest(any())).thenReturn(applied); + when(impl.syncLog(eq(3), eq(applied))) + .thenReturn(new TSStatus(200).setSubStatus(Collections.singletonList(new TSStatus(200)))); + TSyncLogEntriesReq valid = encoded.getRequest().deepCopy(); + TSyncLogEntriesRes result = processor.syncLogEntries(valid); + assertFalse(result.isSetLogCompressionError()); + assertEquals(200, result.getStatuses().get(0).getCode()); + assertEquals(request(), valid); + verify(impl).syncLog(3, applied); + } + } + + @Test + public void testMultiBlockScratchBudgetMatchesSequentialPeak() throws Exception { + memory.setMemoryBlock(new AtomicLongMemoryBlock("compression-tight-pool", null, 512 * 1024)); + TSyncLogEntriesReq raw = request(); + List entries = new ArrayList<>(); + for (int i = 0; i < 16; i++) + entries.add( + new TLogEntry( + Collections.singletonList(ByteBuffer.allocate(16384)), i + 1, false, 16384)); + raw.setLogEntries(entries); + try (LogCompression.EncodedRequest encoded = LogCompression.compress(raw, codec, 4096, LIMIT)) { + assertNotSame(raw, encoded.getRequest()); + assertEquals(16 * 16384, encoded.getRawBytes()); + assertFalse(encoded.isMemoryLimited()); + TSyncLogEntriesReq received = encoded.getRequest().deepCopy(); + LogCompression.decompress(received, LIMIT); + assertEquals(raw, received); + assertTrue(memory.getMemoryBlock().getUsedMemoryInBytes() < 512 * 1024); + } + } + + @Test + public void testMemoryPressureFallsBackPerBlockAndKeepsOtherBlocksCompressed() throws Exception { + memory.setMemoryBlock(new AtomicLongMemoryBlock("compression-partial-pool", null, 48 * 1024)); + TSyncLogEntriesReq raw = request(); + raw.setLogEntries( + Collections.singletonList( + new TLogEntry( + Arrays.asList( + ByteBuffer.allocate(4096), + ByteBuffer.allocate(32768), + ByteBuffer.allocate(4096)), + 1, + true, + 40960))); + try (LogCompression.EncodedRequest encoded = LogCompression.compress(raw, codec, 4096, LIMIT)) { + assertNotSame(raw, encoded.getRequest()); + assertTrue(encoded.isMemoryLimited()); + assertEquals( + codec, + encoded.getRequest().getLogEntries().get(0).getCompressionInfo().get(0).getCodec()); + assertEquals( + TLogCompressionType.NONE, + encoded.getRequest().getLogEntries().get(0).getCompressionInfo().get(1).getCodec()); + assertEquals( + codec, + encoded.getRequest().getLogEntries().get(0).getCompressionInfo().get(2).getCodec()); + TSyncLogEntriesReq received = encoded.getRequest().deepCopy(); + LogCompression.decompress(received, LIMIT); + assertEquals(raw, received); + assertTrue(encoded.getEncodedBytes() < encoded.getRawBytes()); + } + } +} diff --git a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcherCompressionTest.java b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcherCompressionTest.java new file mode 100644 index 0000000000000..81ae68231c2cc --- /dev/null +++ b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcherCompressionTest.java @@ -0,0 +1,178 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.consensus.iot.logdispatcher; + +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.commons.client.IClientManager; +import org.apache.iotdb.commons.client.exception.ClientManagerException; +import org.apache.iotdb.commons.consensus.DataRegionId; +import org.apache.iotdb.commons.memory.AtomicLongMemoryBlock; +import org.apache.iotdb.commons.memory.IMemoryBlock; +import org.apache.iotdb.consensus.IStateMachine; +import org.apache.iotdb.consensus.common.Peer; +import org.apache.iotdb.consensus.config.IoTConsensusConfig; +import org.apache.iotdb.consensus.iot.IoTConsensusServerImpl; +import org.apache.iotdb.consensus.iot.client.AsyncIoTConsensusServiceClient; +import org.apache.iotdb.consensus.iot.client.DispatchLogHandler; +import org.apache.iotdb.consensus.iot.thrift.TLogCompressionType; +import org.apache.iotdb.consensus.iot.thrift.TLogEntry; +import org.apache.iotdb.consensus.iot.thrift.TSyncLogEntriesReq; +import org.apache.iotdb.consensus.iot.thrift.TSyncLogEntriesRes; +import org.apache.iotdb.consensus.iot.util.FakeConsensusReqReader; + +import org.apache.thrift.TException; +import org.apache.thrift.async.AsyncMethodCallback; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Collections; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class LogDispatcherCompressionTest { + @Rule public TemporaryFolder directory = new TemporaryFolder(); + private final IoTConsensusMemoryManager memory = IoTConsensusMemoryManager.getInstance(); + private IMemoryBlock previous; + private LogDispatcher dispatcher; + private LogDispatcher.LogDispatcherThread thread; + @Rule public MockitoRule mocks = MockitoJUnit.rule(); + @Mock private IClientManager clients; + @Captor private ArgumentCaptor> callback; + private AsyncIoTConsensusServiceClient client; + private DispatchLogHandler handler; + private Batch batch; + private Peer target; + + @Before + public void setUp() throws Exception { + previous = memory.getMemoryBlock(); + memory.setMemoryBlock( + new AtomicLongMemoryBlock("sender-compression-test", null, 64L * 1024 * 1024)); + memory.reset(); + IoTConsensusConfig config = + IoTConsensusConfig.newBuilder() + .setReplication( + IoTConsensusConfig.Replication.newBuilder() + .setLogCompressionType(TLogCompressionType.LZ4) + .build()) + .build(); + IoTConsensusServerImpl impl = mock(IoTConsensusServerImpl.class); + IStateMachine stateMachine = mock(IStateMachine.class); + when(impl.getStateMachine()).thenReturn(stateMachine); + when(stateMachine.read(any())).thenReturn(mock(FakeConsensusReqReader.class)); + when(impl.getStorageDir()).thenReturn(directory.newFolder().getAbsolutePath()); + Peer self = new Peer(new DataRegionId(1), 1, new TEndPoint("127.0.0.1", 9000)); + target = new Peer(new DataRegionId(1), 2, new TEndPoint("127.0.0.1", 9001)); + when(impl.getThisNode()).thenReturn(self); + when(impl.getConfiguration()).thenReturn(Collections.singletonList(self)); + client = mock(AsyncIoTConsensusServiceClient.class); + when(clients.borrowClient(target.getEndpoint())) + .thenAnswer( + call -> { + // Encoding is complete before obtaining a client, including its temporary + // reservation. + assertTrue(memory.getMemoryBlock().getUsedMemoryInBytes() > 0); + return client; + }); + dispatcher = new LogDispatcher(impl, clients); + thread = dispatcher.new LogDispatcherThread(target, config, 0); + handler = mock(DispatchLogHandler.class); + batch = new Batch(config); + batch.addTLogEntry( + new TLogEntry(Collections.singletonList(ByteBuffer.allocate(16384)), 1, false, 16384)); + batch.buildIndex(); + } + + @After + public void tearDown() { + Thread.currentThread().interrupt(); + thread.run(); + thread.stop(); + dispatcher.stop(); + assertEquals(0, memory.getMemoryBlock().getUsedMemoryInBytes()); + memory.reset(); + memory.setMemoryBlock(previous); + } + + @Test + public void testSuccessReleasesWireMemoryAndPreservesRawBatch() throws Exception { + verifyAsyncCompletion(false); + } + + @Test + public void testAsyncFailureReleasesWireMemoryAndPreservesRawBatch() throws Exception { + verifyAsyncCompletion(true); + } + + private void verifyAsyncCompletion(boolean failure) throws Exception { + thread.sendBatchAsync(batch, handler); + ArgumentCaptor request = ArgumentCaptor.forClass(TSyncLogEntriesReq.class); + verify(client).syncLogEntries(request.capture(), callback.capture()); + verify(handler).markCompressedAttempt(); + assertTrue(request.getValue().getLogEntries().get(0).isSetCompressionInfo()); + assertTrue(memory.getMemoryBlock().getUsedMemoryInBytes() > 0); + if (failure) { + IOException error = new IOException(); + callback.getValue().onError(error); + verify(handler).onError(error); + } else { + TSyncLogEntriesRes response = new TSyncLogEntriesRes(); + callback.getValue().onComplete(response); + verify(handler).onComplete(response); + } + assertEquals(0, memory.getMemoryBlock().getUsedMemoryInBytes()); + assertEquals(16384, batch.getLogEntries().get(0).getData().get(0).remaining()); + assertTrue(!batch.getLogEntries().get(0).isSetCompressionInfo()); + } + + @Test + public void testBorrowFailureReleasesWireMemory() throws Exception { + ClientManagerException error = new ClientManagerException(new IOException()); + doThrow(error).when(clients).borrowClient(target.getEndpoint()); + thread.sendBatchAsync(batch, handler); + verify(handler).onError(error); + assertEquals(0, memory.getMemoryBlock().getUsedMemoryInBytes()); + } + + @Test + public void testSynchronousSendFailureReleasesWireMemory() throws Exception { + TException error = new TException(); + doThrow(error).when(client).syncLogEntries(any(), any()); + thread.sendBatchAsync(batch, handler); + verify(handler).onError(error); + assertEquals(0, memory.getMemoryBlock().getUsedMemoryInBytes()); + } +} diff --git a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcherThreadMetricsTest.java b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcherThreadMetricsTest.java index 4da003936a9b5..31980378f4301 100644 --- a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcherThreadMetricsTest.java +++ b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcherThreadMetricsTest.java @@ -27,6 +27,7 @@ import org.apache.iotdb.consensus.common.Peer; import org.apache.iotdb.metrics.config.MetricConfig; import org.apache.iotdb.metrics.config.MetricConfigDescriptor; +import org.apache.iotdb.metrics.type.Counter; import org.apache.iotdb.metrics.type.IMetric; import org.apache.iotdb.metrics.type.Timer; import org.apache.iotdb.metrics.utils.MetricInfo; @@ -38,6 +39,7 @@ import org.mockito.Mockito; import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; @@ -116,6 +118,45 @@ public void testStageTimersAreRemovedAfterBindingTwice() { assertNull(findConstructBatchTimer()); } + @Test + public void testCompressionCountersShareRegionOwnership() { + LogDispatcherThreadMetrics first = new LogDispatcherThreadMetrics(mockThread(2)); + LogDispatcherThreadMetrics second = new LogDispatcherThreadMetrics(mockThread(3)); + service.addMetricSet(first); + service.addMetricSet(second); + try { + service.restartService(); + first.recordCompressionBytes(10, 4, true); + second.recordCompressionBytes(20, 6, false); + assertEquals(30, findCompressionCounter(Metric.IOT_LOG_COMPRESSION_BYTES, "raw").getCount()); + assertEquals( + 10, findCompressionCounter(Metric.IOT_LOG_COMPRESSION_BYTES, "encoded").getCount()); + assertEquals( + 1, findCompressionCounter(Metric.IOT_LOG_COMPRESSION_MEMORY_FALLBACKS, null).getCount()); + service.removeMetricSet(second); + first.recordCompressionBytes(5, 2, true); + assertEquals(35, findCompressionCounter(Metric.IOT_LOG_COMPRESSION_BYTES, "raw").getCount()); + assertEquals( + 2, findCompressionCounter(Metric.IOT_LOG_COMPRESSION_MEMORY_FALLBACKS, null).getCount()); + } finally { + service.removeMetricSet(first); + service.removeMetricSet(second); + } + assertNull(findCompressionCounter(Metric.IOT_LOG_COMPRESSION_BYTES, "raw")); + assertNull(findCompressionCounter(Metric.IOT_LOG_COMPRESSION_BYTES, "encoded")); + assertNull(findCompressionCounter(Metric.IOT_LOG_COMPRESSION_MEMORY_FALLBACKS, null)); + } + + private Counter findCompressionCounter(Metric metric, String type) { + for (Map.Entry entry : service.getAllMetrics().entrySet()) { + Map tags = entry.getKey().getTags(); + if (metric.toString().equals(entry.getKey().getName()) + && REGION.toString().equals(tags.get(Tag.REGION.toString())) + && Objects.equals(type, tags.get(Tag.TYPE.toString()))) return (Counter) entry.getValue(); + } + return null; + } + private static LogDispatcher.LogDispatcherThread mockThread(final int peerNodeId) { final LogDispatcher.LogDispatcherThread thread = Mockito.mock(LogDispatcher.LogDispatcherThread.class); diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java index c19b950208a1b..05aa4d589f66f 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java @@ -21,6 +21,7 @@ /** Compile-time i18n constants for DataNode misc subsystems (English). */ public final class DataNodeMiscMessages { + public static final String EXCEPTION_INVALID_IOTCONSENSUS_LOG_COMPRESSION_CONFIGURATION_CODEC_ARG_MIN_SIZE_ARG_AB7603FF = "Invalid IoTConsensus log compression configuration: codec=%s, min_size=%s"; public static final String MESSAGE_QUERY_DOES_NOT_BELONG_TO_CURRENT_SESSION_A1198237 = "The requested query does not belong to the current session."; diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java index 93384a3296ee5..b94b81b94c99e 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java @@ -21,6 +21,7 @@ /** 编译时国际化常量 - DataNode 杂项子系统(中文)。 */ public final class DataNodeMiscMessages { + public static final String EXCEPTION_INVALID_IOTCONSENSUS_LOG_COMPRESSION_CONFIGURATION_CODEC_ARG_MIN_SIZE_ARG_AB7603FF = "无效的 IoTConsensus 日志压缩配置:codec=%s, min_size=%s"; public static final String MESSAGE_QUERY_DOES_NOT_BELONG_TO_CURRENT_SESSION_A1198237 = "请求的查询不属于当前会话。"; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index c6a0d4878969f..03dc68ca78c43 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -30,6 +30,7 @@ import org.apache.iotdb.commons.pipe.config.PipeConfig; import org.apache.iotdb.commons.utils.FileUtils; import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.consensus.iot.thrift.TLogCompressionType; import org.apache.iotdb.db.exception.LoadConfigurationException; import org.apache.iotdb.db.i18n.DataNodeMiscMessages; import org.apache.iotdb.db.protocol.thrift.impl.ClientRPCServiceImpl; @@ -1130,6 +1131,8 @@ public class IoTDBConfig implements DefaultEncodingProvider { // IoTConsensus Config private int maxLogEntriesNumPerBatch = 1024; private int maxSizePerBatch = 16 * 1024 * 1024; + private TLogCompressionType iotLogCompressionType = TLogCompressionType.NONE; + private int iotLogCompressionMinSize = 4096; private int maxPendingBatchesNum = 5; private double maxMemoryRatioForQueue = 0.6; private long regionMigrationSpeedLimitBytesPerSecond = 48 * 1024 * 1024L; @@ -1309,6 +1312,22 @@ public int getMaxLogEntriesNumPerBatch() { return maxLogEntriesNumPerBatch; } + public TLogCompressionType getIotLogCompressionType() { + return iotLogCompressionType; + } + + public void setIotLogCompressionType(TLogCompressionType type) { + iotLogCompressionType = type; + } + + public int getIotLogCompressionMinSize() { + return iotLogCompressionMinSize; + } + + public void setIotLogCompressionMinSize(int size) { + iotLogCompressionMinSize = size; + } + public int getMaxSizePerBatch() { return maxSizePerBatch; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index 04fe3381b7541..0c2b9d5145c7b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -39,6 +39,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TGlobalConfig; import org.apache.iotdb.confignode.rpc.thrift.TRatisConfig; import org.apache.iotdb.consensus.config.IoTConsensusV2Config; +import org.apache.iotdb.consensus.iot.thrift.TLogCompressionType; import org.apache.iotdb.db.conf.rest.IoTDBRestServiceDescriptor; import org.apache.iotdb.db.consensus.DataRegionConsensusImpl; import org.apache.iotdb.db.i18n.DataNodeMiscMessages; @@ -95,6 +96,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Locale; import java.util.Optional; import java.util.Properties; import java.util.ServiceLoader; @@ -1192,6 +1194,25 @@ public void loadProperties(TrimProperties properties) throws BadNodeUrlException properties.getProperty( "schema_thread_count", Integer.toString(conf.getSchemaThreadCount())))); + // Sending policy is a startup setting. Decoding is always available for in-flight requests. + String compression = properties.getProperty("data_region_iot_log_compression", "NONE").trim(); + String compressionMinimum = + properties.getProperty("data_region_iot_log_compression_min_size", "4096"); + try { + TLogCompressionType type = TLogCompressionType.valueOf(compression.toUpperCase(Locale.ROOT)); + int minimum = Integer.parseInt(compressionMinimum); + if (minimum < 0) throw new IllegalArgumentException(); + conf.setIotLogCompressionType(type); + conf.setIotLogCompressionMinSize(minimum); + } catch (IllegalArgumentException e) { + throw new IOException( + String.format( + DataNodeMiscMessages + .EXCEPTION_INVALID_IOTCONSENSUS_LOG_COMPRESSION_CONFIGURATION_CODEC_ARG_MIN_SIZE_ARG_AB7603FF, + compression, + compressionMinimum), + e); + } loadIoTConsensusProps(properties); loadIoTConsensusV2Props(properties); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/DataRegionConsensusImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/DataRegionConsensusImpl.java index ba08b7379bcc3..0bbebf1a43d8d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/DataRegionConsensusImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/DataRegionConsensusImpl.java @@ -197,6 +197,8 @@ private static ConsensusConfig buildConsensusConfig() { .setConsensusMemoryBlock(memoryBlock) .setMaxLogEntriesNumPerBatch(CONF.getMaxLogEntriesNumPerBatch()) .setMaxSizePerBatch(CONF.getMaxSizePerBatch()) + .setLogCompressionType(CONF.getIotLogCompressionType()) + .setLogCompressionMinSize(CONF.getIotLogCompressionMinSize()) .setMaxPendingBatchesNum(CONF.getMaxPendingBatchesNum()) .setMaxMemoryRatioForQueue(CONF.getMaxMemoryRatioForQueue()) .setRegionMigrationSpeedLimitBytesPerSecond( diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template index a3fe0970318c8..8fb4c65bde372 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template @@ -1873,6 +1873,20 @@ enable_wal_compression=true ### IoTConsensus Configuration #################### +# IoTConsensus V1 log payload codec: NONE, LZ4, SNAPPY, ZSTD (level 1). +# Enable only after every peer has the same supporting version and configuration. +# This controls sending; receivers can decode every supported codec even when set to NONE. +# Independent of WAL/TsFile compression and dn_rpc_thrift_compression_enable. +# effectiveMode: restart +# Datatype: String +data_region_iot_log_compression = NONE + +# Minimum individual log buffer size in bytes to try compression. +# Buffers and entries that do not save space including descriptors are sent uncompressed. +# effectiveMode: restart +# Datatype: int +data_region_iot_log_compression_min_size = 4096 + # The maximum log entries num in IoTConsensus Batch # effectiveMode: hot_reload # Datatype: int diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java index a4b853ddc9dc8..8f780d57c32d8 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java @@ -61,6 +61,8 @@ public enum Metric { STAGE("stage"), IOT_CONSENSUS("iot_consensus"), IOT_SEND_LOG("iot_send_log"), + IOT_LOG_COMPRESSION_BYTES("iot_log_compression_bytes"), + IOT_LOG_COMPRESSION_MEMORY_FALLBACKS("iot_log_compression_memory_fallbacks"), IOT_RECEIVE_LOG("iot_receive_log"), IOT_CONSENSUS_V2("iot_consensus_v2"), IOT_CONSENSUS_V2_MODE("iot_consensus_v2_mode"), diff --git a/iotdb-protocol/thrift-consensus/src/main/thrift/iotconsensus.thrift b/iotdb-protocol/thrift-consensus/src/main/thrift/iotconsensus.thrift index 169a7961e4472..cbf0c87576fb1 100644 --- a/iotdb-protocol/thrift-consensus/src/main/thrift/iotconsensus.thrift +++ b/iotdb-protocol/thrift-consensus/src/main/thrift/iotconsensus.thrift @@ -20,6 +20,20 @@ include "common.thrift" namespace java org.apache.iotdb.consensus.iot.thrift +// Codec identifiers are wire values, independent of TsFile compression identifiers. +enum TLogCompressionType { + NONE = 0, + LZ4 = 1, + SNAPPY = 2, + ZSTD = 3 +} + +struct TLogBufferCompression { + 1: required TLogCompressionType codec + 2: required i32 uncompressedSize + 3: required i32 checksum +} + // If this TLogEntry comes from the queue in memory, the list size of the data will only be 1. // If this TLogEntry comes from wal, the list size of the data may be greater than 1. struct TLogEntry { @@ -29,6 +43,8 @@ struct TLogEntry { 4: required i64 memorySize 5: optional i64 routingEpoch 6: optional i64 physicalTime + // One descriptor per data buffer; absent means the original, uncompressed entry. + 7: optional list compressionInfo } struct TSyncLogEntriesReq { @@ -41,6 +57,8 @@ struct TSyncLogEntriesReq { struct TSyncLogEntriesRes { 1: required list statuses 2: optional i64 receiverMemSize + // Rejected before application. This must never be treated as a completed batch. + 3: optional common.TSStatus logCompressionError } struct TSyncWriterSafeTimeBarrierReq { From e9874ce08a3e365898b522ab7950cc48d0fa39cd Mon Sep 17 00:00:00 2001 From: 761417898 <761417898@qq.com> Date: Thu, 12 Nov 2026 18:35:49 +0800 Subject: [PATCH 2/5] Use a fixed minimum size for IoTConsensus log compression --- .../consensus/i18n/IoTConsensusMessages.java | 1 - .../consensus/i18n/IoTConsensusMessages.java | 1 - .../consensus/config/IoTConsensusConfig.java | 22 -------- .../iot/logdispatcher/LogCompression.java | 9 ++-- .../iot/logdispatcher/LogDispatcher.java | 1 - .../iot/logdispatcher/LogCompressionTest.java | 53 ++++++++++++++----- .../iotdb/db/i18n/DataNodeMiscMessages.java | 2 +- .../iotdb/db/i18n/DataNodeMiscMessages.java | 2 +- .../org/apache/iotdb/db/conf/IoTDBConfig.java | 9 ---- .../apache/iotdb/db/conf/IoTDBDescriptor.java | 10 +--- .../db/consensus/DataRegionConsensusImpl.java | 1 - .../conf/iotdb-system.properties.template | 7 +-- 12 files changed, 51 insertions(+), 67 deletions(-) diff --git a/iotdb-core/consensus/src/main/i18n/en/org/apache/iotdb/consensus/i18n/IoTConsensusMessages.java b/iotdb-core/consensus/src/main/i18n/en/org/apache/iotdb/consensus/i18n/IoTConsensusMessages.java index 3ebcd8f93e839..a18447f3744c5 100644 --- a/iotdb-core/consensus/src/main/i18n/en/org/apache/iotdb/consensus/i18n/IoTConsensusMessages.java +++ b/iotdb-core/consensus/src/main/i18n/en/org/apache/iotdb/consensus/i18n/IoTConsensusMessages.java @@ -25,7 +25,6 @@ */ public final class IoTConsensusMessages { - public static final String EXCEPTION_INVALID_IOTCONSENSUS_LOG_COMPRESSION_MINIMUM_SIZE_ARG_ACEEA255 = "Invalid IoTConsensus log compression minimum size: %d"; public static final String EXCEPTION_INVALID_IOTCONSENSUS_COMPRESSED_LOG_ARG_EXPECTED_ARG_ACTUAL_ARG_4986175E = "Invalid IoTConsensus compressed log (%s, expected %d, actual %d)"; diff --git a/iotdb-core/consensus/src/main/i18n/zh/org/apache/iotdb/consensus/i18n/IoTConsensusMessages.java b/iotdb-core/consensus/src/main/i18n/zh/org/apache/iotdb/consensus/i18n/IoTConsensusMessages.java index 5b2ddc42f860d..1d915b9af66cb 100644 --- a/iotdb-core/consensus/src/main/i18n/zh/org/apache/iotdb/consensus/i18n/IoTConsensusMessages.java +++ b/iotdb-core/consensus/src/main/i18n/zh/org/apache/iotdb/consensus/i18n/IoTConsensusMessages.java @@ -24,7 +24,6 @@ */ public final class IoTConsensusMessages { - public static final String EXCEPTION_INVALID_IOTCONSENSUS_LOG_COMPRESSION_MINIMUM_SIZE_ARG_ACEEA255 = "无效的 IoTConsensus 日志压缩最小大小:%d"; public static final String EXCEPTION_INVALID_IOTCONSENSUS_COMPRESSED_LOG_ARG_EXPECTED_ARG_ACTUAL_ARG_4986175E = "无效的 IoTConsensus 压缩日志(%s,预期 %d,实际 %d)"; diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/config/IoTConsensusConfig.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/config/IoTConsensusConfig.java index c31197f0334eb..742992bba1757 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/config/IoTConsensusConfig.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/config/IoTConsensusConfig.java @@ -22,7 +22,6 @@ import org.apache.iotdb.commons.client.property.ClientPoolProperty.DefaultProperty; import org.apache.iotdb.commons.memory.AtomicLongMemoryBlock; import org.apache.iotdb.commons.memory.IMemoryBlock; -import org.apache.iotdb.consensus.i18n.IoTConsensusMessages; import org.apache.iotdb.consensus.iot.thrift.TLogCompressionType; import java.util.Optional; @@ -313,7 +312,6 @@ public static class Replication { private final int maxLogEntriesNumPerBatch; private final int maxSizePerBatch; private final TLogCompressionType logCompressionType; - private final int logCompressionMinSize; private final int maxPendingBatchesNum; private final int maxQueueLength; @@ -335,7 +333,6 @@ private Replication( int maxLogEntriesNumPerBatch, int maxSizePerBatch, TLogCompressionType logCompressionType, - int logCompressionMinSize, int maxPendingBatchesNum, int maxQueueLength, long maxWaitingTimeForWaitBatchInMs, @@ -354,7 +351,6 @@ private Replication( this.maxLogEntriesNumPerBatch = maxLogEntriesNumPerBatch; this.maxSizePerBatch = maxSizePerBatch; this.logCompressionType = logCompressionType; - this.logCompressionMinSize = logCompressionMinSize; this.maxPendingBatchesNum = maxPendingBatchesNum; this.maxQueueLength = maxQueueLength; this.maxWaitingTimeForWaitBatchInMs = maxWaitingTimeForWaitBatchInMs; @@ -384,10 +380,6 @@ public TLogCompressionType getLogCompressionType() { return logCompressionType; } - public int getLogCompressionMinSize() { - return logCompressionMinSize; - } - public int getMaxPendingBatchesNum() { return maxPendingBatchesNum; } @@ -457,7 +449,6 @@ public static class Builder { private int maxLogEntriesNumPerBatch = 1024; private int maxSizePerBatch = 16 * 1024 * 1024; private TLogCompressionType logCompressionType = TLogCompressionType.NONE; - private int logCompressionMinSize = 4096; private int maxPendingBatchesNum = 5; private int maxQueueLength = 4096; private long maxWaitingTimeForWaitBatchInMs = 10 * 1000L; @@ -496,18 +487,6 @@ public Builder setLogCompressionType(TLogCompressionType type) { return this; } - public Builder setLogCompressionMinSize(int size) { - if (size < 0) { - throw new IllegalArgumentException( - String.format( - IoTConsensusMessages - .EXCEPTION_INVALID_IOTCONSENSUS_LOG_COMPRESSION_MINIMUM_SIZE_ARG_ACEEA255, - size)); - } - this.logCompressionMinSize = size; - return this; - } - public Replication.Builder setMaxPendingBatchesNum(int maxPendingBatchesNum) { this.maxPendingBatchesNum = maxPendingBatchesNum; return this; @@ -593,7 +572,6 @@ public Replication build() { maxLogEntriesNumPerBatch, maxSizePerBatch, logCompressionType, - logCompressionMinSize, maxPendingBatchesNum, maxQueueLength, maxWaitingTimeForWaitBatchInMs, diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompression.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompression.java index 1162873f2c4de..24c80984a9c07 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompression.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompression.java @@ -41,6 +41,9 @@ /** Wire-only encoding: the original batch and its buffer positions remain available for retries. */ public final class LogCompression { + // Avoid codec and descriptor overhead on tiny blocks; larger blocks still need a net size + // benefit. + private static final int MIN_COMPRESSION_SIZE = 4096; // Conservative allowance for the additional Thrift descriptor and list field on the wire. private static final int DESCRIPTOR_BYTES = 32; private static final ICompressor LZ4 = ICompressor.getCompressor(CompressionType.LZ4); @@ -100,7 +103,7 @@ public void close() { } public static EncodedRequest compress( - TSyncLogEntriesReq original, TLogCompressionType codec, int minSize, int maxDecodedBytes) + TSyncLogEntriesReq original, TLogCompressionType codec, int maxDecodedBytes) throws IOException { long buffers = 0, rawSize = 0; boolean hasEligibleBuffer = false; @@ -108,7 +111,7 @@ public static EncodedRequest compress( for (ByteBuffer buffer : entry.getData()) { int size = buffer.remaining(); rawSize += size; - if (size >= minSize && size > 0) { + if (size >= MIN_COMPRESSION_SIZE) { hasEligibleBuffer = true; } buffers++; @@ -138,7 +141,7 @@ public static EncodedRequest compress( raw += size; ByteBuffer encoded = buffer; TLogCompressionType actualCodec = TLogCompressionType.NONE; - if (size >= minSize && size > 0) { + if (size >= MIN_COMPRESSION_SIZE) { // Includes an optional input copy, codec-bound output and compact output, with // allowance for array headers and codec bounds on very small buffers. long workspace = 4L * size + 256; diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcher.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcher.java index 7f622c000da5e..972265a4fc09b 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcher.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcher.java @@ -609,7 +609,6 @@ public void sendBatchAsync(Batch batch, DispatchLogHandler handler) { LogCompression.compress( req, config.getReplication().getLogCompressionType(), - config.getReplication().getLogCompressionMinSize(), config.getRpc().getThriftMaxFrameSize()); logDispatcherThreadMetrics.recordCompressTime(System.nanoTime() - compressionStart); logDispatcherThreadMetrics.recordCompressionBytes( diff --git a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompressionTest.java b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompressionTest.java index 09eabafb68198..14a351e1db170 100644 --- a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompressionTest.java +++ b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompressionTest.java @@ -137,7 +137,7 @@ public void testRoundTripThriftAndRetryOwnership() throws Exception { TSyncLogEntriesReq raw = request(); TSyncLogEntriesReq snapshot = raw.deepCopy(); for (int attempt = 0; attempt < 2; attempt++) { - LogCompression.EncodedRequest encoded = LogCompression.compress(raw, codec, 4096, LIMIT); + LogCompression.EncodedRequest encoded = LogCompression.compress(raw, codec, LIMIT); try { assertNotSame(raw, encoded.getRequest()); assertEquals( @@ -169,10 +169,40 @@ public void testRoundTripThriftAndRetryOwnership() throws Exception { } } + @Test + public void testFixedMinimumSizeBoundary() throws Exception { + for (int size : new int[] {4095, 4096, 4097}) { + TSyncLogEntriesReq raw = + new TSyncLogEntriesReq( + 1, + new TConsensusGroupId(TConsensusGroupType.DataRegion, 1), + Collections.singletonList( + new TLogEntry( + Collections.singletonList(ByteBuffer.allocate(size)), 1, false, size))); + try (LogCompression.EncodedRequest encoded = LogCompression.compress(raw, codec, LIMIT)) { + if (size < 4096) { + assertSame(raw, encoded.getRequest()); + assertFalse(encoded.getRequest().getLogEntries().get(0).isSetCompressionInfo()); + } else { + assertNotSame(raw, encoded.getRequest()); + assertEquals( + codec, + encoded.getRequest().getLogEntries().get(0).getCompressionInfo().get(0).getCodec()); + } + TSyncLogEntriesReq received = encoded.getRequest().deepCopy(); + LogCompression.decompress(received, LIMIT); + assertEquals( + raw.getLogEntries().get(0).getData(), received.getLogEntries().get(0).getData()); + assertEquals(0, raw.getLogEntries().get(0).getData().get(0).position()); + } + } + } + @Test public void testAdaptiveAndMemoryFallback() throws Exception { TSyncLogEntriesReq raw = request(); - try (LogCompression.EncodedRequest result = LogCompression.compress(raw, codec, LIMIT, LIMIT)) { + try (LogCompression.EncodedRequest result = + LogCompression.compress(raw, TLogCompressionType.NONE, LIMIT)) { assertSame(raw, result.getRequest()); } byte[] random = new byte[16384]; @@ -182,12 +212,12 @@ public void testAdaptiveAndMemoryFallback() throws Exception { Collections.singletonList( new TLogEntry( Collections.singletonList(ByteBuffer.wrap(random)), 1, false, random.length))); - try (LogCompression.EncodedRequest result = LogCompression.compress(noise, codec, 0, LIMIT)) { + try (LogCompression.EncodedRequest result = LogCompression.compress(noise, codec, LIMIT)) { assertSame(noise, result.getRequest()); } long capacity = memory.getMemoryBlock().getTotalMemorySizeInBytes(); assertTrue(memory.reserveLogCompression(capacity)); - try (LogCompression.EncodedRequest result = LogCompression.compress(raw, codec, 0, LIMIT)) { + try (LogCompression.EncodedRequest result = LogCompression.compress(raw, codec, LIMIT)) { assertSame(raw, result.getRequest()); assertTrue(result.isMemoryLimited()); assertEquals(result.getRawBytes(), result.getEncodedBytes()); @@ -199,8 +229,7 @@ public void testAdaptiveAndMemoryFallback() throws Exception { @Test public void testRejectsEntireRequestBeforePublishingAnyDecodedEntry() throws Exception { - try (LogCompression.EncodedRequest encoded = - LogCompression.compress(request(), codec, 4096, LIMIT)) { + try (LogCompression.EncodedRequest encoded = LogCompression.compress(request(), codec, LIMIT)) { TSyncLogEntriesReq bad = encoded.getRequest().deepCopy(); bad.getLogEntries().get(1).getCompressionInfo().get(0).setChecksum(0); TSyncLogEntriesReq snapshot = bad.deepCopy(); @@ -216,8 +245,7 @@ public void testRejectsEntireRequestBeforePublishingAnyDecodedEntry() throws Exc @Test public void testMalformedMetadataAndCorruptedCodecInput() throws Exception { - try (LogCompression.EncodedRequest encoded = - LogCompression.compress(request(), codec, 4096, LIMIT)) { + try (LogCompression.EncodedRequest encoded = LogCompression.compress(request(), codec, LIMIT)) { TSyncLogEntriesReq count = encoded.getRequest().deepCopy(); count.getLogEntries().get(0).getCompressionInfo().remove(0); assertThrows(IOException.class, () -> LogCompression.decompress(count, LIMIT)); @@ -256,7 +284,7 @@ public void testSingleLogAboveSoftBatchLimit() throws Exception { 1, false, 2 * 1024 * 1024))); - try (LogCompression.EncodedRequest encoded = LogCompression.compress(raw, codec, 4096, LIMIT)) { + try (LogCompression.EncodedRequest encoded = LogCompression.compress(raw, codec, LIMIT)) { TSyncLogEntriesReq received = encoded.getRequest().deepCopy(); LogCompression.decompress(received, LIMIT); assertEquals(raw, received); @@ -276,8 +304,7 @@ public void testProcessorRejectsBeforeDeserializeAndDecodesWithSendingDisabled() when(impl.getIoTConsensusServerMetrics()).thenReturn(mock(IoTConsensusServerMetrics.class)); when(impl.getStateMachine()).thenReturn(stateMachine); IoTConsensusRPCServiceProcessor processor = new IoTConsensusRPCServiceProcessor(consensus); - try (LogCompression.EncodedRequest encoded = - LogCompression.compress(request(), codec, 4096, LIMIT)) { + try (LogCompression.EncodedRequest encoded = LogCompression.compress(request(), codec, LIMIT)) { TSyncLogEntriesReq corrupt = encoded.getRequest().deepCopy(); corrupt.getLogEntries().get(1).getCompressionInfo().get(0).setChecksum(0); assertTrue(processor.syncLogEntries(corrupt).isSetLogCompressionError()); @@ -318,7 +345,7 @@ public void testMultiBlockScratchBudgetMatchesSequentialPeak() throws Exception new TLogEntry( Collections.singletonList(ByteBuffer.allocate(16384)), i + 1, false, 16384)); raw.setLogEntries(entries); - try (LogCompression.EncodedRequest encoded = LogCompression.compress(raw, codec, 4096, LIMIT)) { + try (LogCompression.EncodedRequest encoded = LogCompression.compress(raw, codec, LIMIT)) { assertNotSame(raw, encoded.getRequest()); assertEquals(16 * 16384, encoded.getRawBytes()); assertFalse(encoded.isMemoryLimited()); @@ -343,7 +370,7 @@ public void testMemoryPressureFallsBackPerBlockAndKeepsOtherBlocksCompressed() t 1, true, 40960))); - try (LogCompression.EncodedRequest encoded = LogCompression.compress(raw, codec, 4096, LIMIT)) { + try (LogCompression.EncodedRequest encoded = LogCompression.compress(raw, codec, LIMIT)) { assertNotSame(raw, encoded.getRequest()); assertTrue(encoded.isMemoryLimited()); assertEquals( diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java index 05aa4d589f66f..1fb140df616a5 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java @@ -21,7 +21,7 @@ /** Compile-time i18n constants for DataNode misc subsystems (English). */ public final class DataNodeMiscMessages { - public static final String EXCEPTION_INVALID_IOTCONSENSUS_LOG_COMPRESSION_CONFIGURATION_CODEC_ARG_MIN_SIZE_ARG_AB7603FF = "Invalid IoTConsensus log compression configuration: codec=%s, min_size=%s"; + public static final String EXCEPTION_INVALID_IOTCONSENSUS_LOG_COMPRESSION_CODEC_ARG_2B9C27F7 = "Invalid IoTConsensus log compression codec: %s"; public static final String MESSAGE_QUERY_DOES_NOT_BELONG_TO_CURRENT_SESSION_A1198237 = "The requested query does not belong to the current session."; diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java index b94b81b94c99e..74f25d0d9a955 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java @@ -21,7 +21,7 @@ /** 编译时国际化常量 - DataNode 杂项子系统(中文)。 */ public final class DataNodeMiscMessages { - public static final String EXCEPTION_INVALID_IOTCONSENSUS_LOG_COMPRESSION_CONFIGURATION_CODEC_ARG_MIN_SIZE_ARG_AB7603FF = "无效的 IoTConsensus 日志压缩配置:codec=%s, min_size=%s"; + public static final String EXCEPTION_INVALID_IOTCONSENSUS_LOG_COMPRESSION_CODEC_ARG_2B9C27F7 = "无效的 IoTConsensus 日志压缩算法:%s"; public static final String MESSAGE_QUERY_DOES_NOT_BELONG_TO_CURRENT_SESSION_A1198237 = "请求的查询不属于当前会话。"; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 03dc68ca78c43..41dc78446d863 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -1132,7 +1132,6 @@ public class IoTDBConfig implements DefaultEncodingProvider { private int maxLogEntriesNumPerBatch = 1024; private int maxSizePerBatch = 16 * 1024 * 1024; private TLogCompressionType iotLogCompressionType = TLogCompressionType.NONE; - private int iotLogCompressionMinSize = 4096; private int maxPendingBatchesNum = 5; private double maxMemoryRatioForQueue = 0.6; private long regionMigrationSpeedLimitBytesPerSecond = 48 * 1024 * 1024L; @@ -1320,14 +1319,6 @@ public void setIotLogCompressionType(TLogCompressionType type) { iotLogCompressionType = type; } - public int getIotLogCompressionMinSize() { - return iotLogCompressionMinSize; - } - - public void setIotLogCompressionMinSize(int size) { - iotLogCompressionMinSize = size; - } - public int getMaxSizePerBatch() { return maxSizePerBatch; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index 0c2b9d5145c7b..ce526ff448f59 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -1196,21 +1196,15 @@ public void loadProperties(TrimProperties properties) throws BadNodeUrlException // Sending policy is a startup setting. Decoding is always available for in-flight requests. String compression = properties.getProperty("data_region_iot_log_compression", "NONE").trim(); - String compressionMinimum = - properties.getProperty("data_region_iot_log_compression_min_size", "4096"); try { TLogCompressionType type = TLogCompressionType.valueOf(compression.toUpperCase(Locale.ROOT)); - int minimum = Integer.parseInt(compressionMinimum); - if (minimum < 0) throw new IllegalArgumentException(); conf.setIotLogCompressionType(type); - conf.setIotLogCompressionMinSize(minimum); } catch (IllegalArgumentException e) { throw new IOException( String.format( DataNodeMiscMessages - .EXCEPTION_INVALID_IOTCONSENSUS_LOG_COMPRESSION_CONFIGURATION_CODEC_ARG_MIN_SIZE_ARG_AB7603FF, - compression, - compressionMinimum), + .EXCEPTION_INVALID_IOTCONSENSUS_LOG_COMPRESSION_CODEC_ARG_2B9C27F7, + compression), e); } loadIoTConsensusProps(properties); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/DataRegionConsensusImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/DataRegionConsensusImpl.java index 0bbebf1a43d8d..6efaee537e190 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/DataRegionConsensusImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/DataRegionConsensusImpl.java @@ -198,7 +198,6 @@ private static ConsensusConfig buildConsensusConfig() { .setMaxLogEntriesNumPerBatch(CONF.getMaxLogEntriesNumPerBatch()) .setMaxSizePerBatch(CONF.getMaxSizePerBatch()) .setLogCompressionType(CONF.getIotLogCompressionType()) - .setLogCompressionMinSize(CONF.getIotLogCompressionMinSize()) .setMaxPendingBatchesNum(CONF.getMaxPendingBatchesNum()) .setMaxMemoryRatioForQueue(CONF.getMaxMemoryRatioForQueue()) .setRegionMigrationSpeedLimitBytesPerSecond( diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template index 8fb4c65bde372..9e1bbe1fddab6 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template @@ -1877,16 +1877,11 @@ enable_wal_compression=true # Enable only after every peer has the same supporting version and configuration. # This controls sending; receivers can decode every supported codec even when set to NONE. # Independent of WAL/TsFile compression and dn_rpc_thrift_compression_enable. +# Buffers smaller than 4 KiB or without a net size benefit are sent uncompressed. # effectiveMode: restart # Datatype: String data_region_iot_log_compression = NONE -# Minimum individual log buffer size in bytes to try compression. -# Buffers and entries that do not save space including descriptors are sent uncompressed. -# effectiveMode: restart -# Datatype: int -data_region_iot_log_compression_min_size = 4096 - # The maximum log entries num in IoTConsensus Batch # effectiveMode: hot_reload # Datatype: int From 51bb63c2a8b789705f9087942523f4101f54cbaa Mon Sep 17 00:00:00 2001 From: 761417898 <761417898@qq.com> Date: Fri, 13 Nov 2026 12:57:32 +0800 Subject: [PATCH 3/5] Remove CRC checks from IoTConsensus log compression --- .../iot/logdispatcher/LogCompression.java | 22 +++--------------- .../iot/logdispatcher/LogCompressionTest.java | 23 +++++++++++++++---- .../src/main/thrift/iotconsensus.thrift | 1 - 3 files changed, 22 insertions(+), 24 deletions(-) diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompression.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompression.java index 24c80984a9c07..4c0e8834df646 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompression.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompression.java @@ -37,7 +37,6 @@ import java.util.Arrays; import java.util.List; import java.util.concurrent.atomic.AtomicLong; -import java.util.zip.CRC32C; /** Wire-only encoding: the original batch and its buffer positions remain available for retries. */ public final class LogCompression { @@ -175,13 +174,11 @@ public static EncodedRequest compress( } } data.add(encoded); - info.add(new TLogBufferCompression(actualCodec, size, 0)); + info.add(new TLogBufferCompression(actualCodec, size)); wire += encoded.remaining(); } // Raw descriptors in a mixed WAL entry cost space too. if (compressed && wire + DESCRIPTOR_BYTES * (long) data.size() + 8 < raw) { - for (int i = 0; i < info.size(); i++) - info.get(i).setChecksum(checksum(entry.getData().get(i))); entries.add(copy(entry, data).setCompressionInfo(info)); retained += arrays + data.size() * 256L + 128; result.encodedBytes += wire - raw; @@ -202,7 +199,7 @@ public static EncodedRequest compress( } } - /** Validate every declared size before allocating; apply nothing until all buffers pass CRC. */ + /** Validate declared sizes before allocating; publish nothing until every buffer is decoded. */ public static void decompress(TSyncLogEntriesReq request, int maxDecodedBytes) throws IOException { boolean encoded = false; @@ -255,12 +252,6 @@ public static void decompress(TSyncLogEntriesReq request, int maxDecodedBytes) if (actual != output.length) throw invalid(Failure.LENGTH, output.length, actual); buffer = ByteBuffer.wrap(output); } - int crc = checksum(buffer); - if (crc != info.getChecksum()) - throw invalid( - Failure.CHECKSUM, - Integer.toUnsignedLong(info.getChecksum()), - Integer.toUnsignedLong(crc)); data.add(buffer); } decoded.add(copy(entry, data)); @@ -285,12 +276,6 @@ private static byte[] bytes(ByteBuffer buffer) { return bytes; } - private static int checksum(ByteBuffer buffer) { - CRC32C crc = new CRC32C(); - crc.update(buffer.duplicate()); - return (int) crc.getValue(); - } - private static ICompressor compressor(TLogCompressionType codec) throws IOException { switch (codec) { case LZ4: @@ -319,8 +304,7 @@ private enum Failure { CODEC, LENGTH, LIMIT, - DESCRIPTORS, - CHECKSUM + DESCRIPTORS } private static IOException invalid(Failure failure, long expected, long actual) { diff --git a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompressionTest.java b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompressionTest.java index 14a351e1db170..4d8ce6103a182 100644 --- a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompressionTest.java +++ b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompressionTest.java @@ -231,18 +231,33 @@ public void testAdaptiveAndMemoryFallback() throws Exception { public void testRejectsEntireRequestBeforePublishingAnyDecodedEntry() throws Exception { try (LogCompression.EncodedRequest encoded = LogCompression.compress(request(), codec, LIMIT)) { TSyncLogEntriesReq bad = encoded.getRequest().deepCopy(); - bad.getLogEntries().get(1).getCompressionInfo().get(0).setChecksum(0); + // The first entry decodes successfully before the later entry fails in the codec. + bad.getLogEntries().get(1).getData().set(0, ByteBuffer.wrap(new byte[] {0, 1, 2})); TSyncLogEntriesReq snapshot = bad.deepCopy(); assertThrows(IOException.class, () -> LogCompression.decompress(bad, LIMIT)); assertEquals(snapshot, bad); - // Also cover corruption of an uncompressed block in a mixed WAL entry. + // Raw blocks in mixed entries still have to match their declared lengths. TSyncLogEntriesReq secondBad = encoded.getRequest().deepCopy(); - secondBad.getLogEntries().get(0).getCompressionInfo().get(2).setChecksum(0); + secondBad.getLogEntries().get(0).getCompressionInfo().get(2).setUncompressedSize(4); final TSyncLogEntriesReq mixed = secondBad; assertThrows(IOException.class, () -> LogCompression.decompress(mixed, LIMIT)); } } + @Test + public void testMixedRawBufferContentIsForwardedWithoutChecksum() throws Exception { + try (LogCompression.EncodedRequest encoded = LogCompression.compress(request(), codec, LIMIT)) { + TSyncLogEntriesReq received = encoded.getRequest().deepCopy(); + byte[] changed = {4, 5, 6}; + received.getLogEntries().get(0).getData().set(2, ByteBuffer.wrap(changed)); + TSyncLogEntriesReq expected = request(); + expected.getLogEntries().get(0).getData().set(2, ByteBuffer.wrap(changed)); + LogCompression.decompress(received, LIMIT); + // Same-length raw content is preserved verbatim; there is no payload integrity checksum. + assertEquals(expected, received); + } + } + @Test public void testMalformedMetadataAndCorruptedCodecInput() throws Exception { try (LogCompression.EncodedRequest encoded = LogCompression.compress(request(), codec, LIMIT)) { @@ -306,7 +321,7 @@ public void testProcessorRejectsBeforeDeserializeAndDecodesWithSendingDisabled() IoTConsensusRPCServiceProcessor processor = new IoTConsensusRPCServiceProcessor(consensus); try (LogCompression.EncodedRequest encoded = LogCompression.compress(request(), codec, LIMIT)) { TSyncLogEntriesReq corrupt = encoded.getRequest().deepCopy(); - corrupt.getLogEntries().get(1).getCompressionInfo().get(0).setChecksum(0); + corrupt.getLogEntries().get(1).getData().set(0, ByteBuffer.wrap(new byte[] {0, 1, 2})); assertTrue(processor.syncLogEntries(corrupt).isSetLogCompressionError()); // Snappy reports a short output buffer as IllegalArgumentException; the RPC boundary must // reject it in the same explicit, pre-application error path as codec I/O failures. diff --git a/iotdb-protocol/thrift-consensus/src/main/thrift/iotconsensus.thrift b/iotdb-protocol/thrift-consensus/src/main/thrift/iotconsensus.thrift index cbf0c87576fb1..34bb3a85ce550 100644 --- a/iotdb-protocol/thrift-consensus/src/main/thrift/iotconsensus.thrift +++ b/iotdb-protocol/thrift-consensus/src/main/thrift/iotconsensus.thrift @@ -31,7 +31,6 @@ enum TLogCompressionType { struct TLogBufferCompression { 1: required TLogCompressionType codec 2: required i32 uncompressedSize - 3: required i32 checksum } // If this TLogEntry comes from the queue in memory, the list size of the data will only be 1. From 31f35e67b4890cc21afb3c1477e1de5afb613e1b Mon Sep 17 00:00:00 2001 From: 761417898 <761417898@qq.com> Date: Fri, 13 Nov 2026 17:29:58 +0800 Subject: [PATCH 4/5] Address IoTConsensus compression naming and code style review --- .../iot/logdispatcher/LogCompression.java | 107 +++++++++++------- 1 file changed, 69 insertions(+), 38 deletions(-) diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompression.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompression.java index 4c0e8834df646..1aa770121d2f9 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompression.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompression.java @@ -40,8 +40,7 @@ /** Wire-only encoding: the original batch and its buffer positions remain available for retries. */ public final class LogCompression { - // Avoid codec and descriptor overhead on tiny blocks; larger blocks still need a net size - // benefit. + // Skip tiny blocks to avoid codec and descriptor overhead. private static final int MIN_COMPRESSION_SIZE = 4096; // Conservative allowance for the additional Thrift descriptor and list field on the wire. private static final int DESCRIPTOR_BYTES = 32; @@ -91,34 +90,40 @@ public TSyncLogEntriesReq getRequest() { private void retain(long bytes) { long released = reserved.getAndSet(bytes) - bytes; - if (released > 0) IoTConsensusMemoryManager.getInstance().freeLogCompression(released); + if (released > 0) { + IoTConsensusMemoryManager.getInstance().freeLogCompression(released); + } } @Override public void close() { long bytes = reserved.getAndSet(0); - if (bytes > 0) IoTConsensusMemoryManager.getInstance().freeLogCompression(bytes); + if (bytes > 0) { + IoTConsensusMemoryManager.getInstance().freeLogCompression(bytes); + } } } public static EncodedRequest compress( TSyncLogEntriesReq original, TLogCompressionType codec, int maxDecodedBytes) throws IOException { - long buffers = 0, rawSize = 0; + long buffers = 0; + long rawSize = 0; boolean hasEligibleBuffer = false; for (TLogEntry entry : original.getLogEntries()) { for (ByteBuffer buffer : entry.getData()) { - int size = buffer.remaining(); - rawSize += size; - if (size >= MIN_COMPRESSION_SIZE) { + int uncompressedLength = buffer.remaining(); + rawSize += uncompressedLength; + if (uncompressedLength >= MIN_COMPRESSION_SIZE) { hasEligibleBuffer = true; } buffers++; } } // Preserve the original path for oversized legal entries and when scratch memory is scarce. - if (codec == TLogCompressionType.NONE || !hasEligibleBuffer || rawSize > maxDecodedBytes) + if (codec == TLogCompressionType.NONE || !hasEligibleBuffer || rawSize > maxDecodedBytes) { return new EncodedRequest(original, 0, rawSize, false); + } // Charge descriptor/list/entry overhead first, then one block's working arrays at a time. // Retain only actual compact outputs between blocks, not a whole-batch worst-case estimate. long budget = buffers * 256 + original.getLogEntriesSize() * 128L; @@ -133,37 +138,42 @@ public static EncodedRequest compress( for (TLogEntry entry : original.getLogEntries()) { List data = new ArrayList<>(entry.getDataSize()); List info = new ArrayList<>(entry.getDataSize()); - long raw = 0, wire = 0, arrays = 0; + long raw = 0; + long wire = 0; + long arrays = 0; boolean compressed = false; for (ByteBuffer buffer : entry.getData()) { - int size = buffer.remaining(); - raw += size; + int uncompressedLength = buffer.remaining(); + raw += uncompressedLength; ByteBuffer encoded = buffer; TLogCompressionType actualCodec = TLogCompressionType.NONE; - if (size >= MIN_COMPRESSION_SIZE) { + if (uncompressedLength >= MIN_COMPRESSION_SIZE) { // Includes an optional input copy, codec-bound output and compact output, with // allowance for array headers and codec bounds on very small buffers. - long workspace = 4L * size + 256; + long workspace = 4L * uncompressedLength + 256; long before = result.reserved.get(); if (IoTConsensusMemoryManager.getInstance().reserveLogCompression(workspace)) { result.reserved.addAndGet(workspace); byte[] input = buffer.hasArray() ? buffer.array() : bytes(buffer); int offset = buffer.hasArray() ? buffer.arrayOffset() + buffer.position() : 0; - int bound = + int maxCompressedLength = codec == TLogCompressionType.ZSTD - ? (int) Zstd.compressBound(size) - : compressor(codec).getMaxBytesForCompression(size); - byte[] output = new byte[bound]; - int length = + ? (int) Zstd.compressBound(uncompressedLength) + : compressor(codec).getMaxBytesForCompression(uncompressedLength); + byte[] output = new byte[maxCompressedLength]; + int compressedLength = codec == TLogCompressionType.ZSTD ? (int) - Zstd.compressByteArray(output, 0, output.length, input, offset, size, 1) - : compressor(codec).compress(input, offset, size, output); - if (length < 0) throw invalid(Failure.CODEC, size, length); - if ((long) length + DESCRIPTOR_BYTES < size) { - encoded = ByteBuffer.wrap(Arrays.copyOf(output, length)); + Zstd.compressByteArray( + output, 0, output.length, input, offset, uncompressedLength, 1) + : compressor(codec).compress(input, offset, uncompressedLength, output); + if (compressedLength < 0) { + throw invalid(Failure.CODEC, uncompressedLength, compressedLength); + } + if ((long) compressedLength + DESCRIPTOR_BYTES < uncompressedLength) { + encoded = ByteBuffer.wrap(Arrays.copyOf(output, compressedLength)); actualCodec = codec; - arrays += length; + arrays += compressedLength; compressed = true; } result.retain( @@ -174,7 +184,7 @@ public static EncodedRequest compress( } } data.add(encoded); - info.add(new TLogBufferCompression(actualCodec, size)); + info.add(new TLogBufferCompression(actualCodec, uncompressedLength)); wire += encoded.remaining(); } // Raw descriptors in a mixed WAL entry cost space too. @@ -188,9 +198,10 @@ public static EncodedRequest compress( result.retain(result.reserved.get() - arrays); } } - if (changed) + if (changed) { result.request = new TSyncLogEntriesReq(original.getPeerId(), original.getConsensusGroupId(), entries); + } result.retain(retained); return result; } catch (IOException | RuntimeException e) { @@ -207,25 +218,39 @@ public static void decompress(TSyncLogEntriesReq request, int maxDecodedBytes) for (TLogEntry entry : request.getLogEntries()) { if (entry.isSetCompressionInfo()) { encoded = true; - if (entry.getCompressionInfoSize() != entry.getDataSize()) + if (entry.getCompressionInfoSize() != entry.getDataSize()) { throw invalid(Failure.DESCRIPTORS, entry.getDataSize(), entry.getCompressionInfoSize()); + } for (int i = 0; i < entry.getDataSize(); i++) { TLogBufferCompression info = entry.getCompressionInfo().get(i); int size = info.getUncompressedSize(); - if (size < 0) throw invalid(Failure.LENGTH, 0, size); - if (info.getCodec() == null) throw invalid(Failure.CODEC, 0, -1); + if (size < 0) { + throw invalid(Failure.LENGTH, 0, size); + } + if (info.getCodec() == null) { + throw invalid(Failure.CODEC, 0, -1); + } if (info.getCodec() == TLogCompressionType.NONE - && size != entry.getData().get(i).remaining()) + && size != entry.getData().get(i).remaining()) { throw invalid(Failure.LENGTH, size, entry.getData().get(i).remaining()); + } total += size; - if (total > maxDecodedBytes) throw invalid(Failure.LIMIT, maxDecodedBytes, total); + if (total > maxDecodedBytes) { + throw invalid(Failure.LIMIT, maxDecodedBytes, total); + } } } else { - for (ByteBuffer buffer : entry.getData()) total += buffer.remaining(); + for (ByteBuffer buffer : entry.getData()) { + total += buffer.remaining(); + } } } - if (!encoded) return; - if (total > maxDecodedBytes) throw invalid(Failure.LIMIT, maxDecodedBytes, total); + if (!encoded) { + return; + } + if (total > maxDecodedBytes) { + throw invalid(Failure.LIMIT, maxDecodedBytes, total); + } List decoded = new ArrayList<>(request.getLogEntriesSize()); for (TLogEntry entry : request.getLogEntries()) { if (!entry.isSetCompressionInfo()) { @@ -249,7 +274,9 @@ public static void decompress(TSyncLogEntriesReq request, int maxDecodedBytes) failure.initCause(e); throw failure; } - if (actual != output.length) throw invalid(Failure.LENGTH, output.length, actual); + if (actual != output.length) { + throw invalid(Failure.LENGTH, output.length, actual); + } buffer = ByteBuffer.wrap(output); } data.add(buffer); @@ -264,8 +291,12 @@ public static void decompress(TSyncLogEntriesReq request, int maxDecodedBytes) private static TLogEntry copy(TLogEntry source, List data) { TLogEntry entry = new TLogEntry(data, source.getSearchIndex(), source.isFromWAL(), source.getMemorySize()); - if (source.isSetRoutingEpoch()) entry.setRoutingEpoch(source.getRoutingEpoch()); - if (source.isSetPhysicalTime()) entry.setPhysicalTime(source.getPhysicalTime()); + if (source.isSetRoutingEpoch()) { + entry.setRoutingEpoch(source.getRoutingEpoch()); + } + if (source.isSetPhysicalTime()) { + entry.setPhysicalTime(source.getPhysicalTime()); + } return entry; } From 434292b65c5ed49d8f2bd97f10f61b9f50d07b00 Mon Sep 17 00:00:00 2001 From: 761417898 <761417898@qq.com> Date: Fri, 13 Nov 2026 17:56:59 +0800 Subject: [PATCH 5/5] Reuse TsFile compression types and default codecs for IoTConsensus logs --- iotdb-core/consensus/pom.xml | 4 -- .../consensus/i18n/IoTConsensusMessages.java | 2 + .../consensus/i18n/IoTConsensusMessages.java | 2 + .../consensus/config/IoTConsensusConfig.java | 27 ++++++++--- .../iot/logdispatcher/LogCompression.java | 45 ++++++++++++------- .../iot/logdispatcher/LogDispatcher.java | 4 +- .../config/IoTConsensusConfigTest.java | 37 +++++++++++++++ .../iot/CompressedReplicationTest.java | 8 ++-- .../iot/logdispatcher/LogCompressionTest.java | 25 +++++++---- .../LogDispatcherCompressionTest.java | 4 +- .../org/apache/iotdb/db/conf/IoTDBConfig.java | 7 ++- .../apache/iotdb/db/conf/IoTDBDescriptor.java | 16 +++++-- .../conf/iotdb-system.properties.template | 3 +- 13 files changed, 131 insertions(+), 53 deletions(-) create mode 100644 iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/config/IoTConsensusConfigTest.java diff --git a/iotdb-core/consensus/pom.xml b/iotdb-core/consensus/pom.xml index 922b68801f2fc..7130af748289f 100644 --- a/iotdb-core/consensus/pom.xml +++ b/iotdb-core/consensus/pom.xml @@ -114,10 +114,6 @@ com.google.code.findbugs jsr305 - - com.github.luben - zstd-jni - org.apache.tsfile tsfile diff --git a/iotdb-core/consensus/src/main/i18n/en/org/apache/iotdb/consensus/i18n/IoTConsensusMessages.java b/iotdb-core/consensus/src/main/i18n/en/org/apache/iotdb/consensus/i18n/IoTConsensusMessages.java index a18447f3744c5..023a6c1f0a6d5 100644 --- a/iotdb-core/consensus/src/main/i18n/en/org/apache/iotdb/consensus/i18n/IoTConsensusMessages.java +++ b/iotdb-core/consensus/src/main/i18n/en/org/apache/iotdb/consensus/i18n/IoTConsensusMessages.java @@ -28,6 +28,8 @@ public final class IoTConsensusMessages { public static final String EXCEPTION_INVALID_IOTCONSENSUS_COMPRESSED_LOG_ARG_EXPECTED_ARG_ACTUAL_ARG_4986175E = "Invalid IoTConsensus compressed log (%s, expected %d, actual %d)"; + public static final String EXCEPTION_UNSUPPORTED_IOTCONSENSUS_LOG_COMPRESSION_TYPE_ARG_4008DA9A = "Unsupported IoTConsensus log compression type: %s"; + private IoTConsensusMessages() {} // ===================== IoTConsensus lifecycle ===================== diff --git a/iotdb-core/consensus/src/main/i18n/zh/org/apache/iotdb/consensus/i18n/IoTConsensusMessages.java b/iotdb-core/consensus/src/main/i18n/zh/org/apache/iotdb/consensus/i18n/IoTConsensusMessages.java index 1d915b9af66cb..5243af947a0bf 100644 --- a/iotdb-core/consensus/src/main/i18n/zh/org/apache/iotdb/consensus/i18n/IoTConsensusMessages.java +++ b/iotdb-core/consensus/src/main/i18n/zh/org/apache/iotdb/consensus/i18n/IoTConsensusMessages.java @@ -27,6 +27,8 @@ public final class IoTConsensusMessages { public static final String EXCEPTION_INVALID_IOTCONSENSUS_COMPRESSED_LOG_ARG_EXPECTED_ARG_ACTUAL_ARG_4986175E = "无效的 IoTConsensus 压缩日志(%s,预期 %d,实际 %d)"; + public static final String EXCEPTION_UNSUPPORTED_IOTCONSENSUS_LOG_COMPRESSION_TYPE_ARG_4008DA9A = "不支持的 IoTConsensus 日志压缩类型:%s"; + private IoTConsensusMessages() {} // ===================== IoTConsensus 生命周期 ===================== diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/config/IoTConsensusConfig.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/config/IoTConsensusConfig.java index 742992bba1757..1cd7cb074c23c 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/config/IoTConsensusConfig.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/config/IoTConsensusConfig.java @@ -22,7 +22,9 @@ import org.apache.iotdb.commons.client.property.ClientPoolProperty.DefaultProperty; import org.apache.iotdb.commons.memory.AtomicLongMemoryBlock; import org.apache.iotdb.commons.memory.IMemoryBlock; -import org.apache.iotdb.consensus.iot.thrift.TLogCompressionType; +import org.apache.iotdb.consensus.i18n.IoTConsensusMessages; + +import org.apache.tsfile.file.metadata.enums.CompressionType; import java.util.Optional; import java.util.concurrent.TimeUnit; @@ -311,7 +313,7 @@ public static class Replication { private final int maxLogEntriesNumPerBatch; private final int maxSizePerBatch; - private final TLogCompressionType logCompressionType; + private final CompressionType logCompressionType; private final int maxPendingBatchesNum; private final int maxQueueLength; @@ -332,7 +334,7 @@ public static class Replication { private Replication( int maxLogEntriesNumPerBatch, int maxSizePerBatch, - TLogCompressionType logCompressionType, + CompressionType logCompressionType, int maxPendingBatchesNum, int maxQueueLength, long maxWaitingTimeForWaitBatchInMs, @@ -376,7 +378,7 @@ public int getMaxSizePerBatch() { return maxSizePerBatch; } - public TLogCompressionType getLogCompressionType() { + public CompressionType getLogCompressionType() { return logCompressionType; } @@ -448,7 +450,7 @@ public static class Builder { private int maxLogEntriesNumPerBatch = 1024; private int maxSizePerBatch = 16 * 1024 * 1024; - private TLogCompressionType logCompressionType = TLogCompressionType.NONE; + private CompressionType logCompressionType = CompressionType.UNCOMPRESSED; private int maxPendingBatchesNum = 5; private int maxQueueLength = 4096; private long maxWaitingTimeForWaitBatchInMs = 10 * 1000L; @@ -482,7 +484,20 @@ public Builder setMaxSizePerBatch(int maxSizePerBatch) { return this; } - public Builder setLogCompressionType(TLogCompressionType type) { + public Builder setLogCompressionType(CompressionType type) { + switch (type) { + case UNCOMPRESSED: + case LZ4: + case SNAPPY: + case ZSTD: + break; + default: + throw new IllegalArgumentException( + String.format( + IoTConsensusMessages + .EXCEPTION_UNSUPPORTED_IOTCONSENSUS_LOG_COMPRESSION_TYPE_ARG_4008DA9A, + type)); + } this.logCompressionType = type; return this; } diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompression.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompression.java index 1aa770121d2f9..28003fc8b3f3f 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompression.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompression.java @@ -25,8 +25,6 @@ import org.apache.iotdb.consensus.iot.thrift.TLogEntry; import org.apache.iotdb.consensus.iot.thrift.TSyncLogEntriesReq; -import com.github.luben.zstd.Zstd; -import com.github.luben.zstd.ZstdException; import org.apache.tsfile.compress.ICompressor; import org.apache.tsfile.compress.IUnCompressor; import org.apache.tsfile.file.metadata.enums.CompressionType; @@ -46,6 +44,7 @@ public final class LogCompression { private static final int DESCRIPTOR_BYTES = 32; private static final ICompressor LZ4 = ICompressor.getCompressor(CompressionType.LZ4); private static final ICompressor SNAPPY = ICompressor.getCompressor(CompressionType.SNAPPY); + private static final ICompressor ZSTD = ICompressor.getCompressor(CompressionType.ZSTD); private static final IUnCompressor LZ4_DECODER = IUnCompressor.getUnCompressor(CompressionType.LZ4); private static final IUnCompressor SNAPPY_DECODER = @@ -105,8 +104,8 @@ public void close() { } public static EncodedRequest compress( - TSyncLogEntriesReq original, TLogCompressionType codec, int maxDecodedBytes) - throws IOException { + TSyncLogEntriesReq original, CompressionType codec, int maxDecodedBytes) throws IOException { + TLogCompressionType wireCodec = toWireCodec(codec); long buffers = 0; long rawSize = 0; boolean hasEligibleBuffer = false; @@ -121,7 +120,7 @@ public static EncodedRequest compress( } } // Preserve the original path for oversized legal entries and when scratch memory is scarce. - if (codec == TLogCompressionType.NONE || !hasEligibleBuffer || rawSize > maxDecodedBytes) { + if (codec == CompressionType.UNCOMPRESSED || !hasEligibleBuffer || rawSize > maxDecodedBytes) { return new EncodedRequest(original, 0, rawSize, false); } // Charge descriptor/list/entry overhead first, then one block's working arrays at a time. @@ -157,22 +156,16 @@ public static EncodedRequest compress( byte[] input = buffer.hasArray() ? buffer.array() : bytes(buffer); int offset = buffer.hasArray() ? buffer.arrayOffset() + buffer.position() : 0; int maxCompressedLength = - codec == TLogCompressionType.ZSTD - ? (int) Zstd.compressBound(uncompressedLength) - : compressor(codec).getMaxBytesForCompression(uncompressedLength); + compressor(codec).getMaxBytesForCompression(uncompressedLength); byte[] output = new byte[maxCompressedLength]; int compressedLength = - codec == TLogCompressionType.ZSTD - ? (int) - Zstd.compressByteArray( - output, 0, output.length, input, offset, uncompressedLength, 1) - : compressor(codec).compress(input, offset, uncompressedLength, output); + compressor(codec).compress(input, offset, uncompressedLength, output); if (compressedLength < 0) { throw invalid(Failure.CODEC, uncompressedLength, compressedLength); } if ((long) compressedLength + DESCRIPTOR_BYTES < uncompressedLength) { encoded = ByteBuffer.wrap(Arrays.copyOf(output, compressedLength)); - actualCodec = codec; + actualCodec = wireCodec; arrays += compressedLength; compressed = true; } @@ -269,7 +262,8 @@ public static void decompress(TSyncLogEntriesReq request, int maxDecodedBytes) try { actual = decoder(info.getCodec()).uncompress(input, offset, buffer.remaining(), output, 0); - } catch (ZstdException e) { + } catch (RuntimeException e) { + // TsFile codecs may report malformed input through unchecked exceptions. IOException failure = invalid(Failure.CODEC, output.length, buffer.remaining()); failure.initCause(e); throw failure; @@ -307,14 +301,31 @@ private static byte[] bytes(ByteBuffer buffer) { return bytes; } - private static ICompressor compressor(TLogCompressionType codec) throws IOException { + private static TLogCompressionType toWireCodec(CompressionType codec) throws IOException { + switch (codec) { + case UNCOMPRESSED: + return TLogCompressionType.NONE; + case LZ4: + return TLogCompressionType.LZ4; + case SNAPPY: + return TLogCompressionType.SNAPPY; + case ZSTD: + return TLogCompressionType.ZSTD; + default: + throw invalid(Failure.CODEC, 0, codec.serialize()); + } + } + + private static ICompressor compressor(CompressionType codec) throws IOException { switch (codec) { case LZ4: return LZ4; case SNAPPY: return SNAPPY; + case ZSTD: + return ZSTD; default: - throw invalid(Failure.CODEC, 0, codec.getValue()); + throw invalid(Failure.CODEC, 0, codec.serialize()); } } diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcher.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcher.java index 972265a4fc09b..18e73aa2c6170 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcher.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcher.java @@ -35,12 +35,12 @@ import org.apache.iotdb.consensus.iot.client.DispatchLogHandler; import org.apache.iotdb.consensus.iot.log.ConsensusReqReader; import org.apache.iotdb.consensus.iot.log.GetConsensusReqReaderPlan; -import org.apache.iotdb.consensus.iot.thrift.TLogCompressionType; import org.apache.iotdb.consensus.iot.thrift.TLogEntry; import org.apache.iotdb.consensus.iot.thrift.TSyncLogEntriesReq; import org.apache.iotdb.consensus.iot.thrift.TSyncLogEntriesRes; import org.apache.thrift.async.AsyncMethodCallback; +import org.apache.tsfile.file.metadata.enums.CompressionType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -600,7 +600,7 @@ public void sendBatchAsync(Batch batch, DispatchLogHandler handler) { batch.getStartIndex(), batch.getEndIndex(), peer.getGroupId().convertToTConsensusGroupId()); - if (config.getReplication().getLogCompressionType() == TLogCompressionType.NONE) { + if (config.getReplication().getLogCompressionType() == CompressionType.UNCOMPRESSED) { clientManager.borrowClient(peer.getEndpoint()).syncLogEntries(req, handler); return; } diff --git a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/config/IoTConsensusConfigTest.java b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/config/IoTConsensusConfigTest.java new file mode 100644 index 0000000000000..2c05a9f92bb6f --- /dev/null +++ b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/config/IoTConsensusConfigTest.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.consensus.config; + +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.junit.Test; + +import static org.junit.Assert.assertThrows; + +public class IoTConsensusConfigTest { + @Test + public void testRejectsCompressionTypesUnsupportedByWireProtocol() { + for (CompressionType type : + new CompressionType[] {CompressionType.GZIP, CompressionType.LZMA2}) { + assertThrows( + IllegalArgumentException.class, + () -> IoTConsensusConfig.Replication.newBuilder().setLogCompressionType(type)); + } + } +} diff --git a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/CompressedReplicationTest.java b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/CompressedReplicationTest.java index f01178d24776d..42190e85f3977 100644 --- a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/CompressedReplicationTest.java +++ b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/CompressedReplicationTest.java @@ -29,10 +29,10 @@ import org.apache.iotdb.consensus.common.request.IndexedConsensusRequest; import org.apache.iotdb.consensus.config.ConsensusConfig; import org.apache.iotdb.consensus.config.IoTConsensusConfig; -import org.apache.iotdb.consensus.iot.thrift.TLogCompressionType; import org.apache.iotdb.consensus.iot.util.TestEntry; import org.apache.iotdb.consensus.iot.util.TestStateMachine; +import org.apache.tsfile.file.metadata.enums.CompressionType; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -59,12 +59,10 @@ public class CompressedReplicationTest { @Parameterized.Parameters(name = "{0}") public static Object[] codecs() { - return new Object[] { - TLogCompressionType.LZ4, TLogCompressionType.SNAPPY, TLogCompressionType.ZSTD - }; + return new Object[] {CompressionType.LZ4, CompressionType.SNAPPY, CompressionType.ZSTD}; } - @Parameterized.Parameter public TLogCompressionType codec; + @Parameterized.Parameter public CompressionType codec; private static final int COUNT = 32; private static final int BYTES = 65536; diff --git a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompressionTest.java b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompressionTest.java index 4d8ce6103a182..29440af69c1db 100644 --- a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompressionTest.java +++ b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompressionTest.java @@ -40,6 +40,7 @@ import org.apache.thrift.TDeserializer; import org.apache.thrift.TSerializer; +import org.apache.tsfile.file.metadata.enums.CompressionType; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -73,13 +74,19 @@ @RunWith(Parameterized.class) public class LogCompressionTest { @Parameterized.Parameters(name = "{0}") - public static Object[] codecs() { - return new Object[] { - TLogCompressionType.LZ4, TLogCompressionType.SNAPPY, TLogCompressionType.ZSTD + public static Object[][] codecs() { + return new Object[][] { + {CompressionType.LZ4, TLogCompressionType.LZ4}, + {CompressionType.SNAPPY, TLogCompressionType.SNAPPY}, + {CompressionType.ZSTD, TLogCompressionType.ZSTD} }; } - @Parameterized.Parameter public TLogCompressionType codec; + @Parameterized.Parameter public CompressionType codec; + + @Parameterized.Parameter(1) + public TLogCompressionType expectedWireCodec; + private IMemoryBlock previous; private final IoTConsensusMemoryManager memory = IoTConsensusMemoryManager.getInstance(); private static final int LIMIT = 8 * 1024 * 1024; @@ -149,7 +156,7 @@ public void testRoundTripThriftAndRetryOwnership() throws Exception { assertTrue(encoded.getEncodedBytes() < encoded.getRawBytes()); assertTrue(memory.getMemoryBlock().getUsedMemoryInBytes() > 0); assertEquals( - codec, + expectedWireCodec, encoded.getRequest().getLogEntries().get(0).getCompressionInfo().get(0).getCodec()); assertEquals( TLogCompressionType.NONE, @@ -186,7 +193,7 @@ public void testFixedMinimumSizeBoundary() throws Exception { } else { assertNotSame(raw, encoded.getRequest()); assertEquals( - codec, + expectedWireCodec, encoded.getRequest().getLogEntries().get(0).getCompressionInfo().get(0).getCodec()); } TSyncLogEntriesReq received = encoded.getRequest().deepCopy(); @@ -202,7 +209,7 @@ public void testFixedMinimumSizeBoundary() throws Exception { public void testAdaptiveAndMemoryFallback() throws Exception { TSyncLogEntriesReq raw = request(); try (LogCompression.EncodedRequest result = - LogCompression.compress(raw, TLogCompressionType.NONE, LIMIT)) { + LogCompression.compress(raw, CompressionType.UNCOMPRESSED, LIMIT)) { assertSame(raw, result.getRequest()); } byte[] random = new byte[16384]; @@ -389,13 +396,13 @@ public void testMemoryPressureFallsBackPerBlockAndKeepsOtherBlocksCompressed() t assertNotSame(raw, encoded.getRequest()); assertTrue(encoded.isMemoryLimited()); assertEquals( - codec, + expectedWireCodec, encoded.getRequest().getLogEntries().get(0).getCompressionInfo().get(0).getCodec()); assertEquals( TLogCompressionType.NONE, encoded.getRequest().getLogEntries().get(0).getCompressionInfo().get(1).getCodec()); assertEquals( - codec, + expectedWireCodec, encoded.getRequest().getLogEntries().get(0).getCompressionInfo().get(2).getCodec()); TSyncLogEntriesReq received = encoded.getRequest().deepCopy(); LogCompression.decompress(received, LIMIT); diff --git a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcherCompressionTest.java b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcherCompressionTest.java index 81ae68231c2cc..24d9607167f9b 100644 --- a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcherCompressionTest.java +++ b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcherCompressionTest.java @@ -31,7 +31,6 @@ import org.apache.iotdb.consensus.iot.IoTConsensusServerImpl; import org.apache.iotdb.consensus.iot.client.AsyncIoTConsensusServiceClient; import org.apache.iotdb.consensus.iot.client.DispatchLogHandler; -import org.apache.iotdb.consensus.iot.thrift.TLogCompressionType; import org.apache.iotdb.consensus.iot.thrift.TLogEntry; import org.apache.iotdb.consensus.iot.thrift.TSyncLogEntriesReq; import org.apache.iotdb.consensus.iot.thrift.TSyncLogEntriesRes; @@ -39,6 +38,7 @@ import org.apache.thrift.TException; import org.apache.thrift.async.AsyncMethodCallback; +import org.apache.tsfile.file.metadata.enums.CompressionType; import org.junit.After; import org.junit.Before; import org.junit.Rule; @@ -86,7 +86,7 @@ public void setUp() throws Exception { IoTConsensusConfig.newBuilder() .setReplication( IoTConsensusConfig.Replication.newBuilder() - .setLogCompressionType(TLogCompressionType.LZ4) + .setLogCompressionType(CompressionType.LZ4) .build()) .build(); IoTConsensusServerImpl impl = mock(IoTConsensusServerImpl.class); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 41dc78446d863..bd2895f887642 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -30,7 +30,6 @@ import org.apache.iotdb.commons.pipe.config.PipeConfig; import org.apache.iotdb.commons.utils.FileUtils; import org.apache.iotdb.consensus.ConsensusFactory; -import org.apache.iotdb.consensus.iot.thrift.TLogCompressionType; import org.apache.iotdb.db.exception.LoadConfigurationException; import org.apache.iotdb.db.i18n.DataNodeMiscMessages; import org.apache.iotdb.db.protocol.thrift.impl.ClientRPCServiceImpl; @@ -1131,7 +1130,7 @@ public class IoTDBConfig implements DefaultEncodingProvider { // IoTConsensus Config private int maxLogEntriesNumPerBatch = 1024; private int maxSizePerBatch = 16 * 1024 * 1024; - private TLogCompressionType iotLogCompressionType = TLogCompressionType.NONE; + private CompressionType iotLogCompressionType = CompressionType.UNCOMPRESSED; private int maxPendingBatchesNum = 5; private double maxMemoryRatioForQueue = 0.6; private long regionMigrationSpeedLimitBytesPerSecond = 48 * 1024 * 1024L; @@ -1311,11 +1310,11 @@ public int getMaxLogEntriesNumPerBatch() { return maxLogEntriesNumPerBatch; } - public TLogCompressionType getIotLogCompressionType() { + public CompressionType getIotLogCompressionType() { return iotLogCompressionType; } - public void setIotLogCompressionType(TLogCompressionType type) { + public void setIotLogCompressionType(CompressionType type) { iotLogCompressionType = type; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index ce526ff448f59..968988ca48f8f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -39,7 +39,6 @@ import org.apache.iotdb.confignode.rpc.thrift.TGlobalConfig; import org.apache.iotdb.confignode.rpc.thrift.TRatisConfig; import org.apache.iotdb.consensus.config.IoTConsensusV2Config; -import org.apache.iotdb.consensus.iot.thrift.TLogCompressionType; import org.apache.iotdb.db.conf.rest.IoTDBRestServiceDescriptor; import org.apache.iotdb.db.consensus.DataRegionConsensusImpl; import org.apache.iotdb.db.i18n.DataNodeMiscMessages; @@ -1197,8 +1196,19 @@ public void loadProperties(TrimProperties properties) throws BadNodeUrlException // Sending policy is a startup setting. Decoding is always available for in-flight requests. String compression = properties.getProperty("data_region_iot_log_compression", "NONE").trim(); try { - TLogCompressionType type = TLogCompressionType.valueOf(compression.toUpperCase(Locale.ROOT)); - conf.setIotLogCompressionType(type); + String normalizedCompression = compression.toUpperCase(Locale.ROOT); + switch (normalizedCompression) { + case "NONE": + conf.setIotLogCompressionType(CompressionType.UNCOMPRESSED); + break; + case "LZ4": + case "SNAPPY": + case "ZSTD": + conf.setIotLogCompressionType(CompressionType.valueOf(normalizedCompression)); + break; + default: + throw new IllegalArgumentException(); + } } catch (IllegalArgumentException e) { throw new IOException( String.format( diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template index 9e1bbe1fddab6..7ca2aa736317c 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template @@ -1873,7 +1873,8 @@ enable_wal_compression=true ### IoTConsensus Configuration #################### -# IoTConsensus V1 log payload codec: NONE, LZ4, SNAPPY, ZSTD (level 1). +# IoTConsensus V1 log payload codec: NONE, LZ4, SNAPPY, ZSTD. +# Compression uses the TsFile implementation and its default codec settings. # Enable only after every peer has the same supporting version and configuration. # This controls sending; receivers can decode every supported codec even when set to NONE. # Independent of WAL/TsFile compression and dn_rpc_thrift_compression_enable.