From 960f564758a540cde1304f944bbcf32d87fb9b7a Mon Sep 17 00:00:00 2001
From: 761417898 <761417898@qq.com>
Date: Thu, 12 Nov 2026 16:58:28 +0800
Subject: [PATCH 1/5] Support log payload compression in IoTConsensus V1
---
iotdb-core/consensus/pom.xml | 4 +
.../consensus/i18n/IoTConsensusMessages.java | 4 +
.../consensus/i18n/IoTConsensusMessages.java | 4 +
.../consensus/config/IoTConsensusConfig.java | 37 ++
.../iot/IoTConsensusServerMetrics.java | 20 +
.../iot/client/DispatchLogHandler.java | 13 +-
.../IoTConsensusMemoryManager.java | 9 +
.../iot/logdispatcher/LogCompression.java | 332 ++++++++++++++++
.../iot/logdispatcher/LogDispatcher.java | 45 ++-
.../LogDispatcherThreadMetrics.java | 75 ++++
.../IoTConsensusRPCServiceProcessor.java | 14 +
.../iot/CompressedReplicationTest.java | 178 +++++++++
.../iot/client/DispatchLogHandlerTest.java | 89 +++++
.../iot/logdispatcher/LogCompressionTest.java | 364 ++++++++++++++++++
.../LogDispatcherCompressionTest.java | 178 +++++++++
.../LogDispatcherThreadMetricsTest.java | 41 ++
.../iotdb/db/i18n/DataNodeMiscMessages.java | 1 +
.../iotdb/db/i18n/DataNodeMiscMessages.java | 1 +
.../org/apache/iotdb/db/conf/IoTDBConfig.java | 19 +
.../apache/iotdb/db/conf/IoTDBDescriptor.java | 21 +
.../db/consensus/DataRegionConsensusImpl.java | 2 +
.../conf/iotdb-system.properties.template | 14 +
.../commons/service/metric/enums/Metric.java | 2 +
.../src/main/thrift/iotconsensus.thrift | 18 +
24 files changed, 1482 insertions(+), 3 deletions(-)
create mode 100644 iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompression.java
create mode 100644 iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/CompressedReplicationTest.java
create mode 100644 iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompressionTest.java
create mode 100644 iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcherCompressionTest.java
diff --git a/iotdb-core/consensus/pom.xml b/iotdb-core/consensus/pom.xml
index 7130af748289f..922b68801f2fc 100644
--- a/iotdb-core/consensus/pom.xml
+++ b/iotdb-core/consensus/pom.xml
@@ -114,6 +114,10 @@
com.google.code.findbugs
jsr305
+
+ com.github.luben
+ zstd-jni
+
org.apache.tsfile
tsfile
diff --git a/iotdb-core/consensus/src/main/i18n/en/org/apache/iotdb/consensus/i18n/IoTConsensusMessages.java b/iotdb-core/consensus/src/main/i18n/en/org/apache/iotdb/consensus/i18n/IoTConsensusMessages.java
index d07373015a83e..3ebcd8f93e839 100644
--- a/iotdb-core/consensus/src/main/i18n/en/org/apache/iotdb/consensus/i18n/IoTConsensusMessages.java
+++ b/iotdb-core/consensus/src/main/i18n/en/org/apache/iotdb/consensus/i18n/IoTConsensusMessages.java
@@ -25,6 +25,10 @@
*/
public final class IoTConsensusMessages {
+ public static final String EXCEPTION_INVALID_IOTCONSENSUS_LOG_COMPRESSION_MINIMUM_SIZE_ARG_ACEEA255 = "Invalid IoTConsensus log compression minimum size: %d";
+
+ public static final String EXCEPTION_INVALID_IOTCONSENSUS_COMPRESSED_LOG_ARG_EXPECTED_ARG_ACTUAL_ARG_4986175E = "Invalid IoTConsensus compressed log (%s, expected %d, actual %d)";
+
private IoTConsensusMessages() {}
// ===================== IoTConsensus lifecycle =====================
diff --git a/iotdb-core/consensus/src/main/i18n/zh/org/apache/iotdb/consensus/i18n/IoTConsensusMessages.java b/iotdb-core/consensus/src/main/i18n/zh/org/apache/iotdb/consensus/i18n/IoTConsensusMessages.java
index 0d908aa8613de..5b2ddc42f860d 100644
--- a/iotdb-core/consensus/src/main/i18n/zh/org/apache/iotdb/consensus/i18n/IoTConsensusMessages.java
+++ b/iotdb-core/consensus/src/main/i18n/zh/org/apache/iotdb/consensus/i18n/IoTConsensusMessages.java
@@ -24,6 +24,10 @@
*/
public final class IoTConsensusMessages {
+ public static final String EXCEPTION_INVALID_IOTCONSENSUS_LOG_COMPRESSION_MINIMUM_SIZE_ARG_ACEEA255 = "无效的 IoTConsensus 日志压缩最小大小:%d";
+
+ public static final String EXCEPTION_INVALID_IOTCONSENSUS_COMPRESSED_LOG_ARG_EXPECTED_ARG_ACTUAL_ARG_4986175E = "无效的 IoTConsensus 压缩日志(%s,预期 %d,实际 %d)";
+
private IoTConsensusMessages() {}
// ===================== IoTConsensus 生命周期 =====================
diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/config/IoTConsensusConfig.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/config/IoTConsensusConfig.java
index c6ca9a7e0dd5c..c31197f0334eb 100644
--- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/config/IoTConsensusConfig.java
+++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/config/IoTConsensusConfig.java
@@ -22,6 +22,8 @@
import org.apache.iotdb.commons.client.property.ClientPoolProperty.DefaultProperty;
import org.apache.iotdb.commons.memory.AtomicLongMemoryBlock;
import org.apache.iotdb.commons.memory.IMemoryBlock;
+import org.apache.iotdb.consensus.i18n.IoTConsensusMessages;
+import org.apache.iotdb.consensus.iot.thrift.TLogCompressionType;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
@@ -310,6 +312,8 @@ public static class Replication {
private final int maxLogEntriesNumPerBatch;
private final int maxSizePerBatch;
+ private final TLogCompressionType logCompressionType;
+ private final int logCompressionMinSize;
private final int maxPendingBatchesNum;
private final int maxQueueLength;
@@ -330,6 +334,8 @@ public static class Replication {
private Replication(
int maxLogEntriesNumPerBatch,
int maxSizePerBatch,
+ TLogCompressionType logCompressionType,
+ int logCompressionMinSize,
int maxPendingBatchesNum,
int maxQueueLength,
long maxWaitingTimeForWaitBatchInMs,
@@ -347,6 +353,8 @@ private Replication(
long snapshotTransmissionProgressLogIntervalMs) {
this.maxLogEntriesNumPerBatch = maxLogEntriesNumPerBatch;
this.maxSizePerBatch = maxSizePerBatch;
+ this.logCompressionType = logCompressionType;
+ this.logCompressionMinSize = logCompressionMinSize;
this.maxPendingBatchesNum = maxPendingBatchesNum;
this.maxQueueLength = maxQueueLength;
this.maxWaitingTimeForWaitBatchInMs = maxWaitingTimeForWaitBatchInMs;
@@ -372,6 +380,14 @@ public int getMaxSizePerBatch() {
return maxSizePerBatch;
}
+ public TLogCompressionType getLogCompressionType() {
+ return logCompressionType;
+ }
+
+ public int getLogCompressionMinSize() {
+ return logCompressionMinSize;
+ }
+
public int getMaxPendingBatchesNum() {
return maxPendingBatchesNum;
}
@@ -440,6 +456,8 @@ public static class Builder {
private int maxLogEntriesNumPerBatch = 1024;
private int maxSizePerBatch = 16 * 1024 * 1024;
+ private TLogCompressionType logCompressionType = TLogCompressionType.NONE;
+ private int logCompressionMinSize = 4096;
private int maxPendingBatchesNum = 5;
private int maxQueueLength = 4096;
private long maxWaitingTimeForWaitBatchInMs = 10 * 1000L;
@@ -473,6 +491,23 @@ public Builder setMaxSizePerBatch(int maxSizePerBatch) {
return this;
}
+ public Builder setLogCompressionType(TLogCompressionType type) {
+ this.logCompressionType = type;
+ return this;
+ }
+
+ public Builder setLogCompressionMinSize(int size) {
+ if (size < 0) {
+ throw new IllegalArgumentException(
+ String.format(
+ IoTConsensusMessages
+ .EXCEPTION_INVALID_IOTCONSENSUS_LOG_COMPRESSION_MINIMUM_SIZE_ARG_ACEEA255,
+ size));
+ }
+ this.logCompressionMinSize = size;
+ return this;
+ }
+
public Replication.Builder setMaxPendingBatchesNum(int maxPendingBatchesNum) {
this.maxPendingBatchesNum = maxPendingBatchesNum;
return this;
@@ -557,6 +592,8 @@ public Replication build() {
return new Replication(
maxLogEntriesNumPerBatch,
maxSizePerBatch,
+ logCompressionType,
+ logCompressionMinSize,
maxPendingBatchesNum,
maxQueueLength,
maxWaitingTimeForWaitBatchInMs,
diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerMetrics.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerMetrics.java
index 117f971263572..08714ff2ee40a 100644
--- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerMetrics.java
+++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerMetrics.java
@@ -37,6 +37,7 @@ public class IoTConsensusServerMetrics implements IMetricSet {
private Timer offerRequestToQueueTimer = DoNothingMetricManager.DO_NOTHING_TIMER;
private Timer consensusWriteTimer = DoNothingMetricManager.DO_NOTHING_TIMER;
private static final String IOT_RECEIVE_LOG = Metric.IOT_RECEIVE_LOG.toString();
+ private Timer decompressTimer = DoNothingMetricManager.DO_NOTHING_TIMER;
private Timer deserializeTimer = DoNothingMetricManager.DO_NOTHING_TIMER;
private Timer sortTimer = DoNothingMetricManager.DO_NOTHING_TIMER;
private Timer applyTimer = DoNothingMetricManager.DO_NOTHING_TIMER;
@@ -50,6 +51,10 @@ public IoTConsensusServerMetrics(IoTConsensusServerImpl impl) {
private static final String IMPL = "ioTConsensusServerImpl";
+ public void recordDecompressCost(long nanos) {
+ decompressTimer.updateNanos(nanos);
+ }
+
public void recordDeserializeCost(long costTimeInNanos) {
deserializeTimer.updateNanos(costTimeInNanos);
}
@@ -190,6 +195,14 @@ private void bindStageTimer(AbstractMetricService metricService) {
private void bindSyncLogTimer(AbstractMetricService metricService) {
// bind sync log timers
+ decompressTimer =
+ metricService.getOrCreateTimer(
+ IOT_RECEIVE_LOG,
+ MetricLevel.IMPORTANT,
+ Tag.STAGE.toString(),
+ "decompress",
+ Tag.REGION.toString(),
+ impl.getConsensusGroupId());
deserializeTimer =
metricService.getOrCreateTimer(
IOT_RECEIVE_LOG,
@@ -319,6 +332,13 @@ private void unbindStageTimer(AbstractMetricService metricService) {
private void unbindSyncLogTimer(AbstractMetricService metricService) {
// unbind sync log timers
+ metricService.remove(
+ MetricType.TIMER,
+ IOT_RECEIVE_LOG,
+ Tag.STAGE.toString(),
+ "decompress",
+ Tag.REGION.toString(),
+ impl.getConsensusGroupId());
metricService.remove(
MetricType.TIMER,
IOT_RECEIVE_LOG,
diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/client/DispatchLogHandler.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/client/DispatchLogHandler.java
index 615ecd6b07c46..fe264ef681f54 100644
--- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/client/DispatchLogHandler.java
+++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/client/DispatchLogHandler.java
@@ -39,6 +39,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
@@ -53,6 +54,7 @@ public class DispatchLogHandler implements AsyncMethodCallback RetryUtils.needRetryForWrite(status.getCode()))) {
@@ -138,7 +149,7 @@ public void onError(Exception exception) {
actualCause.toString());
}
// skip TApplicationException caused by follower
- if (actualCause instanceof TApplicationException) {
+ if (actualCause instanceof TApplicationException && !hasCompressedAttempt) {
completeBatch(batch);
LOGGER.warn(IoTConsensusMessages.SKIP_RETRY_TAPPLICATION_EXCEPTION, batch);
logDispatcherThreadMetrics.recordSyncLogTimePerRequest(System.nanoTime() - createTime);
diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/IoTConsensusMemoryManager.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/IoTConsensusMemoryManager.java
index 4d57b32e50699..c1a1d4277d3ff 100644
--- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/IoTConsensusMemoryManager.java
+++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/IoTConsensusMemoryManager.java
@@ -129,6 +129,15 @@ public void free(IndexedConsensusRequest request) {
}
}
+ /** Non-blocking reservation for the additional arrays held by an encoded RPC attempt. */
+ public boolean reserveLogCompression(long size) {
+ return reserve(size, false);
+ }
+
+ public void freeLogCompression(long size) {
+ free(size, false);
+ }
+
public void free(Batch batch) {
free(batch.getMemorySize(), false);
if (logger.isDebugEnabled()) {
diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompression.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompression.java
new file mode 100644
index 0000000000000..1162873f2c4de
--- /dev/null
+++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompression.java
@@ -0,0 +1,332 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.consensus.iot.logdispatcher;
+
+import org.apache.iotdb.consensus.i18n.IoTConsensusMessages;
+import org.apache.iotdb.consensus.iot.thrift.TLogBufferCompression;
+import org.apache.iotdb.consensus.iot.thrift.TLogCompressionType;
+import org.apache.iotdb.consensus.iot.thrift.TLogEntry;
+import org.apache.iotdb.consensus.iot.thrift.TSyncLogEntriesReq;
+
+import com.github.luben.zstd.Zstd;
+import com.github.luben.zstd.ZstdException;
+import org.apache.tsfile.compress.ICompressor;
+import org.apache.tsfile.compress.IUnCompressor;
+import org.apache.tsfile.file.metadata.enums.CompressionType;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.zip.CRC32C;
+
+/** Wire-only encoding: the original batch and its buffer positions remain available for retries. */
+public final class LogCompression {
+ // Conservative allowance for the additional Thrift descriptor and list field on the wire.
+ private static final int DESCRIPTOR_BYTES = 32;
+ private static final ICompressor LZ4 = ICompressor.getCompressor(CompressionType.LZ4);
+ private static final ICompressor SNAPPY = ICompressor.getCompressor(CompressionType.SNAPPY);
+ private static final IUnCompressor LZ4_DECODER =
+ IUnCompressor.getUnCompressor(CompressionType.LZ4);
+ private static final IUnCompressor SNAPPY_DECODER =
+ IUnCompressor.getUnCompressor(CompressionType.SNAPPY);
+ private static final IUnCompressor ZSTD_DECODER =
+ IUnCompressor.getUnCompressor(CompressionType.ZSTD);
+
+ private LogCompression() {}
+
+ /** Accounts for arrays throughout the async attempt, including synchronous send failure. */
+ public static final class EncodedRequest implements AutoCloseable {
+ private TSyncLogEntriesReq request;
+ private final AtomicLong reserved;
+ private final long rawBytes;
+ private long encodedBytes;
+ private boolean memoryLimited;
+
+ private EncodedRequest(
+ TSyncLogEntriesReq request, long reserved, long rawBytes, boolean memoryLimited) {
+ this.request = request;
+ this.reserved = new AtomicLong(reserved);
+ this.rawBytes = rawBytes;
+ this.encodedBytes = rawBytes;
+ this.memoryLimited = memoryLimited;
+ }
+
+ public long getRawBytes() {
+ return rawBytes;
+ }
+
+ public long getEncodedBytes() {
+ return encodedBytes;
+ }
+
+ public boolean isMemoryLimited() {
+ return memoryLimited;
+ }
+
+ public TSyncLogEntriesReq getRequest() {
+ return request;
+ }
+
+ private void retain(long bytes) {
+ long released = reserved.getAndSet(bytes) - bytes;
+ if (released > 0) IoTConsensusMemoryManager.getInstance().freeLogCompression(released);
+ }
+
+ @Override
+ public void close() {
+ long bytes = reserved.getAndSet(0);
+ if (bytes > 0) IoTConsensusMemoryManager.getInstance().freeLogCompression(bytes);
+ }
+ }
+
+ public static EncodedRequest compress(
+ TSyncLogEntriesReq original, TLogCompressionType codec, int minSize, int maxDecodedBytes)
+ throws IOException {
+ long buffers = 0, rawSize = 0;
+ boolean hasEligibleBuffer = false;
+ for (TLogEntry entry : original.getLogEntries()) {
+ for (ByteBuffer buffer : entry.getData()) {
+ int size = buffer.remaining();
+ rawSize += size;
+ if (size >= minSize && size > 0) {
+ hasEligibleBuffer = true;
+ }
+ buffers++;
+ }
+ }
+ // Preserve the original path for oversized legal entries and when scratch memory is scarce.
+ if (codec == TLogCompressionType.NONE || !hasEligibleBuffer || rawSize > maxDecodedBytes)
+ return new EncodedRequest(original, 0, rawSize, false);
+ // Charge descriptor/list/entry overhead first, then one block's working arrays at a time.
+ // Retain only actual compact outputs between blocks, not a whole-batch worst-case estimate.
+ long budget = buffers * 256 + original.getLogEntriesSize() * 128L;
+ if (!IoTConsensusMemoryManager.getInstance().reserveLogCompression(budget)) {
+ return new EncodedRequest(original, 0, rawSize, true);
+ }
+ EncodedRequest result = new EncodedRequest(original, budget, rawSize, false);
+ try {
+ List entries = new ArrayList<>(original.getLogEntriesSize());
+ long retained = 0;
+ boolean changed = false;
+ for (TLogEntry entry : original.getLogEntries()) {
+ List data = new ArrayList<>(entry.getDataSize());
+ List info = new ArrayList<>(entry.getDataSize());
+ long raw = 0, wire = 0, arrays = 0;
+ boolean compressed = false;
+ for (ByteBuffer buffer : entry.getData()) {
+ int size = buffer.remaining();
+ raw += size;
+ ByteBuffer encoded = buffer;
+ TLogCompressionType actualCodec = TLogCompressionType.NONE;
+ if (size >= minSize && size > 0) {
+ // Includes an optional input copy, codec-bound output and compact output, with
+ // allowance for array headers and codec bounds on very small buffers.
+ long workspace = 4L * size + 256;
+ long before = result.reserved.get();
+ if (IoTConsensusMemoryManager.getInstance().reserveLogCompression(workspace)) {
+ result.reserved.addAndGet(workspace);
+ byte[] input = buffer.hasArray() ? buffer.array() : bytes(buffer);
+ int offset = buffer.hasArray() ? buffer.arrayOffset() + buffer.position() : 0;
+ int bound =
+ codec == TLogCompressionType.ZSTD
+ ? (int) Zstd.compressBound(size)
+ : compressor(codec).getMaxBytesForCompression(size);
+ byte[] output = new byte[bound];
+ int length =
+ codec == TLogCompressionType.ZSTD
+ ? (int)
+ Zstd.compressByteArray(output, 0, output.length, input, offset, size, 1)
+ : compressor(codec).compress(input, offset, size, output);
+ if (length < 0) throw invalid(Failure.CODEC, size, length);
+ if ((long) length + DESCRIPTOR_BYTES < size) {
+ encoded = ByteBuffer.wrap(Arrays.copyOf(output, length));
+ actualCodec = codec;
+ arrays += length;
+ compressed = true;
+ }
+ result.retain(
+ before + (actualCodec == TLogCompressionType.NONE ? 0 : encoded.remaining()));
+ } else {
+ // Keep this block raw; other smaller blocks in the entry may still fit.
+ result.memoryLimited = true;
+ }
+ }
+ data.add(encoded);
+ info.add(new TLogBufferCompression(actualCodec, size, 0));
+ wire += encoded.remaining();
+ }
+ // Raw descriptors in a mixed WAL entry cost space too.
+ if (compressed && wire + DESCRIPTOR_BYTES * (long) data.size() + 8 < raw) {
+ for (int i = 0; i < info.size(); i++)
+ info.get(i).setChecksum(checksum(entry.getData().get(i)));
+ entries.add(copy(entry, data).setCompressionInfo(info));
+ retained += arrays + data.size() * 256L + 128;
+ result.encodedBytes += wire - raw;
+ changed = true;
+ } else {
+ entries.add(entry);
+ result.retain(result.reserved.get() - arrays);
+ }
+ }
+ if (changed)
+ result.request =
+ new TSyncLogEntriesReq(original.getPeerId(), original.getConsensusGroupId(), entries);
+ result.retain(retained);
+ return result;
+ } catch (IOException | RuntimeException e) {
+ result.close();
+ throw e;
+ }
+ }
+
+ /** Validate every declared size before allocating; apply nothing until all buffers pass CRC. */
+ public static void decompress(TSyncLogEntriesReq request, int maxDecodedBytes)
+ throws IOException {
+ boolean encoded = false;
+ long total = 0;
+ for (TLogEntry entry : request.getLogEntries()) {
+ if (entry.isSetCompressionInfo()) {
+ encoded = true;
+ if (entry.getCompressionInfoSize() != entry.getDataSize())
+ throw invalid(Failure.DESCRIPTORS, entry.getDataSize(), entry.getCompressionInfoSize());
+ for (int i = 0; i < entry.getDataSize(); i++) {
+ TLogBufferCompression info = entry.getCompressionInfo().get(i);
+ int size = info.getUncompressedSize();
+ if (size < 0) throw invalid(Failure.LENGTH, 0, size);
+ if (info.getCodec() == null) throw invalid(Failure.CODEC, 0, -1);
+ if (info.getCodec() == TLogCompressionType.NONE
+ && size != entry.getData().get(i).remaining())
+ throw invalid(Failure.LENGTH, size, entry.getData().get(i).remaining());
+ total += size;
+ if (total > maxDecodedBytes) throw invalid(Failure.LIMIT, maxDecodedBytes, total);
+ }
+ } else {
+ for (ByteBuffer buffer : entry.getData()) total += buffer.remaining();
+ }
+ }
+ if (!encoded) return;
+ if (total > maxDecodedBytes) throw invalid(Failure.LIMIT, maxDecodedBytes, total);
+ List decoded = new ArrayList<>(request.getLogEntriesSize());
+ for (TLogEntry entry : request.getLogEntries()) {
+ if (!entry.isSetCompressionInfo()) {
+ decoded.add(entry);
+ continue;
+ }
+ List data = new ArrayList<>(entry.getDataSize());
+ for (int i = 0; i < entry.getDataSize(); i++) {
+ TLogBufferCompression info = entry.getCompressionInfo().get(i);
+ ByteBuffer buffer = entry.getData().get(i);
+ if (info.getCodec() != TLogCompressionType.NONE) {
+ byte[] input = buffer.hasArray() ? buffer.array() : bytes(buffer);
+ int offset = buffer.hasArray() ? buffer.arrayOffset() + buffer.position() : 0;
+ byte[] output = new byte[info.getUncompressedSize()];
+ int actual;
+ try {
+ actual =
+ decoder(info.getCodec()).uncompress(input, offset, buffer.remaining(), output, 0);
+ } catch (ZstdException e) {
+ IOException failure = invalid(Failure.CODEC, output.length, buffer.remaining());
+ failure.initCause(e);
+ throw failure;
+ }
+ if (actual != output.length) throw invalid(Failure.LENGTH, output.length, actual);
+ buffer = ByteBuffer.wrap(output);
+ }
+ int crc = checksum(buffer);
+ if (crc != info.getChecksum())
+ throw invalid(
+ Failure.CHECKSUM,
+ Integer.toUnsignedLong(info.getChecksum()),
+ Integer.toUnsignedLong(crc));
+ data.add(buffer);
+ }
+ decoded.add(copy(entry, data));
+ }
+ // The RPC exclusively owns this request. Dropping the encoded entries also drops their arrays.
+ // No partially decoded request is exposed on failure.
+ request.setLogEntries(decoded);
+ }
+
+ private static TLogEntry copy(TLogEntry source, List data) {
+ TLogEntry entry =
+ new TLogEntry(data, source.getSearchIndex(), source.isFromWAL(), source.getMemorySize());
+ if (source.isSetRoutingEpoch()) entry.setRoutingEpoch(source.getRoutingEpoch());
+ if (source.isSetPhysicalTime()) entry.setPhysicalTime(source.getPhysicalTime());
+ return entry;
+ }
+
+ private static byte[] bytes(ByteBuffer buffer) {
+ ByteBuffer copy = buffer.duplicate();
+ byte[] bytes = new byte[copy.remaining()];
+ copy.get(bytes);
+ return bytes;
+ }
+
+ private static int checksum(ByteBuffer buffer) {
+ CRC32C crc = new CRC32C();
+ crc.update(buffer.duplicate());
+ return (int) crc.getValue();
+ }
+
+ private static ICompressor compressor(TLogCompressionType codec) throws IOException {
+ switch (codec) {
+ case LZ4:
+ return LZ4;
+ case SNAPPY:
+ return SNAPPY;
+ default:
+ throw invalid(Failure.CODEC, 0, codec.getValue());
+ }
+ }
+
+ private static IUnCompressor decoder(TLogCompressionType codec) throws IOException {
+ switch (codec) {
+ case LZ4:
+ return LZ4_DECODER;
+ case SNAPPY:
+ return SNAPPY_DECODER;
+ case ZSTD:
+ return ZSTD_DECODER;
+ default:
+ throw invalid(Failure.CODEC, 0, codec.getValue());
+ }
+ }
+
+ private enum Failure {
+ CODEC,
+ LENGTH,
+ LIMIT,
+ DESCRIPTORS,
+ CHECKSUM
+ }
+
+ private static IOException invalid(Failure failure, long expected, long actual) {
+ return new IOException(
+ String.format(
+ IoTConsensusMessages
+ .EXCEPTION_INVALID_IOTCONSENSUS_COMPRESSED_LOG_ARG_EXPECTED_ARG_ACTUAL_ARG_4986175E,
+ failure.name(),
+ expected,
+ actual));
+ }
+}
diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcher.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcher.java
index 0e4f41b9d8412..7f622c000da5e 100644
--- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcher.java
+++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcher.java
@@ -35,9 +35,12 @@
import org.apache.iotdb.consensus.iot.client.DispatchLogHandler;
import org.apache.iotdb.consensus.iot.log.ConsensusReqReader;
import org.apache.iotdb.consensus.iot.log.GetConsensusReqReaderPlan;
+import org.apache.iotdb.consensus.iot.thrift.TLogCompressionType;
import org.apache.iotdb.consensus.iot.thrift.TLogEntry;
import org.apache.iotdb.consensus.iot.thrift.TSyncLogEntriesReq;
+import org.apache.iotdb.consensus.iot.thrift.TSyncLogEntriesRes;
+import org.apache.thrift.async.AsyncMethodCallback;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -587,8 +590,8 @@ private boolean shouldSendIdleWriterSafeTimeBarrier() {
}
public void sendBatchAsync(Batch batch, DispatchLogHandler handler) {
+ LogCompression.EncodedRequest encoded = null;
try {
- AsyncIoTConsensusServiceClient client = clientManager.borrowClient(peer.getEndpoint());
TSyncLogEntriesReq req =
new TSyncLogEntriesReq(
selfPeerId, peer.getGroupId().convertToTConsensusGroupId(), batch.getLogEntries());
@@ -597,8 +600,46 @@ public void sendBatchAsync(Batch batch, DispatchLogHandler handler) {
batch.getStartIndex(),
batch.getEndIndex(),
peer.getGroupId().convertToTConsensusGroupId());
- client.syncLogEntries(req, handler);
+ if (config.getReplication().getLogCompressionType() == TLogCompressionType.NONE) {
+ clientManager.borrowClient(peer.getEndpoint()).syncLogEntries(req, handler);
+ return;
+ }
+ long compressionStart = System.nanoTime();
+ encoded =
+ LogCompression.compress(
+ req,
+ config.getReplication().getLogCompressionType(),
+ config.getReplication().getLogCompressionMinSize(),
+ config.getRpc().getThriftMaxFrameSize());
+ logDispatcherThreadMetrics.recordCompressTime(System.nanoTime() - compressionStart);
+ logDispatcherThreadMetrics.recordCompressionBytes(
+ encoded.getRawBytes(), encoded.getEncodedBytes(), encoded.isMemoryLimited());
+ if (encoded.getRequest() == req) {
+ encoded.close();
+ clientManager.borrowClient(peer.getEndpoint()).syncLogEntries(req, handler);
+ return;
+ }
+ handler.markCompressedAttempt();
+ final LogCompression.EncodedRequest attempt = encoded;
+ clientManager
+ .borrowClient(peer.getEndpoint())
+ .syncLogEntries(
+ attempt.getRequest(),
+ new AsyncMethodCallback() {
+ @Override
+ public void onComplete(TSyncLogEntriesRes response) {
+ attempt.close();
+ handler.onComplete(response);
+ }
+
+ @Override
+ public void onError(Exception exception) {
+ attempt.close();
+ handler.onError(exception);
+ }
+ });
} catch (Exception e) {
+ if (encoded != null) encoded.close();
logger.error(IoTConsensusMessages.CANNOT_SYNC_LOGS_TO_PEER, peer, e);
handler.onError(e);
}
diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcherThreadMetrics.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcherThreadMetrics.java
index 9a5dd6c4b28ec..501979fbeb7e8 100644
--- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcherThreadMetrics.java
+++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcherThreadMetrics.java
@@ -24,6 +24,7 @@
import org.apache.iotdb.metrics.AbstractMetricService;
import org.apache.iotdb.metrics.impl.DoNothingMetricManager;
import org.apache.iotdb.metrics.metricsets.IMetricSet;
+import org.apache.iotdb.metrics.type.Counter;
import org.apache.iotdb.metrics.type.Timer;
import org.apache.iotdb.metrics.utils.MetricLevel;
import org.apache.iotdb.metrics.utils.MetricType;
@@ -45,6 +46,10 @@ public class LogDispatcherThreadMetrics implements IMetricSet {
private final LogDispatcher.LogDispatcherThread logDispatcherThread;
private final String peerGroupId;
+ private Counter rawBytes = DoNothingMetricManager.DO_NOTHING_COUNTER;
+ private Counter encodedBytes = DoNothingMetricManager.DO_NOTHING_COUNTER;
+ private Counter memoryFallbacks = DoNothingMetricManager.DO_NOTHING_COUNTER;
+ private Timer compressTimer = DoNothingMetricManager.DO_NOTHING_TIMER;
private Timer constructBatchTimer = DoNothingMetricManager.DO_NOTHING_TIMER;
private Timer syncLogTimePerRequestTimer = DoNothingMetricManager.DO_NOTHING_TIMER;
@@ -53,6 +58,16 @@ public LogDispatcherThreadMetrics(LogDispatcher.LogDispatcherThread logDispatche
this.peerGroupId = logDispatcherThread.getPeer().getGroupId().toString();
}
+ public void recordCompressionBytes(long raw, long encoded, boolean memoryLimited) {
+ rawBytes.inc(raw);
+ encodedBytes.inc(encoded);
+ if (memoryLimited) memoryFallbacks.inc();
+ }
+
+ public void recordCompressTime(long nanos) {
+ compressTimer.updateNanos(nanos);
+ }
+
public void recordConstructBatchTime(long costTimeInNanos) {
constructBatchTimer.updateNanos(costTimeInNanos);
}
@@ -108,6 +123,38 @@ private void bindStageTimer(AbstractMetricService metricService) {
REGION_TO_STAGE_TIMER_OWNERS
.computeIfAbsent(peerGroupId, region -> new HashSet<>())
.add(this);
+ rawBytes =
+ metricService.getOrCreateCounter(
+ Metric.IOT_LOG_COMPRESSION_BYTES.toString(),
+ MetricLevel.IMPORTANT,
+ Tag.REGION.toString(),
+ peerGroupId,
+ Tag.TYPE.toString(),
+ "raw");
+ encodedBytes =
+ metricService.getOrCreateCounter(
+ Metric.IOT_LOG_COMPRESSION_BYTES.toString(),
+ MetricLevel.IMPORTANT,
+ Tag.REGION.toString(),
+ peerGroupId,
+ Tag.TYPE.toString(),
+ "encoded");
+ memoryFallbacks =
+ metricService.getOrCreateCounter(
+ Metric.IOT_LOG_COMPRESSION_MEMORY_FALLBACKS.toString(),
+ MetricLevel.IMPORTANT,
+ Tag.REGION.toString(),
+ peerGroupId);
+ compressTimer =
+ metricService.getOrCreateTimer(
+ Metric.IOT_SEND_LOG.toString(),
+ MetricLevel.IMPORTANT,
+ Tag.NAME.toString(),
+ Metric.IOT_CONSENSUS.toString(),
+ Tag.STAGE.toString(),
+ "compress",
+ Tag.REGION.toString(),
+ peerGroupId);
constructBatchTimer =
metricService.getOrCreateTimer(
Metric.IOT_SEND_LOG.toString(),
@@ -140,6 +187,34 @@ private void unbindStageTimer(AbstractMetricService metricService) {
return;
}
REGION_TO_STAGE_TIMER_OWNERS.remove(peerGroupId);
+ metricService.remove(
+ MetricType.COUNTER,
+ Metric.IOT_LOG_COMPRESSION_BYTES.toString(),
+ Tag.REGION.toString(),
+ peerGroupId,
+ Tag.TYPE.toString(),
+ "raw");
+ metricService.remove(
+ MetricType.COUNTER,
+ Metric.IOT_LOG_COMPRESSION_BYTES.toString(),
+ Tag.REGION.toString(),
+ peerGroupId,
+ Tag.TYPE.toString(),
+ "encoded");
+ metricService.remove(
+ MetricType.COUNTER,
+ Metric.IOT_LOG_COMPRESSION_MEMORY_FALLBACKS.toString(),
+ Tag.REGION.toString(),
+ peerGroupId);
+ metricService.remove(
+ MetricType.TIMER,
+ Metric.IOT_SEND_LOG.toString(),
+ Tag.NAME.toString(),
+ Metric.IOT_CONSENSUS.toString(),
+ Tag.STAGE.toString(),
+ "compress",
+ Tag.REGION.toString(),
+ peerGroupId);
}
metricService.remove(
MetricType.TIMER,
diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/service/IoTConsensusRPCServiceProcessor.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/service/IoTConsensusRPCServiceProcessor.java
index 6db087f7fe283..503e5e6ab7175 100644
--- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/service/IoTConsensusRPCServiceProcessor.java
+++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/service/IoTConsensusRPCServiceProcessor.java
@@ -35,6 +35,7 @@
import org.apache.iotdb.consensus.i18n.IoTConsensusV2Messages;
import org.apache.iotdb.consensus.iot.IoTConsensus;
import org.apache.iotdb.consensus.iot.IoTConsensusServerImpl;
+import org.apache.iotdb.consensus.iot.logdispatcher.LogCompression;
import org.apache.iotdb.consensus.iot.thrift.IoTConsensusIService;
import org.apache.iotdb.consensus.iot.thrift.TActivatePeerReq;
import org.apache.iotdb.consensus.iot.thrift.TActivatePeerRes;
@@ -65,6 +66,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
import java.util.Collections;
import java.util.stream.Collectors;
@@ -110,6 +112,18 @@ public TSyncLogEntriesRes syncLogEntries(TSyncLogEntriesReq req) {
status.setMessage(message);
return new TSyncLogEntriesRes(Collections.singletonList(status));
}
+ long decompressStart = System.nanoTime();
+ try {
+ // Every block is checked before constructing requests or entering the sync/apply path.
+ LogCompression.decompress(req, impl.getConfig().getRpc().getThriftMaxFrameSize());
+ } catch (IOException | IllegalArgumentException e) {
+ TSStatus status = new TSStatus(TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ status.setMessage(e.getMessage());
+ return new TSyncLogEntriesRes(Collections.singletonList(status))
+ .setLogCompressionError(status);
+ } finally {
+ impl.getIoTConsensusServerMetrics().recordDecompressCost(System.nanoTime() - decompressStart);
+ }
BatchIndexedConsensusRequest logEntriesInThisBatch =
new BatchIndexedConsensusRequest(req.peerId);
final int sourceNodeId = req.peerId;
diff --git a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/CompressedReplicationTest.java b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/CompressedReplicationTest.java
new file mode 100644
index 0000000000000..f01178d24776d
--- /dev/null
+++ b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/CompressedReplicationTest.java
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.consensus.iot;
+
+import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType;
+import org.apache.iotdb.common.rpc.thrift.TEndPoint;
+import org.apache.iotdb.common.rpc.thrift.TSStatus;
+import org.apache.iotdb.commons.consensus.DataRegionId;
+import org.apache.iotdb.commons.request.IConsensusRequest;
+import org.apache.iotdb.consensus.common.Peer;
+import org.apache.iotdb.consensus.common.request.BatchIndexedConsensusRequest;
+import org.apache.iotdb.consensus.common.request.IndexedConsensusRequest;
+import org.apache.iotdb.consensus.config.ConsensusConfig;
+import org.apache.iotdb.consensus.config.IoTConsensusConfig;
+import org.apache.iotdb.consensus.iot.thrift.TLogCompressionType;
+import org.apache.iotdb.consensus.iot.util.TestEntry;
+import org.apache.iotdb.consensus.iot.util.TestStateMachine;
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.net.ServerSocket;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+/** Real async Thrift in both directions, using the existing state machine/reader fixture. */
+@RunWith(Parameterized.class)
+public class CompressedReplicationTest {
+ @Rule public TemporaryFolder directory = new TemporaryFolder();
+
+ @Parameterized.Parameters(name = "{0}")
+ public static Object[] codecs() {
+ return new Object[] {
+ TLogCompressionType.LZ4, TLogCompressionType.SNAPPY, TLogCompressionType.ZSTD
+ };
+ }
+
+ @Parameterized.Parameter public TLogCompressionType codec;
+ private static final int COUNT = 32;
+ private static final int BYTES = 65536;
+
+ @Test
+ public void testBidirectionalReplicationPreservesFullPayloadAndMetadata() throws Exception {
+ DataRegionId group = new DataRegionId(1);
+ int firstPort, secondPort;
+ try (ServerSocket first = new ServerSocket(0);
+ ServerSocket second = new ServerSocket(0)) {
+ firstPort = first.getLocalPort();
+ secondPort = second.getLocalPort();
+ }
+ List peers =
+ Arrays.asList(
+ new Peer(group, 1, new TEndPoint("127.0.0.1", firstPort)),
+ new Peer(group, 2, new TEndPoint("127.0.0.1", secondPort)));
+ IoTConsensus[] servers = new IoTConsensus[2];
+ ObservedStateMachine[] machines = {new ObservedStateMachine(), new ObservedStateMachine()};
+ try {
+ for (int i = 0; i < 2; i++) {
+ final int node = i;
+ ConsensusConfig config =
+ ConsensusConfig.newBuilder()
+ .setThisNodeId(i + 1)
+ .setThisNode(peers.get(i).getEndpoint())
+ .setConsensusGroupType(TConsensusGroupType.DataRegion)
+ .setStorageDir(directory.newFolder().getAbsolutePath())
+ .setRecvSnapshotDirs(
+ Collections.singletonList(directory.newFolder().getAbsolutePath()))
+ .setIoTConsensusConfig(
+ IoTConsensusConfig.newBuilder()
+ .setReplication(
+ IoTConsensusConfig.Replication.newBuilder()
+ .setLogCompressionType(codec)
+ .setMaxWaitingTimeForWaitBatchInMs(100)
+ .build())
+ .build())
+ .build();
+ servers[i] = new IoTConsensus(config, ignored -> machines[node]);
+ servers[i].recordCorrectPeerListBeforeStarting(Collections.singletonMap(group, peers));
+ servers[i].start();
+ }
+ for (IoTConsensus server : servers) server.createLocalPeer(group, peers);
+ for (int j = 0; j < COUNT; j++) {
+ for (int i = 0; i < 2; i++) {
+ final TestEntry entry =
+ new TestEntry(j, peers.get(i)) {
+ @Override
+ public ByteBuffer serializeToByteBuffer() {
+ ByteBuffer result = ByteBuffer.allocate(BYTES);
+ result.put(super.serializeToByteBuffer());
+ // A repeated nonzero tail makes corruption visible beyond the parsed TestEntry.
+ while (result.hasRemaining()) result.put((byte) 42);
+ result.flip();
+ return result;
+ }
+ };
+ assertEquals(200, servers[i].write(group, entry).getCode());
+ }
+ }
+ for (int i = 0; i < 2; i++) {
+ assertTrue(
+ "remote requests must all be applied", machines[i].applied.await(30, TimeUnit.SECONDS));
+ assertNull(machines[i].failure.get());
+ assertEquals(COUNT * 2, machines[i].getData().size());
+ for (Peer peer : peers)
+ for (int j = 0; j < COUNT; j++)
+ assertTrue(machines[i].getData().contains(new TestEntry(j, peer)));
+ }
+ } finally {
+ for (IoTConsensus server : servers) if (server != null) server.stop();
+ }
+ }
+
+ private static class ObservedStateMachine extends TestStateMachine {
+ private final CountDownLatch applied = new CountDownLatch(COUNT);
+ private final AtomicReference failure = new AtomicReference<>();
+
+ @Override
+ public IConsensusRequest deserializeRequest(IConsensusRequest request) {
+ if (request instanceof BatchIndexedConsensusRequest) {
+ try {
+ for (IndexedConsensusRequest indexed :
+ ((BatchIndexedConsensusRequest) request).getRequests()) {
+ assertTrue(indexed.getPhysicalTime() > 0);
+ for (IConsensusRequest inner : indexed.getRequests()) {
+ ByteBuffer buffer = inner.serializeToByteBuffer().duplicate();
+ assertEquals(BYTES, buffer.remaining());
+ buffer.getInt();
+ Peer.deserialize(buffer);
+ while (buffer.hasRemaining()) assertEquals(42, buffer.get());
+ }
+ }
+ } catch (AssertionError e) {
+ failure.compareAndSet(null, e);
+ }
+ }
+ return super.deserializeRequest(request);
+ }
+
+ @Override
+ public TSStatus write(IConsensusRequest request) {
+ TSStatus result = super.write(request);
+ if (request instanceof IndexedConsensusRequest
+ && ((IndexedConsensusRequest) request).getSearchIndex() == -1) {
+ applied.countDown();
+ }
+ return result;
+ }
+ }
+}
diff --git a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/client/DispatchLogHandlerTest.java b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/client/DispatchLogHandlerTest.java
index 13736903902a9..224d5dfd92fab 100644
--- a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/client/DispatchLogHandlerTest.java
+++ b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/client/DispatchLogHandlerTest.java
@@ -24,21 +24,42 @@
import org.apache.iotdb.commons.audit.UserDataTransferAuditEvent;
import org.apache.iotdb.commons.audit.UserDataTransferAuditHandler;
import org.apache.iotdb.commons.audit.UserDataTransferProtectionMethod;
+import org.apache.iotdb.commons.consensus.DataRegionId;
+import org.apache.iotdb.consensus.common.Peer;
import org.apache.iotdb.consensus.config.IoTConsensusConfig;
+import org.apache.iotdb.consensus.iot.IoTConsensusServerImpl;
import org.apache.iotdb.consensus.iot.logdispatcher.Batch;
+import org.apache.iotdb.consensus.iot.logdispatcher.IndexController;
+import org.apache.iotdb.consensus.iot.logdispatcher.LogDispatcher.LogDispatcherThread;
+import org.apache.iotdb.consensus.iot.logdispatcher.LogDispatcherThreadMetrics;
+import org.apache.iotdb.consensus.iot.logdispatcher.SyncStatus;
import org.apache.iotdb.consensus.iot.thrift.TLogEntry;
import org.apache.iotdb.consensus.iot.thrift.TSyncLogEntriesRes;
+import org.apache.thrift.TApplicationException;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.mockito.ArgumentCaptor;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.ArgumentMatchers.same;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
public class DispatchLogHandlerTest {
@@ -164,6 +185,74 @@ public boolean isEnabled() {
null);
}
+ @Rule public TemporaryFolder directory = new TemporaryFolder();
+
+ @Test
+ public void testCompressedProtocolFailureRetriesWithoutAdvancingIndex() throws Exception {
+ verifyCompressionFailure(true);
+ }
+
+ @Test
+ public void testExplicitDecodeErrorOverridesNonRetryStatus() throws Exception {
+ verifyCompressionFailure(false);
+ }
+
+ private void verifyCompressionFailure(boolean protocolError) throws Exception {
+ IoTConsensusConfig config = IoTConsensusConfig.newBuilder().build();
+ Peer peer = new Peer(new DataRegionId(1), 2, TARGET);
+ IndexController controller =
+ new IndexController(directory.newFolder().getAbsolutePath(), peer, 0, 500);
+ SyncStatus status = new SyncStatus(controller, config);
+ Batch batch = createBatch(false);
+ LogDispatcherThread thread = mock(LogDispatcherThread.class);
+ IoTConsensusServerImpl impl = mock(IoTConsensusServerImpl.class);
+ ScheduledExecutorService scheduler = mock(ScheduledExecutorService.class);
+ when(thread.getConfig()).thenReturn(config);
+ when(thread.getPeer()).thenReturn(peer);
+ when(thread.getSyncStatus()).thenReturn(status);
+ when(thread.getImpl()).thenReturn(impl);
+ when(impl.getBackgroundTaskService()).thenReturn(scheduler);
+ when(impl.getUserDataTransferAuditHandler()).thenReturn(UserDataTransferAuditHandler.NO_OP);
+ status.addNextBatch(batch);
+ try {
+ DispatchLogHandler handler =
+ new DispatchLogHandler(thread, mock(LogDispatcherThreadMetrics.class), batch);
+ if (protocolError) {
+ handler.markCompressedAttempt();
+ handler.onError(new TApplicationException(TApplicationException.PROTOCOL_ERROR));
+ } else {
+ TSStatus error = new TSStatus(200).setMessage("decode failed");
+ handler.onComplete(
+ new TSyncLogEntriesRes(Collections.singletonList(error)).setLogCompressionError(error));
+ }
+ assertEquals(0, controller.getCurrentIndex());
+ assertFalse(batch.isSynced());
+ assertTrue(status.hasPendingBatches());
+ // A later in-flight batch may ACK first. Its success must not advance past the rejected
+ // batch or release either raw batch before the missing prefix has succeeded.
+ Batch later = new Batch(config);
+ later.addTLogEntry(new TLogEntry().setSearchIndex(2).setMemorySize(1));
+ later.buildIndex();
+ status.addNextBatch(later);
+ DispatchLogHandler laterHandler =
+ new DispatchLogHandler(thread, mock(LogDispatcherThreadMetrics.class), later);
+ laterHandler.onComplete(new TSyncLogEntriesRes(Collections.singletonList(new TSStatus(200))));
+ assertTrue(later.isSynced());
+ assertEquals(0, controller.getCurrentIndex());
+ assertEquals(2, status.getPendingBatches().size());
+ ArgumentCaptor retry = ArgumentCaptor.forClass(Runnable.class);
+ verify(scheduler).schedule(retry.capture(), anyLong(), eq(TimeUnit.MILLISECONDS));
+ retry.getValue().run();
+ verify(thread).sendBatchAsync(same(batch), same(handler));
+ handler.onComplete(new TSyncLogEntriesRes(Collections.singletonList(new TSStatus(200))));
+ assertEquals(2, controller.getCurrentIndex());
+ assertFalse(status.hasPendingBatches());
+ verify(thread, times(2)).updateSafelyDeletedSearchIndex();
+ } finally {
+ status.free();
+ }
+ }
+
private static Batch createBatch(boolean containsUserData) {
final Batch batch = new Batch(IoTConsensusConfig.newBuilder().build());
batch.addTLogEntry(new TLogEntry().setSearchIndex(1).setMemorySize(1), containsUserData);
diff --git a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompressionTest.java b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompressionTest.java
new file mode 100644
index 0000000000000..09eabafb68198
--- /dev/null
+++ b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompressionTest.java
@@ -0,0 +1,364 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.consensus.iot.logdispatcher;
+
+import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId;
+import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType;
+import org.apache.iotdb.common.rpc.thrift.TSStatus;
+import org.apache.iotdb.commons.consensus.DataRegionId;
+import org.apache.iotdb.commons.memory.AtomicLongMemoryBlock;
+import org.apache.iotdb.commons.memory.IMemoryBlock;
+import org.apache.iotdb.consensus.IStateMachine;
+import org.apache.iotdb.consensus.common.request.ByteBufferConsensusRequest;
+import org.apache.iotdb.consensus.common.request.IndexedConsensusRequest;
+import org.apache.iotdb.consensus.config.IoTConsensusConfig;
+import org.apache.iotdb.consensus.iot.IoTConsensus;
+import org.apache.iotdb.consensus.iot.IoTConsensusServerImpl;
+import org.apache.iotdb.consensus.iot.IoTConsensusServerMetrics;
+import org.apache.iotdb.consensus.iot.service.IoTConsensusRPCServiceProcessor;
+import org.apache.iotdb.consensus.iot.thrift.TLogCompressionType;
+import org.apache.iotdb.consensus.iot.thrift.TLogEntry;
+import org.apache.iotdb.consensus.iot.thrift.TSyncLogEntriesReq;
+import org.apache.iotdb.consensus.iot.thrift.TSyncLogEntriesRes;
+
+import org.apache.thrift.TDeserializer;
+import org.apache.thrift.TSerializer;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Random;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertThrows;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.ArgumentMatchers.anyList;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+@RunWith(Parameterized.class)
+public class LogCompressionTest {
+ @Parameterized.Parameters(name = "{0}")
+ public static Object[] codecs() {
+ return new Object[] {
+ TLogCompressionType.LZ4, TLogCompressionType.SNAPPY, TLogCompressionType.ZSTD
+ };
+ }
+
+ @Parameterized.Parameter public TLogCompressionType codec;
+ private IMemoryBlock previous;
+ private final IoTConsensusMemoryManager memory = IoTConsensusMemoryManager.getInstance();
+ private static final int LIMIT = 8 * 1024 * 1024;
+
+ @Before
+ public void setUp() {
+ previous = memory.getMemoryBlock();
+ memory.setMemoryBlock(new AtomicLongMemoryBlock("compression-test", null, 64L * 1024 * 1024));
+ memory.reset();
+ }
+
+ @After
+ public void tearDown() {
+ assertEquals(0, memory.getMemoryBlock().getUsedMemoryInBytes());
+ memory.reset();
+ memory.setMemoryBlock(previous);
+ }
+
+ private TSyncLogEntriesReq request() {
+ byte[] compressible = new byte[16384];
+ Arrays.fill(compressible, (byte) 42);
+ for (int i = 0; i < 17; i++) compressible[i] = (byte) i;
+ ByteBuffer sliced = ByteBuffer.wrap(compressible);
+ sliced.position(17);
+ sliced.limit(16000);
+ ByteBuffer direct = ByteBuffer.allocateDirect(8192);
+ while (direct.hasRemaining()) direct.put((byte) 7);
+ direct.flip();
+ byte[] random = new byte[8192];
+ new Random(123).nextBytes(random);
+ TLogEntry wal =
+ new TLogEntry(
+ Arrays.asList(
+ sliced,
+ direct.asReadOnlyBuffer(),
+ ByteBuffer.wrap(new byte[] {1, 2, 3}),
+ ByteBuffer.wrap(random)),
+ 1,
+ true,
+ 50000)
+ .setRoutingEpoch(9)
+ .setPhysicalTime(12345);
+ TLogEntry queue =
+ new TLogEntry(
+ Collections.singletonList(ByteBuffer.wrap(compressible).asReadOnlyBuffer()),
+ 2,
+ false,
+ 16384);
+ return new TSyncLogEntriesReq(
+ 3, new TConsensusGroupId(TConsensusGroupType.DataRegion, 5), Arrays.asList(wal, queue));
+ }
+
+ @Test
+ public void testRoundTripThriftAndRetryOwnership() throws Exception {
+ TSyncLogEntriesReq raw = request();
+ TSyncLogEntriesReq snapshot = raw.deepCopy();
+ for (int attempt = 0; attempt < 2; attempt++) {
+ LogCompression.EncodedRequest encoded = LogCompression.compress(raw, codec, 4096, LIMIT);
+ try {
+ assertNotSame(raw, encoded.getRequest());
+ assertEquals(
+ encoded.getRequest().getLogEntries().stream()
+ .flatMap(e -> e.getData().stream())
+ .mapToLong(ByteBuffer::remaining)
+ .sum(),
+ encoded.getEncodedBytes());
+ assertTrue(encoded.getEncodedBytes() < encoded.getRawBytes());
+ assertTrue(memory.getMemoryBlock().getUsedMemoryInBytes() > 0);
+ assertEquals(
+ codec,
+ encoded.getRequest().getLogEntries().get(0).getCompressionInfo().get(0).getCodec());
+ assertEquals(
+ TLogCompressionType.NONE,
+ encoded.getRequest().getLogEntries().get(0).getCompressionInfo().get(2).getCodec());
+ TSyncLogEntriesReq received = new TSyncLogEntriesReq();
+ new TDeserializer()
+ .deserialize(received, new TSerializer().serialize(encoded.getRequest()));
+ LogCompression.decompress(received, LIMIT);
+ assertEquals(snapshot, received);
+ assertEquals(snapshot, raw);
+ assertEquals(17, raw.getLogEntries().get(0).getData().get(0).position());
+ assertEquals(16000, raw.getLogEntries().get(0).getData().get(0).limit());
+ } finally {
+ encoded.close();
+ encoded.close();
+ }
+ }
+ }
+
+ @Test
+ public void testAdaptiveAndMemoryFallback() throws Exception {
+ TSyncLogEntriesReq raw = request();
+ try (LogCompression.EncodedRequest result = LogCompression.compress(raw, codec, LIMIT, LIMIT)) {
+ assertSame(raw, result.getRequest());
+ }
+ byte[] random = new byte[16384];
+ new Random(1).nextBytes(random);
+ TSyncLogEntriesReq noise = raw.deepCopy();
+ noise.setLogEntries(
+ Collections.singletonList(
+ new TLogEntry(
+ Collections.singletonList(ByteBuffer.wrap(random)), 1, false, random.length)));
+ try (LogCompression.EncodedRequest result = LogCompression.compress(noise, codec, 0, LIMIT)) {
+ assertSame(noise, result.getRequest());
+ }
+ long capacity = memory.getMemoryBlock().getTotalMemorySizeInBytes();
+ assertTrue(memory.reserveLogCompression(capacity));
+ try (LogCompression.EncodedRequest result = LogCompression.compress(raw, codec, 0, LIMIT)) {
+ assertSame(raw, result.getRequest());
+ assertTrue(result.isMemoryLimited());
+ assertEquals(result.getRawBytes(), result.getEncodedBytes());
+ assertEquals(capacity, memory.getMemoryBlock().getUsedMemoryInBytes());
+ } finally {
+ memory.freeLogCompression(capacity);
+ }
+ }
+
+ @Test
+ public void testRejectsEntireRequestBeforePublishingAnyDecodedEntry() throws Exception {
+ try (LogCompression.EncodedRequest encoded =
+ LogCompression.compress(request(), codec, 4096, LIMIT)) {
+ TSyncLogEntriesReq bad = encoded.getRequest().deepCopy();
+ bad.getLogEntries().get(1).getCompressionInfo().get(0).setChecksum(0);
+ TSyncLogEntriesReq snapshot = bad.deepCopy();
+ assertThrows(IOException.class, () -> LogCompression.decompress(bad, LIMIT));
+ assertEquals(snapshot, bad);
+ // Also cover corruption of an uncompressed block in a mixed WAL entry.
+ TSyncLogEntriesReq secondBad = encoded.getRequest().deepCopy();
+ secondBad.getLogEntries().get(0).getCompressionInfo().get(2).setChecksum(0);
+ final TSyncLogEntriesReq mixed = secondBad;
+ assertThrows(IOException.class, () -> LogCompression.decompress(mixed, LIMIT));
+ }
+ }
+
+ @Test
+ public void testMalformedMetadataAndCorruptedCodecInput() throws Exception {
+ try (LogCompression.EncodedRequest encoded =
+ LogCompression.compress(request(), codec, 4096, LIMIT)) {
+ TSyncLogEntriesReq count = encoded.getRequest().deepCopy();
+ count.getLogEntries().get(0).getCompressionInfo().remove(0);
+ assertThrows(IOException.class, () -> LogCompression.decompress(count, LIMIT));
+ TSyncLogEntriesReq huge = encoded.getRequest().deepCopy();
+ huge.getLogEntries()
+ .get(0)
+ .getCompressionInfo()
+ .get(0)
+ .setUncompressedSize(Integer.MAX_VALUE);
+ assertThrows(IOException.class, () -> LogCompression.decompress(huge, LIMIT));
+ TSyncLogEntriesReq longOutput = encoded.getRequest().deepCopy();
+ longOutput.getLogEntries().get(0).getCompressionInfo().get(0).setUncompressedSize(16000);
+ assertThrows(IOException.class, () -> LogCompression.decompress(longOutput, LIMIT));
+ TSyncLogEntriesReq rawLength = encoded.getRequest().deepCopy();
+ rawLength.getLogEntries().get(0).getCompressionInfo().get(2).setUncompressedSize(4);
+ assertThrows(IOException.class, () -> LogCompression.decompress(rawLength, LIMIT));
+ TSyncLogEntriesReq negative = encoded.getRequest().deepCopy();
+ negative.getLogEntries().get(0).getCompressionInfo().get(0).setUncompressedSize(-1);
+ assertThrows(IOException.class, () -> LogCompression.decompress(negative, LIMIT));
+ TSyncLogEntriesReq unknown = encoded.getRequest().deepCopy();
+ unknown.getLogEntries().get(0).getCompressionInfo().get(0).setCodec(null);
+ assertThrows(IOException.class, () -> LogCompression.decompress(unknown, LIMIT));
+ TSyncLogEntriesReq corrupt = encoded.getRequest().deepCopy();
+ corrupt.getLogEntries().get(0).getData().set(0, ByteBuffer.wrap(new byte[] {0, 1, 2}));
+ assertThrows(IOException.class, () -> LogCompression.decompress(corrupt, LIMIT));
+ }
+ }
+
+ @Test
+ public void testSingleLogAboveSoftBatchLimit() throws Exception {
+ TSyncLogEntriesReq raw = request();
+ raw.setLogEntries(
+ Collections.singletonList(
+ new TLogEntry(
+ Collections.singletonList(ByteBuffer.allocate(2 * 1024 * 1024)),
+ 1,
+ false,
+ 2 * 1024 * 1024)));
+ try (LogCompression.EncodedRequest encoded = LogCompression.compress(raw, codec, 4096, LIMIT)) {
+ TSyncLogEntriesReq received = encoded.getRequest().deepCopy();
+ LogCompression.decompress(received, LIMIT);
+ assertEquals(raw, received);
+ }
+ }
+
+ @Test
+ public void testProcessorRejectsBeforeDeserializeAndDecodesWithSendingDisabled()
+ throws Exception {
+ IoTConsensus consensus = mock(IoTConsensus.class);
+ IoTConsensusServerImpl impl = mock(IoTConsensusServerImpl.class);
+ IStateMachine stateMachine = mock(IStateMachine.class);
+ when(consensus.getImpl(new DataRegionId(5))).thenReturn(impl);
+ when(impl.isActive()).thenReturn(true);
+ // Decoder support must not depend on the sending codec (default NONE).
+ when(impl.getConfig()).thenReturn(IoTConsensusConfig.newBuilder().build());
+ when(impl.getIoTConsensusServerMetrics()).thenReturn(mock(IoTConsensusServerMetrics.class));
+ when(impl.getStateMachine()).thenReturn(stateMachine);
+ IoTConsensusRPCServiceProcessor processor = new IoTConsensusRPCServiceProcessor(consensus);
+ try (LogCompression.EncodedRequest encoded =
+ LogCompression.compress(request(), codec, 4096, LIMIT)) {
+ TSyncLogEntriesReq corrupt = encoded.getRequest().deepCopy();
+ corrupt.getLogEntries().get(1).getCompressionInfo().get(0).setChecksum(0);
+ assertTrue(processor.syncLogEntries(corrupt).isSetLogCompressionError());
+ // Snappy reports a short output buffer as IllegalArgumentException; the RPC boundary must
+ // reject it in the same explicit, pre-application error path as codec I/O failures.
+ TSyncLogEntriesReq shortOutput = encoded.getRequest().deepCopy();
+ shortOutput.getLogEntries().get(0).getCompressionInfo().get(0).setUncompressedSize(1);
+ assertTrue(processor.syncLogEntries(shortOutput).isSetLogCompressionError());
+ verify(impl, never()).getStateMachine();
+ verify(impl, never()).syncLog(anyInt(), any());
+ verify(impl, never())
+ .buildIndexedConsensusRequestForRemoteRequest(
+ anyLong(), anyLong(), anyLong(), anyInt(), anyList());
+ when(impl.buildIndexedConsensusRequestForRemoteRequest(
+ anyLong(), anyLong(), anyLong(), anyInt(), anyList()))
+ .thenAnswer(
+ call -> new IndexedConsensusRequest(-1, call.getArgument(0), call.getArgument(4)));
+ ByteBufferConsensusRequest applied = new ByteBufferConsensusRequest(ByteBuffer.allocate(1));
+ when(stateMachine.deserializeRequest(any())).thenReturn(applied);
+ when(impl.syncLog(eq(3), eq(applied)))
+ .thenReturn(new TSStatus(200).setSubStatus(Collections.singletonList(new TSStatus(200))));
+ TSyncLogEntriesReq valid = encoded.getRequest().deepCopy();
+ TSyncLogEntriesRes result = processor.syncLogEntries(valid);
+ assertFalse(result.isSetLogCompressionError());
+ assertEquals(200, result.getStatuses().get(0).getCode());
+ assertEquals(request(), valid);
+ verify(impl).syncLog(3, applied);
+ }
+ }
+
+ @Test
+ public void testMultiBlockScratchBudgetMatchesSequentialPeak() throws Exception {
+ memory.setMemoryBlock(new AtomicLongMemoryBlock("compression-tight-pool", null, 512 * 1024));
+ TSyncLogEntriesReq raw = request();
+ List entries = new ArrayList<>();
+ for (int i = 0; i < 16; i++)
+ entries.add(
+ new TLogEntry(
+ Collections.singletonList(ByteBuffer.allocate(16384)), i + 1, false, 16384));
+ raw.setLogEntries(entries);
+ try (LogCompression.EncodedRequest encoded = LogCompression.compress(raw, codec, 4096, LIMIT)) {
+ assertNotSame(raw, encoded.getRequest());
+ assertEquals(16 * 16384, encoded.getRawBytes());
+ assertFalse(encoded.isMemoryLimited());
+ TSyncLogEntriesReq received = encoded.getRequest().deepCopy();
+ LogCompression.decompress(received, LIMIT);
+ assertEquals(raw, received);
+ assertTrue(memory.getMemoryBlock().getUsedMemoryInBytes() < 512 * 1024);
+ }
+ }
+
+ @Test
+ public void testMemoryPressureFallsBackPerBlockAndKeepsOtherBlocksCompressed() throws Exception {
+ memory.setMemoryBlock(new AtomicLongMemoryBlock("compression-partial-pool", null, 48 * 1024));
+ TSyncLogEntriesReq raw = request();
+ raw.setLogEntries(
+ Collections.singletonList(
+ new TLogEntry(
+ Arrays.asList(
+ ByteBuffer.allocate(4096),
+ ByteBuffer.allocate(32768),
+ ByteBuffer.allocate(4096)),
+ 1,
+ true,
+ 40960)));
+ try (LogCompression.EncodedRequest encoded = LogCompression.compress(raw, codec, 4096, LIMIT)) {
+ assertNotSame(raw, encoded.getRequest());
+ assertTrue(encoded.isMemoryLimited());
+ assertEquals(
+ codec,
+ encoded.getRequest().getLogEntries().get(0).getCompressionInfo().get(0).getCodec());
+ assertEquals(
+ TLogCompressionType.NONE,
+ encoded.getRequest().getLogEntries().get(0).getCompressionInfo().get(1).getCodec());
+ assertEquals(
+ codec,
+ encoded.getRequest().getLogEntries().get(0).getCompressionInfo().get(2).getCodec());
+ TSyncLogEntriesReq received = encoded.getRequest().deepCopy();
+ LogCompression.decompress(received, LIMIT);
+ assertEquals(raw, received);
+ assertTrue(encoded.getEncodedBytes() < encoded.getRawBytes());
+ }
+ }
+}
diff --git a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcherCompressionTest.java b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcherCompressionTest.java
new file mode 100644
index 0000000000000..81ae68231c2cc
--- /dev/null
+++ b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcherCompressionTest.java
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.consensus.iot.logdispatcher;
+
+import org.apache.iotdb.common.rpc.thrift.TEndPoint;
+import org.apache.iotdb.commons.client.IClientManager;
+import org.apache.iotdb.commons.client.exception.ClientManagerException;
+import org.apache.iotdb.commons.consensus.DataRegionId;
+import org.apache.iotdb.commons.memory.AtomicLongMemoryBlock;
+import org.apache.iotdb.commons.memory.IMemoryBlock;
+import org.apache.iotdb.consensus.IStateMachine;
+import org.apache.iotdb.consensus.common.Peer;
+import org.apache.iotdb.consensus.config.IoTConsensusConfig;
+import org.apache.iotdb.consensus.iot.IoTConsensusServerImpl;
+import org.apache.iotdb.consensus.iot.client.AsyncIoTConsensusServiceClient;
+import org.apache.iotdb.consensus.iot.client.DispatchLogHandler;
+import org.apache.iotdb.consensus.iot.thrift.TLogCompressionType;
+import org.apache.iotdb.consensus.iot.thrift.TLogEntry;
+import org.apache.iotdb.consensus.iot.thrift.TSyncLogEntriesReq;
+import org.apache.iotdb.consensus.iot.thrift.TSyncLogEntriesRes;
+import org.apache.iotdb.consensus.iot.util.FakeConsensusReqReader;
+
+import org.apache.thrift.TException;
+import org.apache.thrift.async.AsyncMethodCallback;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Captor;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnit;
+import org.mockito.junit.MockitoRule;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Collections;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class LogDispatcherCompressionTest {
+ @Rule public TemporaryFolder directory = new TemporaryFolder();
+ private final IoTConsensusMemoryManager memory = IoTConsensusMemoryManager.getInstance();
+ private IMemoryBlock previous;
+ private LogDispatcher dispatcher;
+ private LogDispatcher.LogDispatcherThread thread;
+ @Rule public MockitoRule mocks = MockitoJUnit.rule();
+ @Mock private IClientManager clients;
+ @Captor private ArgumentCaptor> callback;
+ private AsyncIoTConsensusServiceClient client;
+ private DispatchLogHandler handler;
+ private Batch batch;
+ private Peer target;
+
+ @Before
+ public void setUp() throws Exception {
+ previous = memory.getMemoryBlock();
+ memory.setMemoryBlock(
+ new AtomicLongMemoryBlock("sender-compression-test", null, 64L * 1024 * 1024));
+ memory.reset();
+ IoTConsensusConfig config =
+ IoTConsensusConfig.newBuilder()
+ .setReplication(
+ IoTConsensusConfig.Replication.newBuilder()
+ .setLogCompressionType(TLogCompressionType.LZ4)
+ .build())
+ .build();
+ IoTConsensusServerImpl impl = mock(IoTConsensusServerImpl.class);
+ IStateMachine stateMachine = mock(IStateMachine.class);
+ when(impl.getStateMachine()).thenReturn(stateMachine);
+ when(stateMachine.read(any())).thenReturn(mock(FakeConsensusReqReader.class));
+ when(impl.getStorageDir()).thenReturn(directory.newFolder().getAbsolutePath());
+ Peer self = new Peer(new DataRegionId(1), 1, new TEndPoint("127.0.0.1", 9000));
+ target = new Peer(new DataRegionId(1), 2, new TEndPoint("127.0.0.1", 9001));
+ when(impl.getThisNode()).thenReturn(self);
+ when(impl.getConfiguration()).thenReturn(Collections.singletonList(self));
+ client = mock(AsyncIoTConsensusServiceClient.class);
+ when(clients.borrowClient(target.getEndpoint()))
+ .thenAnswer(
+ call -> {
+ // Encoding is complete before obtaining a client, including its temporary
+ // reservation.
+ assertTrue(memory.getMemoryBlock().getUsedMemoryInBytes() > 0);
+ return client;
+ });
+ dispatcher = new LogDispatcher(impl, clients);
+ thread = dispatcher.new LogDispatcherThread(target, config, 0);
+ handler = mock(DispatchLogHandler.class);
+ batch = new Batch(config);
+ batch.addTLogEntry(
+ new TLogEntry(Collections.singletonList(ByteBuffer.allocate(16384)), 1, false, 16384));
+ batch.buildIndex();
+ }
+
+ @After
+ public void tearDown() {
+ Thread.currentThread().interrupt();
+ thread.run();
+ thread.stop();
+ dispatcher.stop();
+ assertEquals(0, memory.getMemoryBlock().getUsedMemoryInBytes());
+ memory.reset();
+ memory.setMemoryBlock(previous);
+ }
+
+ @Test
+ public void testSuccessReleasesWireMemoryAndPreservesRawBatch() throws Exception {
+ verifyAsyncCompletion(false);
+ }
+
+ @Test
+ public void testAsyncFailureReleasesWireMemoryAndPreservesRawBatch() throws Exception {
+ verifyAsyncCompletion(true);
+ }
+
+ private void verifyAsyncCompletion(boolean failure) throws Exception {
+ thread.sendBatchAsync(batch, handler);
+ ArgumentCaptor request = ArgumentCaptor.forClass(TSyncLogEntriesReq.class);
+ verify(client).syncLogEntries(request.capture(), callback.capture());
+ verify(handler).markCompressedAttempt();
+ assertTrue(request.getValue().getLogEntries().get(0).isSetCompressionInfo());
+ assertTrue(memory.getMemoryBlock().getUsedMemoryInBytes() > 0);
+ if (failure) {
+ IOException error = new IOException();
+ callback.getValue().onError(error);
+ verify(handler).onError(error);
+ } else {
+ TSyncLogEntriesRes response = new TSyncLogEntriesRes();
+ callback.getValue().onComplete(response);
+ verify(handler).onComplete(response);
+ }
+ assertEquals(0, memory.getMemoryBlock().getUsedMemoryInBytes());
+ assertEquals(16384, batch.getLogEntries().get(0).getData().get(0).remaining());
+ assertTrue(!batch.getLogEntries().get(0).isSetCompressionInfo());
+ }
+
+ @Test
+ public void testBorrowFailureReleasesWireMemory() throws Exception {
+ ClientManagerException error = new ClientManagerException(new IOException());
+ doThrow(error).when(clients).borrowClient(target.getEndpoint());
+ thread.sendBatchAsync(batch, handler);
+ verify(handler).onError(error);
+ assertEquals(0, memory.getMemoryBlock().getUsedMemoryInBytes());
+ }
+
+ @Test
+ public void testSynchronousSendFailureReleasesWireMemory() throws Exception {
+ TException error = new TException();
+ doThrow(error).when(client).syncLogEntries(any(), any());
+ thread.sendBatchAsync(batch, handler);
+ verify(handler).onError(error);
+ assertEquals(0, memory.getMemoryBlock().getUsedMemoryInBytes());
+ }
+}
diff --git a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcherThreadMetricsTest.java b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcherThreadMetricsTest.java
index 4da003936a9b5..31980378f4301 100644
--- a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcherThreadMetricsTest.java
+++ b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcherThreadMetricsTest.java
@@ -27,6 +27,7 @@
import org.apache.iotdb.consensus.common.Peer;
import org.apache.iotdb.metrics.config.MetricConfig;
import org.apache.iotdb.metrics.config.MetricConfigDescriptor;
+import org.apache.iotdb.metrics.type.Counter;
import org.apache.iotdb.metrics.type.IMetric;
import org.apache.iotdb.metrics.type.Timer;
import org.apache.iotdb.metrics.utils.MetricInfo;
@@ -38,6 +39,7 @@
import org.mockito.Mockito;
import java.util.Map;
+import java.util.Objects;
import java.util.stream.Collectors;
import static org.junit.Assert.assertEquals;
@@ -116,6 +118,45 @@ public void testStageTimersAreRemovedAfterBindingTwice() {
assertNull(findConstructBatchTimer());
}
+ @Test
+ public void testCompressionCountersShareRegionOwnership() {
+ LogDispatcherThreadMetrics first = new LogDispatcherThreadMetrics(mockThread(2));
+ LogDispatcherThreadMetrics second = new LogDispatcherThreadMetrics(mockThread(3));
+ service.addMetricSet(first);
+ service.addMetricSet(second);
+ try {
+ service.restartService();
+ first.recordCompressionBytes(10, 4, true);
+ second.recordCompressionBytes(20, 6, false);
+ assertEquals(30, findCompressionCounter(Metric.IOT_LOG_COMPRESSION_BYTES, "raw").getCount());
+ assertEquals(
+ 10, findCompressionCounter(Metric.IOT_LOG_COMPRESSION_BYTES, "encoded").getCount());
+ assertEquals(
+ 1, findCompressionCounter(Metric.IOT_LOG_COMPRESSION_MEMORY_FALLBACKS, null).getCount());
+ service.removeMetricSet(second);
+ first.recordCompressionBytes(5, 2, true);
+ assertEquals(35, findCompressionCounter(Metric.IOT_LOG_COMPRESSION_BYTES, "raw").getCount());
+ assertEquals(
+ 2, findCompressionCounter(Metric.IOT_LOG_COMPRESSION_MEMORY_FALLBACKS, null).getCount());
+ } finally {
+ service.removeMetricSet(first);
+ service.removeMetricSet(second);
+ }
+ assertNull(findCompressionCounter(Metric.IOT_LOG_COMPRESSION_BYTES, "raw"));
+ assertNull(findCompressionCounter(Metric.IOT_LOG_COMPRESSION_BYTES, "encoded"));
+ assertNull(findCompressionCounter(Metric.IOT_LOG_COMPRESSION_MEMORY_FALLBACKS, null));
+ }
+
+ private Counter findCompressionCounter(Metric metric, String type) {
+ for (Map.Entry entry : service.getAllMetrics().entrySet()) {
+ Map tags = entry.getKey().getTags();
+ if (metric.toString().equals(entry.getKey().getName())
+ && REGION.toString().equals(tags.get(Tag.REGION.toString()))
+ && Objects.equals(type, tags.get(Tag.TYPE.toString()))) return (Counter) entry.getValue();
+ }
+ return null;
+ }
+
private static LogDispatcher.LogDispatcherThread mockThread(final int peerNodeId) {
final LogDispatcher.LogDispatcherThread thread =
Mockito.mock(LogDispatcher.LogDispatcherThread.class);
diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java
index c19b950208a1b..05aa4d589f66f 100644
--- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java
+++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java
@@ -21,6 +21,7 @@
/** Compile-time i18n constants for DataNode misc subsystems (English). */
public final class DataNodeMiscMessages {
+ public static final String EXCEPTION_INVALID_IOTCONSENSUS_LOG_COMPRESSION_CONFIGURATION_CODEC_ARG_MIN_SIZE_ARG_AB7603FF = "Invalid IoTConsensus log compression configuration: codec=%s, min_size=%s";
public static final String MESSAGE_QUERY_DOES_NOT_BELONG_TO_CURRENT_SESSION_A1198237 =
"The requested query does not belong to the current session.";
diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java
index 93384a3296ee5..b94b81b94c99e 100644
--- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java
+++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java
@@ -21,6 +21,7 @@
/** 编译时国际化常量 - DataNode 杂项子系统(中文)。 */
public final class DataNodeMiscMessages {
+ public static final String EXCEPTION_INVALID_IOTCONSENSUS_LOG_COMPRESSION_CONFIGURATION_CODEC_ARG_MIN_SIZE_ARG_AB7603FF = "无效的 IoTConsensus 日志压缩配置:codec=%s, min_size=%s";
public static final String MESSAGE_QUERY_DOES_NOT_BELONG_TO_CURRENT_SESSION_A1198237 =
"请求的查询不属于当前会话。";
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index c6a0d4878969f..03dc68ca78c43 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -30,6 +30,7 @@
import org.apache.iotdb.commons.pipe.config.PipeConfig;
import org.apache.iotdb.commons.utils.FileUtils;
import org.apache.iotdb.consensus.ConsensusFactory;
+import org.apache.iotdb.consensus.iot.thrift.TLogCompressionType;
import org.apache.iotdb.db.exception.LoadConfigurationException;
import org.apache.iotdb.db.i18n.DataNodeMiscMessages;
import org.apache.iotdb.db.protocol.thrift.impl.ClientRPCServiceImpl;
@@ -1130,6 +1131,8 @@ public class IoTDBConfig implements DefaultEncodingProvider {
// IoTConsensus Config
private int maxLogEntriesNumPerBatch = 1024;
private int maxSizePerBatch = 16 * 1024 * 1024;
+ private TLogCompressionType iotLogCompressionType = TLogCompressionType.NONE;
+ private int iotLogCompressionMinSize = 4096;
private int maxPendingBatchesNum = 5;
private double maxMemoryRatioForQueue = 0.6;
private long regionMigrationSpeedLimitBytesPerSecond = 48 * 1024 * 1024L;
@@ -1309,6 +1312,22 @@ public int getMaxLogEntriesNumPerBatch() {
return maxLogEntriesNumPerBatch;
}
+ public TLogCompressionType getIotLogCompressionType() {
+ return iotLogCompressionType;
+ }
+
+ public void setIotLogCompressionType(TLogCompressionType type) {
+ iotLogCompressionType = type;
+ }
+
+ public int getIotLogCompressionMinSize() {
+ return iotLogCompressionMinSize;
+ }
+
+ public void setIotLogCompressionMinSize(int size) {
+ iotLogCompressionMinSize = size;
+ }
+
public int getMaxSizePerBatch() {
return maxSizePerBatch;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 04fe3381b7541..0c2b9d5145c7b 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -39,6 +39,7 @@
import org.apache.iotdb.confignode.rpc.thrift.TGlobalConfig;
import org.apache.iotdb.confignode.rpc.thrift.TRatisConfig;
import org.apache.iotdb.consensus.config.IoTConsensusV2Config;
+import org.apache.iotdb.consensus.iot.thrift.TLogCompressionType;
import org.apache.iotdb.db.conf.rest.IoTDBRestServiceDescriptor;
import org.apache.iotdb.db.consensus.DataRegionConsensusImpl;
import org.apache.iotdb.db.i18n.DataNodeMiscMessages;
@@ -95,6 +96,7 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import java.util.Locale;
import java.util.Optional;
import java.util.Properties;
import java.util.ServiceLoader;
@@ -1192,6 +1194,25 @@ public void loadProperties(TrimProperties properties) throws BadNodeUrlException
properties.getProperty(
"schema_thread_count", Integer.toString(conf.getSchemaThreadCount()))));
+ // Sending policy is a startup setting. Decoding is always available for in-flight requests.
+ String compression = properties.getProperty("data_region_iot_log_compression", "NONE").trim();
+ String compressionMinimum =
+ properties.getProperty("data_region_iot_log_compression_min_size", "4096");
+ try {
+ TLogCompressionType type = TLogCompressionType.valueOf(compression.toUpperCase(Locale.ROOT));
+ int minimum = Integer.parseInt(compressionMinimum);
+ if (minimum < 0) throw new IllegalArgumentException();
+ conf.setIotLogCompressionType(type);
+ conf.setIotLogCompressionMinSize(minimum);
+ } catch (IllegalArgumentException e) {
+ throw new IOException(
+ String.format(
+ DataNodeMiscMessages
+ .EXCEPTION_INVALID_IOTCONSENSUS_LOG_COMPRESSION_CONFIGURATION_CODEC_ARG_MIN_SIZE_ARG_AB7603FF,
+ compression,
+ compressionMinimum),
+ e);
+ }
loadIoTConsensusProps(properties);
loadIoTConsensusV2Props(properties);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/DataRegionConsensusImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/DataRegionConsensusImpl.java
index ba08b7379bcc3..0bbebf1a43d8d 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/DataRegionConsensusImpl.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/DataRegionConsensusImpl.java
@@ -197,6 +197,8 @@ private static ConsensusConfig buildConsensusConfig() {
.setConsensusMemoryBlock(memoryBlock)
.setMaxLogEntriesNumPerBatch(CONF.getMaxLogEntriesNumPerBatch())
.setMaxSizePerBatch(CONF.getMaxSizePerBatch())
+ .setLogCompressionType(CONF.getIotLogCompressionType())
+ .setLogCompressionMinSize(CONF.getIotLogCompressionMinSize())
.setMaxPendingBatchesNum(CONF.getMaxPendingBatchesNum())
.setMaxMemoryRatioForQueue(CONF.getMaxMemoryRatioForQueue())
.setRegionMigrationSpeedLimitBytesPerSecond(
diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
index a3fe0970318c8..8fb4c65bde372 100644
--- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
+++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
@@ -1873,6 +1873,20 @@ enable_wal_compression=true
### IoTConsensus Configuration
####################
+# IoTConsensus V1 log payload codec: NONE, LZ4, SNAPPY, ZSTD (level 1).
+# Enable only after every peer has the same supporting version and configuration.
+# This controls sending; receivers can decode every supported codec even when set to NONE.
+# Independent of WAL/TsFile compression and dn_rpc_thrift_compression_enable.
+# effectiveMode: restart
+# Datatype: String
+data_region_iot_log_compression = NONE
+
+# Minimum individual log buffer size in bytes to try compression.
+# Buffers and entries that do not save space including descriptors are sent uncompressed.
+# effectiveMode: restart
+# Datatype: int
+data_region_iot_log_compression_min_size = 4096
+
# The maximum log entries num in IoTConsensus Batch
# effectiveMode: hot_reload
# Datatype: int
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java
index a4b853ddc9dc8..8f780d57c32d8 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java
@@ -61,6 +61,8 @@ public enum Metric {
STAGE("stage"),
IOT_CONSENSUS("iot_consensus"),
IOT_SEND_LOG("iot_send_log"),
+ IOT_LOG_COMPRESSION_BYTES("iot_log_compression_bytes"),
+ IOT_LOG_COMPRESSION_MEMORY_FALLBACKS("iot_log_compression_memory_fallbacks"),
IOT_RECEIVE_LOG("iot_receive_log"),
IOT_CONSENSUS_V2("iot_consensus_v2"),
IOT_CONSENSUS_V2_MODE("iot_consensus_v2_mode"),
diff --git a/iotdb-protocol/thrift-consensus/src/main/thrift/iotconsensus.thrift b/iotdb-protocol/thrift-consensus/src/main/thrift/iotconsensus.thrift
index 169a7961e4472..cbf0c87576fb1 100644
--- a/iotdb-protocol/thrift-consensus/src/main/thrift/iotconsensus.thrift
+++ b/iotdb-protocol/thrift-consensus/src/main/thrift/iotconsensus.thrift
@@ -20,6 +20,20 @@
include "common.thrift"
namespace java org.apache.iotdb.consensus.iot.thrift
+// Codec identifiers are wire values, independent of TsFile compression identifiers.
+enum TLogCompressionType {
+ NONE = 0,
+ LZ4 = 1,
+ SNAPPY = 2,
+ ZSTD = 3
+}
+
+struct TLogBufferCompression {
+ 1: required TLogCompressionType codec
+ 2: required i32 uncompressedSize
+ 3: required i32 checksum
+}
+
// If this TLogEntry comes from the queue in memory, the list size of the data will only be 1.
// If this TLogEntry comes from wal, the list size of the data may be greater than 1.
struct TLogEntry {
@@ -29,6 +43,8 @@ struct TLogEntry {
4: required i64 memorySize
5: optional i64 routingEpoch
6: optional i64 physicalTime
+ // One descriptor per data buffer; absent means the original, uncompressed entry.
+ 7: optional list compressionInfo
}
struct TSyncLogEntriesReq {
@@ -41,6 +57,8 @@ struct TSyncLogEntriesReq {
struct TSyncLogEntriesRes {
1: required list statuses
2: optional i64 receiverMemSize
+ // Rejected before application. This must never be treated as a completed batch.
+ 3: optional common.TSStatus logCompressionError
}
struct TSyncWriterSafeTimeBarrierReq {
From e9874ce08a3e365898b522ab7950cc48d0fa39cd Mon Sep 17 00:00:00 2001
From: 761417898 <761417898@qq.com>
Date: Thu, 12 Nov 2026 18:35:49 +0800
Subject: [PATCH 2/5] Use a fixed minimum size for IoTConsensus log compression
---
.../consensus/i18n/IoTConsensusMessages.java | 1 -
.../consensus/i18n/IoTConsensusMessages.java | 1 -
.../consensus/config/IoTConsensusConfig.java | 22 --------
.../iot/logdispatcher/LogCompression.java | 9 ++--
.../iot/logdispatcher/LogDispatcher.java | 1 -
.../iot/logdispatcher/LogCompressionTest.java | 53 ++++++++++++++-----
.../iotdb/db/i18n/DataNodeMiscMessages.java | 2 +-
.../iotdb/db/i18n/DataNodeMiscMessages.java | 2 +-
.../org/apache/iotdb/db/conf/IoTDBConfig.java | 9 ----
.../apache/iotdb/db/conf/IoTDBDescriptor.java | 10 +---
.../db/consensus/DataRegionConsensusImpl.java | 1 -
.../conf/iotdb-system.properties.template | 7 +--
12 files changed, 51 insertions(+), 67 deletions(-)
diff --git a/iotdb-core/consensus/src/main/i18n/en/org/apache/iotdb/consensus/i18n/IoTConsensusMessages.java b/iotdb-core/consensus/src/main/i18n/en/org/apache/iotdb/consensus/i18n/IoTConsensusMessages.java
index 3ebcd8f93e839..a18447f3744c5 100644
--- a/iotdb-core/consensus/src/main/i18n/en/org/apache/iotdb/consensus/i18n/IoTConsensusMessages.java
+++ b/iotdb-core/consensus/src/main/i18n/en/org/apache/iotdb/consensus/i18n/IoTConsensusMessages.java
@@ -25,7 +25,6 @@
*/
public final class IoTConsensusMessages {
- public static final String EXCEPTION_INVALID_IOTCONSENSUS_LOG_COMPRESSION_MINIMUM_SIZE_ARG_ACEEA255 = "Invalid IoTConsensus log compression minimum size: %d";
public static final String EXCEPTION_INVALID_IOTCONSENSUS_COMPRESSED_LOG_ARG_EXPECTED_ARG_ACTUAL_ARG_4986175E = "Invalid IoTConsensus compressed log (%s, expected %d, actual %d)";
diff --git a/iotdb-core/consensus/src/main/i18n/zh/org/apache/iotdb/consensus/i18n/IoTConsensusMessages.java b/iotdb-core/consensus/src/main/i18n/zh/org/apache/iotdb/consensus/i18n/IoTConsensusMessages.java
index 5b2ddc42f860d..1d915b9af66cb 100644
--- a/iotdb-core/consensus/src/main/i18n/zh/org/apache/iotdb/consensus/i18n/IoTConsensusMessages.java
+++ b/iotdb-core/consensus/src/main/i18n/zh/org/apache/iotdb/consensus/i18n/IoTConsensusMessages.java
@@ -24,7 +24,6 @@
*/
public final class IoTConsensusMessages {
- public static final String EXCEPTION_INVALID_IOTCONSENSUS_LOG_COMPRESSION_MINIMUM_SIZE_ARG_ACEEA255 = "无效的 IoTConsensus 日志压缩最小大小:%d";
public static final String EXCEPTION_INVALID_IOTCONSENSUS_COMPRESSED_LOG_ARG_EXPECTED_ARG_ACTUAL_ARG_4986175E = "无效的 IoTConsensus 压缩日志(%s,预期 %d,实际 %d)";
diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/config/IoTConsensusConfig.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/config/IoTConsensusConfig.java
index c31197f0334eb..742992bba1757 100644
--- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/config/IoTConsensusConfig.java
+++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/config/IoTConsensusConfig.java
@@ -22,7 +22,6 @@
import org.apache.iotdb.commons.client.property.ClientPoolProperty.DefaultProperty;
import org.apache.iotdb.commons.memory.AtomicLongMemoryBlock;
import org.apache.iotdb.commons.memory.IMemoryBlock;
-import org.apache.iotdb.consensus.i18n.IoTConsensusMessages;
import org.apache.iotdb.consensus.iot.thrift.TLogCompressionType;
import java.util.Optional;
@@ -313,7 +312,6 @@ public static class Replication {
private final int maxLogEntriesNumPerBatch;
private final int maxSizePerBatch;
private final TLogCompressionType logCompressionType;
- private final int logCompressionMinSize;
private final int maxPendingBatchesNum;
private final int maxQueueLength;
@@ -335,7 +333,6 @@ private Replication(
int maxLogEntriesNumPerBatch,
int maxSizePerBatch,
TLogCompressionType logCompressionType,
- int logCompressionMinSize,
int maxPendingBatchesNum,
int maxQueueLength,
long maxWaitingTimeForWaitBatchInMs,
@@ -354,7 +351,6 @@ private Replication(
this.maxLogEntriesNumPerBatch = maxLogEntriesNumPerBatch;
this.maxSizePerBatch = maxSizePerBatch;
this.logCompressionType = logCompressionType;
- this.logCompressionMinSize = logCompressionMinSize;
this.maxPendingBatchesNum = maxPendingBatchesNum;
this.maxQueueLength = maxQueueLength;
this.maxWaitingTimeForWaitBatchInMs = maxWaitingTimeForWaitBatchInMs;
@@ -384,10 +380,6 @@ public TLogCompressionType getLogCompressionType() {
return logCompressionType;
}
- public int getLogCompressionMinSize() {
- return logCompressionMinSize;
- }
-
public int getMaxPendingBatchesNum() {
return maxPendingBatchesNum;
}
@@ -457,7 +449,6 @@ public static class Builder {
private int maxLogEntriesNumPerBatch = 1024;
private int maxSizePerBatch = 16 * 1024 * 1024;
private TLogCompressionType logCompressionType = TLogCompressionType.NONE;
- private int logCompressionMinSize = 4096;
private int maxPendingBatchesNum = 5;
private int maxQueueLength = 4096;
private long maxWaitingTimeForWaitBatchInMs = 10 * 1000L;
@@ -496,18 +487,6 @@ public Builder setLogCompressionType(TLogCompressionType type) {
return this;
}
- public Builder setLogCompressionMinSize(int size) {
- if (size < 0) {
- throw new IllegalArgumentException(
- String.format(
- IoTConsensusMessages
- .EXCEPTION_INVALID_IOTCONSENSUS_LOG_COMPRESSION_MINIMUM_SIZE_ARG_ACEEA255,
- size));
- }
- this.logCompressionMinSize = size;
- return this;
- }
-
public Replication.Builder setMaxPendingBatchesNum(int maxPendingBatchesNum) {
this.maxPendingBatchesNum = maxPendingBatchesNum;
return this;
@@ -593,7 +572,6 @@ public Replication build() {
maxLogEntriesNumPerBatch,
maxSizePerBatch,
logCompressionType,
- logCompressionMinSize,
maxPendingBatchesNum,
maxQueueLength,
maxWaitingTimeForWaitBatchInMs,
diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompression.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompression.java
index 1162873f2c4de..24c80984a9c07 100644
--- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompression.java
+++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompression.java
@@ -41,6 +41,9 @@
/** Wire-only encoding: the original batch and its buffer positions remain available for retries. */
public final class LogCompression {
+ // Avoid codec and descriptor overhead on tiny blocks; larger blocks still need a net size
+ // benefit.
+ private static final int MIN_COMPRESSION_SIZE = 4096;
// Conservative allowance for the additional Thrift descriptor and list field on the wire.
private static final int DESCRIPTOR_BYTES = 32;
private static final ICompressor LZ4 = ICompressor.getCompressor(CompressionType.LZ4);
@@ -100,7 +103,7 @@ public void close() {
}
public static EncodedRequest compress(
- TSyncLogEntriesReq original, TLogCompressionType codec, int minSize, int maxDecodedBytes)
+ TSyncLogEntriesReq original, TLogCompressionType codec, int maxDecodedBytes)
throws IOException {
long buffers = 0, rawSize = 0;
boolean hasEligibleBuffer = false;
@@ -108,7 +111,7 @@ public static EncodedRequest compress(
for (ByteBuffer buffer : entry.getData()) {
int size = buffer.remaining();
rawSize += size;
- if (size >= minSize && size > 0) {
+ if (size >= MIN_COMPRESSION_SIZE) {
hasEligibleBuffer = true;
}
buffers++;
@@ -138,7 +141,7 @@ public static EncodedRequest compress(
raw += size;
ByteBuffer encoded = buffer;
TLogCompressionType actualCodec = TLogCompressionType.NONE;
- if (size >= minSize && size > 0) {
+ if (size >= MIN_COMPRESSION_SIZE) {
// Includes an optional input copy, codec-bound output and compact output, with
// allowance for array headers and codec bounds on very small buffers.
long workspace = 4L * size + 256;
diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcher.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcher.java
index 7f622c000da5e..972265a4fc09b 100644
--- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcher.java
+++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcher.java
@@ -609,7 +609,6 @@ public void sendBatchAsync(Batch batch, DispatchLogHandler handler) {
LogCompression.compress(
req,
config.getReplication().getLogCompressionType(),
- config.getReplication().getLogCompressionMinSize(),
config.getRpc().getThriftMaxFrameSize());
logDispatcherThreadMetrics.recordCompressTime(System.nanoTime() - compressionStart);
logDispatcherThreadMetrics.recordCompressionBytes(
diff --git a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompressionTest.java b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompressionTest.java
index 09eabafb68198..14a351e1db170 100644
--- a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompressionTest.java
+++ b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompressionTest.java
@@ -137,7 +137,7 @@ public void testRoundTripThriftAndRetryOwnership() throws Exception {
TSyncLogEntriesReq raw = request();
TSyncLogEntriesReq snapshot = raw.deepCopy();
for (int attempt = 0; attempt < 2; attempt++) {
- LogCompression.EncodedRequest encoded = LogCompression.compress(raw, codec, 4096, LIMIT);
+ LogCompression.EncodedRequest encoded = LogCompression.compress(raw, codec, LIMIT);
try {
assertNotSame(raw, encoded.getRequest());
assertEquals(
@@ -169,10 +169,40 @@ public void testRoundTripThriftAndRetryOwnership() throws Exception {
}
}
+ @Test
+ public void testFixedMinimumSizeBoundary() throws Exception {
+ for (int size : new int[] {4095, 4096, 4097}) {
+ TSyncLogEntriesReq raw =
+ new TSyncLogEntriesReq(
+ 1,
+ new TConsensusGroupId(TConsensusGroupType.DataRegion, 1),
+ Collections.singletonList(
+ new TLogEntry(
+ Collections.singletonList(ByteBuffer.allocate(size)), 1, false, size)));
+ try (LogCompression.EncodedRequest encoded = LogCompression.compress(raw, codec, LIMIT)) {
+ if (size < 4096) {
+ assertSame(raw, encoded.getRequest());
+ assertFalse(encoded.getRequest().getLogEntries().get(0).isSetCompressionInfo());
+ } else {
+ assertNotSame(raw, encoded.getRequest());
+ assertEquals(
+ codec,
+ encoded.getRequest().getLogEntries().get(0).getCompressionInfo().get(0).getCodec());
+ }
+ TSyncLogEntriesReq received = encoded.getRequest().deepCopy();
+ LogCompression.decompress(received, LIMIT);
+ assertEquals(
+ raw.getLogEntries().get(0).getData(), received.getLogEntries().get(0).getData());
+ assertEquals(0, raw.getLogEntries().get(0).getData().get(0).position());
+ }
+ }
+ }
+
@Test
public void testAdaptiveAndMemoryFallback() throws Exception {
TSyncLogEntriesReq raw = request();
- try (LogCompression.EncodedRequest result = LogCompression.compress(raw, codec, LIMIT, LIMIT)) {
+ try (LogCompression.EncodedRequest result =
+ LogCompression.compress(raw, TLogCompressionType.NONE, LIMIT)) {
assertSame(raw, result.getRequest());
}
byte[] random = new byte[16384];
@@ -182,12 +212,12 @@ public void testAdaptiveAndMemoryFallback() throws Exception {
Collections.singletonList(
new TLogEntry(
Collections.singletonList(ByteBuffer.wrap(random)), 1, false, random.length)));
- try (LogCompression.EncodedRequest result = LogCompression.compress(noise, codec, 0, LIMIT)) {
+ try (LogCompression.EncodedRequest result = LogCompression.compress(noise, codec, LIMIT)) {
assertSame(noise, result.getRequest());
}
long capacity = memory.getMemoryBlock().getTotalMemorySizeInBytes();
assertTrue(memory.reserveLogCompression(capacity));
- try (LogCompression.EncodedRequest result = LogCompression.compress(raw, codec, 0, LIMIT)) {
+ try (LogCompression.EncodedRequest result = LogCompression.compress(raw, codec, LIMIT)) {
assertSame(raw, result.getRequest());
assertTrue(result.isMemoryLimited());
assertEquals(result.getRawBytes(), result.getEncodedBytes());
@@ -199,8 +229,7 @@ public void testAdaptiveAndMemoryFallback() throws Exception {
@Test
public void testRejectsEntireRequestBeforePublishingAnyDecodedEntry() throws Exception {
- try (LogCompression.EncodedRequest encoded =
- LogCompression.compress(request(), codec, 4096, LIMIT)) {
+ try (LogCompression.EncodedRequest encoded = LogCompression.compress(request(), codec, LIMIT)) {
TSyncLogEntriesReq bad = encoded.getRequest().deepCopy();
bad.getLogEntries().get(1).getCompressionInfo().get(0).setChecksum(0);
TSyncLogEntriesReq snapshot = bad.deepCopy();
@@ -216,8 +245,7 @@ public void testRejectsEntireRequestBeforePublishingAnyDecodedEntry() throws Exc
@Test
public void testMalformedMetadataAndCorruptedCodecInput() throws Exception {
- try (LogCompression.EncodedRequest encoded =
- LogCompression.compress(request(), codec, 4096, LIMIT)) {
+ try (LogCompression.EncodedRequest encoded = LogCompression.compress(request(), codec, LIMIT)) {
TSyncLogEntriesReq count = encoded.getRequest().deepCopy();
count.getLogEntries().get(0).getCompressionInfo().remove(0);
assertThrows(IOException.class, () -> LogCompression.decompress(count, LIMIT));
@@ -256,7 +284,7 @@ public void testSingleLogAboveSoftBatchLimit() throws Exception {
1,
false,
2 * 1024 * 1024)));
- try (LogCompression.EncodedRequest encoded = LogCompression.compress(raw, codec, 4096, LIMIT)) {
+ try (LogCompression.EncodedRequest encoded = LogCompression.compress(raw, codec, LIMIT)) {
TSyncLogEntriesReq received = encoded.getRequest().deepCopy();
LogCompression.decompress(received, LIMIT);
assertEquals(raw, received);
@@ -276,8 +304,7 @@ public void testProcessorRejectsBeforeDeserializeAndDecodesWithSendingDisabled()
when(impl.getIoTConsensusServerMetrics()).thenReturn(mock(IoTConsensusServerMetrics.class));
when(impl.getStateMachine()).thenReturn(stateMachine);
IoTConsensusRPCServiceProcessor processor = new IoTConsensusRPCServiceProcessor(consensus);
- try (LogCompression.EncodedRequest encoded =
- LogCompression.compress(request(), codec, 4096, LIMIT)) {
+ try (LogCompression.EncodedRequest encoded = LogCompression.compress(request(), codec, LIMIT)) {
TSyncLogEntriesReq corrupt = encoded.getRequest().deepCopy();
corrupt.getLogEntries().get(1).getCompressionInfo().get(0).setChecksum(0);
assertTrue(processor.syncLogEntries(corrupt).isSetLogCompressionError());
@@ -318,7 +345,7 @@ public void testMultiBlockScratchBudgetMatchesSequentialPeak() throws Exception
new TLogEntry(
Collections.singletonList(ByteBuffer.allocate(16384)), i + 1, false, 16384));
raw.setLogEntries(entries);
- try (LogCompression.EncodedRequest encoded = LogCompression.compress(raw, codec, 4096, LIMIT)) {
+ try (LogCompression.EncodedRequest encoded = LogCompression.compress(raw, codec, LIMIT)) {
assertNotSame(raw, encoded.getRequest());
assertEquals(16 * 16384, encoded.getRawBytes());
assertFalse(encoded.isMemoryLimited());
@@ -343,7 +370,7 @@ public void testMemoryPressureFallsBackPerBlockAndKeepsOtherBlocksCompressed() t
1,
true,
40960)));
- try (LogCompression.EncodedRequest encoded = LogCompression.compress(raw, codec, 4096, LIMIT)) {
+ try (LogCompression.EncodedRequest encoded = LogCompression.compress(raw, codec, LIMIT)) {
assertNotSame(raw, encoded.getRequest());
assertTrue(encoded.isMemoryLimited());
assertEquals(
diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java
index 05aa4d589f66f..1fb140df616a5 100644
--- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java
+++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java
@@ -21,7 +21,7 @@
/** Compile-time i18n constants for DataNode misc subsystems (English). */
public final class DataNodeMiscMessages {
- public static final String EXCEPTION_INVALID_IOTCONSENSUS_LOG_COMPRESSION_CONFIGURATION_CODEC_ARG_MIN_SIZE_ARG_AB7603FF = "Invalid IoTConsensus log compression configuration: codec=%s, min_size=%s";
+ public static final String EXCEPTION_INVALID_IOTCONSENSUS_LOG_COMPRESSION_CODEC_ARG_2B9C27F7 = "Invalid IoTConsensus log compression codec: %s";
public static final String MESSAGE_QUERY_DOES_NOT_BELONG_TO_CURRENT_SESSION_A1198237 =
"The requested query does not belong to the current session.";
diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java
index b94b81b94c99e..74f25d0d9a955 100644
--- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java
+++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java
@@ -21,7 +21,7 @@
/** 编译时国际化常量 - DataNode 杂项子系统(中文)。 */
public final class DataNodeMiscMessages {
- public static final String EXCEPTION_INVALID_IOTCONSENSUS_LOG_COMPRESSION_CONFIGURATION_CODEC_ARG_MIN_SIZE_ARG_AB7603FF = "无效的 IoTConsensus 日志压缩配置:codec=%s, min_size=%s";
+ public static final String EXCEPTION_INVALID_IOTCONSENSUS_LOG_COMPRESSION_CODEC_ARG_2B9C27F7 = "无效的 IoTConsensus 日志压缩算法:%s";
public static final String MESSAGE_QUERY_DOES_NOT_BELONG_TO_CURRENT_SESSION_A1198237 =
"请求的查询不属于当前会话。";
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 03dc68ca78c43..41dc78446d863 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -1132,7 +1132,6 @@ public class IoTDBConfig implements DefaultEncodingProvider {
private int maxLogEntriesNumPerBatch = 1024;
private int maxSizePerBatch = 16 * 1024 * 1024;
private TLogCompressionType iotLogCompressionType = TLogCompressionType.NONE;
- private int iotLogCompressionMinSize = 4096;
private int maxPendingBatchesNum = 5;
private double maxMemoryRatioForQueue = 0.6;
private long regionMigrationSpeedLimitBytesPerSecond = 48 * 1024 * 1024L;
@@ -1320,14 +1319,6 @@ public void setIotLogCompressionType(TLogCompressionType type) {
iotLogCompressionType = type;
}
- public int getIotLogCompressionMinSize() {
- return iotLogCompressionMinSize;
- }
-
- public void setIotLogCompressionMinSize(int size) {
- iotLogCompressionMinSize = size;
- }
-
public int getMaxSizePerBatch() {
return maxSizePerBatch;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 0c2b9d5145c7b..ce526ff448f59 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -1196,21 +1196,15 @@ public void loadProperties(TrimProperties properties) throws BadNodeUrlException
// Sending policy is a startup setting. Decoding is always available for in-flight requests.
String compression = properties.getProperty("data_region_iot_log_compression", "NONE").trim();
- String compressionMinimum =
- properties.getProperty("data_region_iot_log_compression_min_size", "4096");
try {
TLogCompressionType type = TLogCompressionType.valueOf(compression.toUpperCase(Locale.ROOT));
- int minimum = Integer.parseInt(compressionMinimum);
- if (minimum < 0) throw new IllegalArgumentException();
conf.setIotLogCompressionType(type);
- conf.setIotLogCompressionMinSize(minimum);
} catch (IllegalArgumentException e) {
throw new IOException(
String.format(
DataNodeMiscMessages
- .EXCEPTION_INVALID_IOTCONSENSUS_LOG_COMPRESSION_CONFIGURATION_CODEC_ARG_MIN_SIZE_ARG_AB7603FF,
- compression,
- compressionMinimum),
+ .EXCEPTION_INVALID_IOTCONSENSUS_LOG_COMPRESSION_CODEC_ARG_2B9C27F7,
+ compression),
e);
}
loadIoTConsensusProps(properties);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/DataRegionConsensusImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/DataRegionConsensusImpl.java
index 0bbebf1a43d8d..6efaee537e190 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/DataRegionConsensusImpl.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/DataRegionConsensusImpl.java
@@ -198,7 +198,6 @@ private static ConsensusConfig buildConsensusConfig() {
.setMaxLogEntriesNumPerBatch(CONF.getMaxLogEntriesNumPerBatch())
.setMaxSizePerBatch(CONF.getMaxSizePerBatch())
.setLogCompressionType(CONF.getIotLogCompressionType())
- .setLogCompressionMinSize(CONF.getIotLogCompressionMinSize())
.setMaxPendingBatchesNum(CONF.getMaxPendingBatchesNum())
.setMaxMemoryRatioForQueue(CONF.getMaxMemoryRatioForQueue())
.setRegionMigrationSpeedLimitBytesPerSecond(
diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
index 8fb4c65bde372..9e1bbe1fddab6 100644
--- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
+++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
@@ -1877,16 +1877,11 @@ enable_wal_compression=true
# Enable only after every peer has the same supporting version and configuration.
# This controls sending; receivers can decode every supported codec even when set to NONE.
# Independent of WAL/TsFile compression and dn_rpc_thrift_compression_enable.
+# Buffers smaller than 4 KiB or without a net size benefit are sent uncompressed.
# effectiveMode: restart
# Datatype: String
data_region_iot_log_compression = NONE
-# Minimum individual log buffer size in bytes to try compression.
-# Buffers and entries that do not save space including descriptors are sent uncompressed.
-# effectiveMode: restart
-# Datatype: int
-data_region_iot_log_compression_min_size = 4096
-
# The maximum log entries num in IoTConsensus Batch
# effectiveMode: hot_reload
# Datatype: int
From 51bb63c2a8b789705f9087942523f4101f54cbaa Mon Sep 17 00:00:00 2001
From: 761417898 <761417898@qq.com>
Date: Fri, 13 Nov 2026 12:57:32 +0800
Subject: [PATCH 3/5] Remove CRC checks from IoTConsensus log compression
---
.../iot/logdispatcher/LogCompression.java | 22 +++---------------
.../iot/logdispatcher/LogCompressionTest.java | 23 +++++++++++++++----
.../src/main/thrift/iotconsensus.thrift | 1 -
3 files changed, 22 insertions(+), 24 deletions(-)
diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompression.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompression.java
index 24c80984a9c07..4c0e8834df646 100644
--- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompression.java
+++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompression.java
@@ -37,7 +37,6 @@
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.atomic.AtomicLong;
-import java.util.zip.CRC32C;
/** Wire-only encoding: the original batch and its buffer positions remain available for retries. */
public final class LogCompression {
@@ -175,13 +174,11 @@ public static EncodedRequest compress(
}
}
data.add(encoded);
- info.add(new TLogBufferCompression(actualCodec, size, 0));
+ info.add(new TLogBufferCompression(actualCodec, size));
wire += encoded.remaining();
}
// Raw descriptors in a mixed WAL entry cost space too.
if (compressed && wire + DESCRIPTOR_BYTES * (long) data.size() + 8 < raw) {
- for (int i = 0; i < info.size(); i++)
- info.get(i).setChecksum(checksum(entry.getData().get(i)));
entries.add(copy(entry, data).setCompressionInfo(info));
retained += arrays + data.size() * 256L + 128;
result.encodedBytes += wire - raw;
@@ -202,7 +199,7 @@ public static EncodedRequest compress(
}
}
- /** Validate every declared size before allocating; apply nothing until all buffers pass CRC. */
+ /** Validate declared sizes before allocating; publish nothing until every buffer is decoded. */
public static void decompress(TSyncLogEntriesReq request, int maxDecodedBytes)
throws IOException {
boolean encoded = false;
@@ -255,12 +252,6 @@ public static void decompress(TSyncLogEntriesReq request, int maxDecodedBytes)
if (actual != output.length) throw invalid(Failure.LENGTH, output.length, actual);
buffer = ByteBuffer.wrap(output);
}
- int crc = checksum(buffer);
- if (crc != info.getChecksum())
- throw invalid(
- Failure.CHECKSUM,
- Integer.toUnsignedLong(info.getChecksum()),
- Integer.toUnsignedLong(crc));
data.add(buffer);
}
decoded.add(copy(entry, data));
@@ -285,12 +276,6 @@ private static byte[] bytes(ByteBuffer buffer) {
return bytes;
}
- private static int checksum(ByteBuffer buffer) {
- CRC32C crc = new CRC32C();
- crc.update(buffer.duplicate());
- return (int) crc.getValue();
- }
-
private static ICompressor compressor(TLogCompressionType codec) throws IOException {
switch (codec) {
case LZ4:
@@ -319,8 +304,7 @@ private enum Failure {
CODEC,
LENGTH,
LIMIT,
- DESCRIPTORS,
- CHECKSUM
+ DESCRIPTORS
}
private static IOException invalid(Failure failure, long expected, long actual) {
diff --git a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompressionTest.java b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompressionTest.java
index 14a351e1db170..4d8ce6103a182 100644
--- a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompressionTest.java
+++ b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompressionTest.java
@@ -231,18 +231,33 @@ public void testAdaptiveAndMemoryFallback() throws Exception {
public void testRejectsEntireRequestBeforePublishingAnyDecodedEntry() throws Exception {
try (LogCompression.EncodedRequest encoded = LogCompression.compress(request(), codec, LIMIT)) {
TSyncLogEntriesReq bad = encoded.getRequest().deepCopy();
- bad.getLogEntries().get(1).getCompressionInfo().get(0).setChecksum(0);
+ // The first entry decodes successfully before the later entry fails in the codec.
+ bad.getLogEntries().get(1).getData().set(0, ByteBuffer.wrap(new byte[] {0, 1, 2}));
TSyncLogEntriesReq snapshot = bad.deepCopy();
assertThrows(IOException.class, () -> LogCompression.decompress(bad, LIMIT));
assertEquals(snapshot, bad);
- // Also cover corruption of an uncompressed block in a mixed WAL entry.
+ // Raw blocks in mixed entries still have to match their declared lengths.
TSyncLogEntriesReq secondBad = encoded.getRequest().deepCopy();
- secondBad.getLogEntries().get(0).getCompressionInfo().get(2).setChecksum(0);
+ secondBad.getLogEntries().get(0).getCompressionInfo().get(2).setUncompressedSize(4);
final TSyncLogEntriesReq mixed = secondBad;
assertThrows(IOException.class, () -> LogCompression.decompress(mixed, LIMIT));
}
}
+ @Test
+ public void testMixedRawBufferContentIsForwardedWithoutChecksum() throws Exception {
+ try (LogCompression.EncodedRequest encoded = LogCompression.compress(request(), codec, LIMIT)) {
+ TSyncLogEntriesReq received = encoded.getRequest().deepCopy();
+ byte[] changed = {4, 5, 6};
+ received.getLogEntries().get(0).getData().set(2, ByteBuffer.wrap(changed));
+ TSyncLogEntriesReq expected = request();
+ expected.getLogEntries().get(0).getData().set(2, ByteBuffer.wrap(changed));
+ LogCompression.decompress(received, LIMIT);
+ // Same-length raw content is preserved verbatim; there is no payload integrity checksum.
+ assertEquals(expected, received);
+ }
+ }
+
@Test
public void testMalformedMetadataAndCorruptedCodecInput() throws Exception {
try (LogCompression.EncodedRequest encoded = LogCompression.compress(request(), codec, LIMIT)) {
@@ -306,7 +321,7 @@ public void testProcessorRejectsBeforeDeserializeAndDecodesWithSendingDisabled()
IoTConsensusRPCServiceProcessor processor = new IoTConsensusRPCServiceProcessor(consensus);
try (LogCompression.EncodedRequest encoded = LogCompression.compress(request(), codec, LIMIT)) {
TSyncLogEntriesReq corrupt = encoded.getRequest().deepCopy();
- corrupt.getLogEntries().get(1).getCompressionInfo().get(0).setChecksum(0);
+ corrupt.getLogEntries().get(1).getData().set(0, ByteBuffer.wrap(new byte[] {0, 1, 2}));
assertTrue(processor.syncLogEntries(corrupt).isSetLogCompressionError());
// Snappy reports a short output buffer as IllegalArgumentException; the RPC boundary must
// reject it in the same explicit, pre-application error path as codec I/O failures.
diff --git a/iotdb-protocol/thrift-consensus/src/main/thrift/iotconsensus.thrift b/iotdb-protocol/thrift-consensus/src/main/thrift/iotconsensus.thrift
index cbf0c87576fb1..34bb3a85ce550 100644
--- a/iotdb-protocol/thrift-consensus/src/main/thrift/iotconsensus.thrift
+++ b/iotdb-protocol/thrift-consensus/src/main/thrift/iotconsensus.thrift
@@ -31,7 +31,6 @@ enum TLogCompressionType {
struct TLogBufferCompression {
1: required TLogCompressionType codec
2: required i32 uncompressedSize
- 3: required i32 checksum
}
// If this TLogEntry comes from the queue in memory, the list size of the data will only be 1.
From 31f35e67b4890cc21afb3c1477e1de5afb613e1b Mon Sep 17 00:00:00 2001
From: 761417898 <761417898@qq.com>
Date: Fri, 13 Nov 2026 17:29:58 +0800
Subject: [PATCH 4/5] Address IoTConsensus compression naming and code style
review
---
.../iot/logdispatcher/LogCompression.java | 107 +++++++++++-------
1 file changed, 69 insertions(+), 38 deletions(-)
diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompression.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompression.java
index 4c0e8834df646..1aa770121d2f9 100644
--- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompression.java
+++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompression.java
@@ -40,8 +40,7 @@
/** Wire-only encoding: the original batch and its buffer positions remain available for retries. */
public final class LogCompression {
- // Avoid codec and descriptor overhead on tiny blocks; larger blocks still need a net size
- // benefit.
+ // Skip tiny blocks to avoid codec and descriptor overhead.
private static final int MIN_COMPRESSION_SIZE = 4096;
// Conservative allowance for the additional Thrift descriptor and list field on the wire.
private static final int DESCRIPTOR_BYTES = 32;
@@ -91,34 +90,40 @@ public TSyncLogEntriesReq getRequest() {
private void retain(long bytes) {
long released = reserved.getAndSet(bytes) - bytes;
- if (released > 0) IoTConsensusMemoryManager.getInstance().freeLogCompression(released);
+ if (released > 0) {
+ IoTConsensusMemoryManager.getInstance().freeLogCompression(released);
+ }
}
@Override
public void close() {
long bytes = reserved.getAndSet(0);
- if (bytes > 0) IoTConsensusMemoryManager.getInstance().freeLogCompression(bytes);
+ if (bytes > 0) {
+ IoTConsensusMemoryManager.getInstance().freeLogCompression(bytes);
+ }
}
}
public static EncodedRequest compress(
TSyncLogEntriesReq original, TLogCompressionType codec, int maxDecodedBytes)
throws IOException {
- long buffers = 0, rawSize = 0;
+ long buffers = 0;
+ long rawSize = 0;
boolean hasEligibleBuffer = false;
for (TLogEntry entry : original.getLogEntries()) {
for (ByteBuffer buffer : entry.getData()) {
- int size = buffer.remaining();
- rawSize += size;
- if (size >= MIN_COMPRESSION_SIZE) {
+ int uncompressedLength = buffer.remaining();
+ rawSize += uncompressedLength;
+ if (uncompressedLength >= MIN_COMPRESSION_SIZE) {
hasEligibleBuffer = true;
}
buffers++;
}
}
// Preserve the original path for oversized legal entries and when scratch memory is scarce.
- if (codec == TLogCompressionType.NONE || !hasEligibleBuffer || rawSize > maxDecodedBytes)
+ if (codec == TLogCompressionType.NONE || !hasEligibleBuffer || rawSize > maxDecodedBytes) {
return new EncodedRequest(original, 0, rawSize, false);
+ }
// Charge descriptor/list/entry overhead first, then one block's working arrays at a time.
// Retain only actual compact outputs between blocks, not a whole-batch worst-case estimate.
long budget = buffers * 256 + original.getLogEntriesSize() * 128L;
@@ -133,37 +138,42 @@ public static EncodedRequest compress(
for (TLogEntry entry : original.getLogEntries()) {
List data = new ArrayList<>(entry.getDataSize());
List info = new ArrayList<>(entry.getDataSize());
- long raw = 0, wire = 0, arrays = 0;
+ long raw = 0;
+ long wire = 0;
+ long arrays = 0;
boolean compressed = false;
for (ByteBuffer buffer : entry.getData()) {
- int size = buffer.remaining();
- raw += size;
+ int uncompressedLength = buffer.remaining();
+ raw += uncompressedLength;
ByteBuffer encoded = buffer;
TLogCompressionType actualCodec = TLogCompressionType.NONE;
- if (size >= MIN_COMPRESSION_SIZE) {
+ if (uncompressedLength >= MIN_COMPRESSION_SIZE) {
// Includes an optional input copy, codec-bound output and compact output, with
// allowance for array headers and codec bounds on very small buffers.
- long workspace = 4L * size + 256;
+ long workspace = 4L * uncompressedLength + 256;
long before = result.reserved.get();
if (IoTConsensusMemoryManager.getInstance().reserveLogCompression(workspace)) {
result.reserved.addAndGet(workspace);
byte[] input = buffer.hasArray() ? buffer.array() : bytes(buffer);
int offset = buffer.hasArray() ? buffer.arrayOffset() + buffer.position() : 0;
- int bound =
+ int maxCompressedLength =
codec == TLogCompressionType.ZSTD
- ? (int) Zstd.compressBound(size)
- : compressor(codec).getMaxBytesForCompression(size);
- byte[] output = new byte[bound];
- int length =
+ ? (int) Zstd.compressBound(uncompressedLength)
+ : compressor(codec).getMaxBytesForCompression(uncompressedLength);
+ byte[] output = new byte[maxCompressedLength];
+ int compressedLength =
codec == TLogCompressionType.ZSTD
? (int)
- Zstd.compressByteArray(output, 0, output.length, input, offset, size, 1)
- : compressor(codec).compress(input, offset, size, output);
- if (length < 0) throw invalid(Failure.CODEC, size, length);
- if ((long) length + DESCRIPTOR_BYTES < size) {
- encoded = ByteBuffer.wrap(Arrays.copyOf(output, length));
+ Zstd.compressByteArray(
+ output, 0, output.length, input, offset, uncompressedLength, 1)
+ : compressor(codec).compress(input, offset, uncompressedLength, output);
+ if (compressedLength < 0) {
+ throw invalid(Failure.CODEC, uncompressedLength, compressedLength);
+ }
+ if ((long) compressedLength + DESCRIPTOR_BYTES < uncompressedLength) {
+ encoded = ByteBuffer.wrap(Arrays.copyOf(output, compressedLength));
actualCodec = codec;
- arrays += length;
+ arrays += compressedLength;
compressed = true;
}
result.retain(
@@ -174,7 +184,7 @@ public static EncodedRequest compress(
}
}
data.add(encoded);
- info.add(new TLogBufferCompression(actualCodec, size));
+ info.add(new TLogBufferCompression(actualCodec, uncompressedLength));
wire += encoded.remaining();
}
// Raw descriptors in a mixed WAL entry cost space too.
@@ -188,9 +198,10 @@ public static EncodedRequest compress(
result.retain(result.reserved.get() - arrays);
}
}
- if (changed)
+ if (changed) {
result.request =
new TSyncLogEntriesReq(original.getPeerId(), original.getConsensusGroupId(), entries);
+ }
result.retain(retained);
return result;
} catch (IOException | RuntimeException e) {
@@ -207,25 +218,39 @@ public static void decompress(TSyncLogEntriesReq request, int maxDecodedBytes)
for (TLogEntry entry : request.getLogEntries()) {
if (entry.isSetCompressionInfo()) {
encoded = true;
- if (entry.getCompressionInfoSize() != entry.getDataSize())
+ if (entry.getCompressionInfoSize() != entry.getDataSize()) {
throw invalid(Failure.DESCRIPTORS, entry.getDataSize(), entry.getCompressionInfoSize());
+ }
for (int i = 0; i < entry.getDataSize(); i++) {
TLogBufferCompression info = entry.getCompressionInfo().get(i);
int size = info.getUncompressedSize();
- if (size < 0) throw invalid(Failure.LENGTH, 0, size);
- if (info.getCodec() == null) throw invalid(Failure.CODEC, 0, -1);
+ if (size < 0) {
+ throw invalid(Failure.LENGTH, 0, size);
+ }
+ if (info.getCodec() == null) {
+ throw invalid(Failure.CODEC, 0, -1);
+ }
if (info.getCodec() == TLogCompressionType.NONE
- && size != entry.getData().get(i).remaining())
+ && size != entry.getData().get(i).remaining()) {
throw invalid(Failure.LENGTH, size, entry.getData().get(i).remaining());
+ }
total += size;
- if (total > maxDecodedBytes) throw invalid(Failure.LIMIT, maxDecodedBytes, total);
+ if (total > maxDecodedBytes) {
+ throw invalid(Failure.LIMIT, maxDecodedBytes, total);
+ }
}
} else {
- for (ByteBuffer buffer : entry.getData()) total += buffer.remaining();
+ for (ByteBuffer buffer : entry.getData()) {
+ total += buffer.remaining();
+ }
}
}
- if (!encoded) return;
- if (total > maxDecodedBytes) throw invalid(Failure.LIMIT, maxDecodedBytes, total);
+ if (!encoded) {
+ return;
+ }
+ if (total > maxDecodedBytes) {
+ throw invalid(Failure.LIMIT, maxDecodedBytes, total);
+ }
List decoded = new ArrayList<>(request.getLogEntriesSize());
for (TLogEntry entry : request.getLogEntries()) {
if (!entry.isSetCompressionInfo()) {
@@ -249,7 +274,9 @@ public static void decompress(TSyncLogEntriesReq request, int maxDecodedBytes)
failure.initCause(e);
throw failure;
}
- if (actual != output.length) throw invalid(Failure.LENGTH, output.length, actual);
+ if (actual != output.length) {
+ throw invalid(Failure.LENGTH, output.length, actual);
+ }
buffer = ByteBuffer.wrap(output);
}
data.add(buffer);
@@ -264,8 +291,12 @@ public static void decompress(TSyncLogEntriesReq request, int maxDecodedBytes)
private static TLogEntry copy(TLogEntry source, List data) {
TLogEntry entry =
new TLogEntry(data, source.getSearchIndex(), source.isFromWAL(), source.getMemorySize());
- if (source.isSetRoutingEpoch()) entry.setRoutingEpoch(source.getRoutingEpoch());
- if (source.isSetPhysicalTime()) entry.setPhysicalTime(source.getPhysicalTime());
+ if (source.isSetRoutingEpoch()) {
+ entry.setRoutingEpoch(source.getRoutingEpoch());
+ }
+ if (source.isSetPhysicalTime()) {
+ entry.setPhysicalTime(source.getPhysicalTime());
+ }
return entry;
}
From 434292b65c5ed49d8f2bd97f10f61b9f50d07b00 Mon Sep 17 00:00:00 2001
From: 761417898 <761417898@qq.com>
Date: Fri, 13 Nov 2026 17:56:59 +0800
Subject: [PATCH 5/5] Reuse TsFile compression types and default codecs for
IoTConsensus logs
---
iotdb-core/consensus/pom.xml | 4 --
.../consensus/i18n/IoTConsensusMessages.java | 2 +
.../consensus/i18n/IoTConsensusMessages.java | 2 +
.../consensus/config/IoTConsensusConfig.java | 27 ++++++++---
.../iot/logdispatcher/LogCompression.java | 45 ++++++++++++-------
.../iot/logdispatcher/LogDispatcher.java | 4 +-
.../config/IoTConsensusConfigTest.java | 37 +++++++++++++++
.../iot/CompressedReplicationTest.java | 8 ++--
.../iot/logdispatcher/LogCompressionTest.java | 25 +++++++----
.../LogDispatcherCompressionTest.java | 4 +-
.../org/apache/iotdb/db/conf/IoTDBConfig.java | 7 ++-
.../apache/iotdb/db/conf/IoTDBDescriptor.java | 16 +++++--
.../conf/iotdb-system.properties.template | 3 +-
13 files changed, 131 insertions(+), 53 deletions(-)
create mode 100644 iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/config/IoTConsensusConfigTest.java
diff --git a/iotdb-core/consensus/pom.xml b/iotdb-core/consensus/pom.xml
index 922b68801f2fc..7130af748289f 100644
--- a/iotdb-core/consensus/pom.xml
+++ b/iotdb-core/consensus/pom.xml
@@ -114,10 +114,6 @@
com.google.code.findbugs
jsr305
-
- com.github.luben
- zstd-jni
-
org.apache.tsfile
tsfile
diff --git a/iotdb-core/consensus/src/main/i18n/en/org/apache/iotdb/consensus/i18n/IoTConsensusMessages.java b/iotdb-core/consensus/src/main/i18n/en/org/apache/iotdb/consensus/i18n/IoTConsensusMessages.java
index a18447f3744c5..023a6c1f0a6d5 100644
--- a/iotdb-core/consensus/src/main/i18n/en/org/apache/iotdb/consensus/i18n/IoTConsensusMessages.java
+++ b/iotdb-core/consensus/src/main/i18n/en/org/apache/iotdb/consensus/i18n/IoTConsensusMessages.java
@@ -28,6 +28,8 @@ public final class IoTConsensusMessages {
public static final String EXCEPTION_INVALID_IOTCONSENSUS_COMPRESSED_LOG_ARG_EXPECTED_ARG_ACTUAL_ARG_4986175E = "Invalid IoTConsensus compressed log (%s, expected %d, actual %d)";
+ public static final String EXCEPTION_UNSUPPORTED_IOTCONSENSUS_LOG_COMPRESSION_TYPE_ARG_4008DA9A = "Unsupported IoTConsensus log compression type: %s";
+
private IoTConsensusMessages() {}
// ===================== IoTConsensus lifecycle =====================
diff --git a/iotdb-core/consensus/src/main/i18n/zh/org/apache/iotdb/consensus/i18n/IoTConsensusMessages.java b/iotdb-core/consensus/src/main/i18n/zh/org/apache/iotdb/consensus/i18n/IoTConsensusMessages.java
index 1d915b9af66cb..5243af947a0bf 100644
--- a/iotdb-core/consensus/src/main/i18n/zh/org/apache/iotdb/consensus/i18n/IoTConsensusMessages.java
+++ b/iotdb-core/consensus/src/main/i18n/zh/org/apache/iotdb/consensus/i18n/IoTConsensusMessages.java
@@ -27,6 +27,8 @@ public final class IoTConsensusMessages {
public static final String EXCEPTION_INVALID_IOTCONSENSUS_COMPRESSED_LOG_ARG_EXPECTED_ARG_ACTUAL_ARG_4986175E = "无效的 IoTConsensus 压缩日志(%s,预期 %d,实际 %d)";
+ public static final String EXCEPTION_UNSUPPORTED_IOTCONSENSUS_LOG_COMPRESSION_TYPE_ARG_4008DA9A = "不支持的 IoTConsensus 日志压缩类型:%s";
+
private IoTConsensusMessages() {}
// ===================== IoTConsensus 生命周期 =====================
diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/config/IoTConsensusConfig.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/config/IoTConsensusConfig.java
index 742992bba1757..1cd7cb074c23c 100644
--- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/config/IoTConsensusConfig.java
+++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/config/IoTConsensusConfig.java
@@ -22,7 +22,9 @@
import org.apache.iotdb.commons.client.property.ClientPoolProperty.DefaultProperty;
import org.apache.iotdb.commons.memory.AtomicLongMemoryBlock;
import org.apache.iotdb.commons.memory.IMemoryBlock;
-import org.apache.iotdb.consensus.iot.thrift.TLogCompressionType;
+import org.apache.iotdb.consensus.i18n.IoTConsensusMessages;
+
+import org.apache.tsfile.file.metadata.enums.CompressionType;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
@@ -311,7 +313,7 @@ public static class Replication {
private final int maxLogEntriesNumPerBatch;
private final int maxSizePerBatch;
- private final TLogCompressionType logCompressionType;
+ private final CompressionType logCompressionType;
private final int maxPendingBatchesNum;
private final int maxQueueLength;
@@ -332,7 +334,7 @@ public static class Replication {
private Replication(
int maxLogEntriesNumPerBatch,
int maxSizePerBatch,
- TLogCompressionType logCompressionType,
+ CompressionType logCompressionType,
int maxPendingBatchesNum,
int maxQueueLength,
long maxWaitingTimeForWaitBatchInMs,
@@ -376,7 +378,7 @@ public int getMaxSizePerBatch() {
return maxSizePerBatch;
}
- public TLogCompressionType getLogCompressionType() {
+ public CompressionType getLogCompressionType() {
return logCompressionType;
}
@@ -448,7 +450,7 @@ public static class Builder {
private int maxLogEntriesNumPerBatch = 1024;
private int maxSizePerBatch = 16 * 1024 * 1024;
- private TLogCompressionType logCompressionType = TLogCompressionType.NONE;
+ private CompressionType logCompressionType = CompressionType.UNCOMPRESSED;
private int maxPendingBatchesNum = 5;
private int maxQueueLength = 4096;
private long maxWaitingTimeForWaitBatchInMs = 10 * 1000L;
@@ -482,7 +484,20 @@ public Builder setMaxSizePerBatch(int maxSizePerBatch) {
return this;
}
- public Builder setLogCompressionType(TLogCompressionType type) {
+ public Builder setLogCompressionType(CompressionType type) {
+ switch (type) {
+ case UNCOMPRESSED:
+ case LZ4:
+ case SNAPPY:
+ case ZSTD:
+ break;
+ default:
+ throw new IllegalArgumentException(
+ String.format(
+ IoTConsensusMessages
+ .EXCEPTION_UNSUPPORTED_IOTCONSENSUS_LOG_COMPRESSION_TYPE_ARG_4008DA9A,
+ type));
+ }
this.logCompressionType = type;
return this;
}
diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompression.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompression.java
index 1aa770121d2f9..28003fc8b3f3f 100644
--- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompression.java
+++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompression.java
@@ -25,8 +25,6 @@
import org.apache.iotdb.consensus.iot.thrift.TLogEntry;
import org.apache.iotdb.consensus.iot.thrift.TSyncLogEntriesReq;
-import com.github.luben.zstd.Zstd;
-import com.github.luben.zstd.ZstdException;
import org.apache.tsfile.compress.ICompressor;
import org.apache.tsfile.compress.IUnCompressor;
import org.apache.tsfile.file.metadata.enums.CompressionType;
@@ -46,6 +44,7 @@ public final class LogCompression {
private static final int DESCRIPTOR_BYTES = 32;
private static final ICompressor LZ4 = ICompressor.getCompressor(CompressionType.LZ4);
private static final ICompressor SNAPPY = ICompressor.getCompressor(CompressionType.SNAPPY);
+ private static final ICompressor ZSTD = ICompressor.getCompressor(CompressionType.ZSTD);
private static final IUnCompressor LZ4_DECODER =
IUnCompressor.getUnCompressor(CompressionType.LZ4);
private static final IUnCompressor SNAPPY_DECODER =
@@ -105,8 +104,8 @@ public void close() {
}
public static EncodedRequest compress(
- TSyncLogEntriesReq original, TLogCompressionType codec, int maxDecodedBytes)
- throws IOException {
+ TSyncLogEntriesReq original, CompressionType codec, int maxDecodedBytes) throws IOException {
+ TLogCompressionType wireCodec = toWireCodec(codec);
long buffers = 0;
long rawSize = 0;
boolean hasEligibleBuffer = false;
@@ -121,7 +120,7 @@ public static EncodedRequest compress(
}
}
// Preserve the original path for oversized legal entries and when scratch memory is scarce.
- if (codec == TLogCompressionType.NONE || !hasEligibleBuffer || rawSize > maxDecodedBytes) {
+ if (codec == CompressionType.UNCOMPRESSED || !hasEligibleBuffer || rawSize > maxDecodedBytes) {
return new EncodedRequest(original, 0, rawSize, false);
}
// Charge descriptor/list/entry overhead first, then one block's working arrays at a time.
@@ -157,22 +156,16 @@ public static EncodedRequest compress(
byte[] input = buffer.hasArray() ? buffer.array() : bytes(buffer);
int offset = buffer.hasArray() ? buffer.arrayOffset() + buffer.position() : 0;
int maxCompressedLength =
- codec == TLogCompressionType.ZSTD
- ? (int) Zstd.compressBound(uncompressedLength)
- : compressor(codec).getMaxBytesForCompression(uncompressedLength);
+ compressor(codec).getMaxBytesForCompression(uncompressedLength);
byte[] output = new byte[maxCompressedLength];
int compressedLength =
- codec == TLogCompressionType.ZSTD
- ? (int)
- Zstd.compressByteArray(
- output, 0, output.length, input, offset, uncompressedLength, 1)
- : compressor(codec).compress(input, offset, uncompressedLength, output);
+ compressor(codec).compress(input, offset, uncompressedLength, output);
if (compressedLength < 0) {
throw invalid(Failure.CODEC, uncompressedLength, compressedLength);
}
if ((long) compressedLength + DESCRIPTOR_BYTES < uncompressedLength) {
encoded = ByteBuffer.wrap(Arrays.copyOf(output, compressedLength));
- actualCodec = codec;
+ actualCodec = wireCodec;
arrays += compressedLength;
compressed = true;
}
@@ -269,7 +262,8 @@ public static void decompress(TSyncLogEntriesReq request, int maxDecodedBytes)
try {
actual =
decoder(info.getCodec()).uncompress(input, offset, buffer.remaining(), output, 0);
- } catch (ZstdException e) {
+ } catch (RuntimeException e) {
+ // TsFile codecs may report malformed input through unchecked exceptions.
IOException failure = invalid(Failure.CODEC, output.length, buffer.remaining());
failure.initCause(e);
throw failure;
@@ -307,14 +301,31 @@ private static byte[] bytes(ByteBuffer buffer) {
return bytes;
}
- private static ICompressor compressor(TLogCompressionType codec) throws IOException {
+ private static TLogCompressionType toWireCodec(CompressionType codec) throws IOException {
+ switch (codec) {
+ case UNCOMPRESSED:
+ return TLogCompressionType.NONE;
+ case LZ4:
+ return TLogCompressionType.LZ4;
+ case SNAPPY:
+ return TLogCompressionType.SNAPPY;
+ case ZSTD:
+ return TLogCompressionType.ZSTD;
+ default:
+ throw invalid(Failure.CODEC, 0, codec.serialize());
+ }
+ }
+
+ private static ICompressor compressor(CompressionType codec) throws IOException {
switch (codec) {
case LZ4:
return LZ4;
case SNAPPY:
return SNAPPY;
+ case ZSTD:
+ return ZSTD;
default:
- throw invalid(Failure.CODEC, 0, codec.getValue());
+ throw invalid(Failure.CODEC, 0, codec.serialize());
}
}
diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcher.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcher.java
index 972265a4fc09b..18e73aa2c6170 100644
--- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcher.java
+++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcher.java
@@ -35,12 +35,12 @@
import org.apache.iotdb.consensus.iot.client.DispatchLogHandler;
import org.apache.iotdb.consensus.iot.log.ConsensusReqReader;
import org.apache.iotdb.consensus.iot.log.GetConsensusReqReaderPlan;
-import org.apache.iotdb.consensus.iot.thrift.TLogCompressionType;
import org.apache.iotdb.consensus.iot.thrift.TLogEntry;
import org.apache.iotdb.consensus.iot.thrift.TSyncLogEntriesReq;
import org.apache.iotdb.consensus.iot.thrift.TSyncLogEntriesRes;
import org.apache.thrift.async.AsyncMethodCallback;
+import org.apache.tsfile.file.metadata.enums.CompressionType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -600,7 +600,7 @@ public void sendBatchAsync(Batch batch, DispatchLogHandler handler) {
batch.getStartIndex(),
batch.getEndIndex(),
peer.getGroupId().convertToTConsensusGroupId());
- if (config.getReplication().getLogCompressionType() == TLogCompressionType.NONE) {
+ if (config.getReplication().getLogCompressionType() == CompressionType.UNCOMPRESSED) {
clientManager.borrowClient(peer.getEndpoint()).syncLogEntries(req, handler);
return;
}
diff --git a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/config/IoTConsensusConfigTest.java b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/config/IoTConsensusConfigTest.java
new file mode 100644
index 0000000000000..2c05a9f92bb6f
--- /dev/null
+++ b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/config/IoTConsensusConfigTest.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.consensus.config;
+
+import org.apache.tsfile.file.metadata.enums.CompressionType;
+import org.junit.Test;
+
+import static org.junit.Assert.assertThrows;
+
+public class IoTConsensusConfigTest {
+ @Test
+ public void testRejectsCompressionTypesUnsupportedByWireProtocol() {
+ for (CompressionType type :
+ new CompressionType[] {CompressionType.GZIP, CompressionType.LZMA2}) {
+ assertThrows(
+ IllegalArgumentException.class,
+ () -> IoTConsensusConfig.Replication.newBuilder().setLogCompressionType(type));
+ }
+ }
+}
diff --git a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/CompressedReplicationTest.java b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/CompressedReplicationTest.java
index f01178d24776d..42190e85f3977 100644
--- a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/CompressedReplicationTest.java
+++ b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/CompressedReplicationTest.java
@@ -29,10 +29,10 @@
import org.apache.iotdb.consensus.common.request.IndexedConsensusRequest;
import org.apache.iotdb.consensus.config.ConsensusConfig;
import org.apache.iotdb.consensus.config.IoTConsensusConfig;
-import org.apache.iotdb.consensus.iot.thrift.TLogCompressionType;
import org.apache.iotdb.consensus.iot.util.TestEntry;
import org.apache.iotdb.consensus.iot.util.TestStateMachine;
+import org.apache.tsfile.file.metadata.enums.CompressionType;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
@@ -59,12 +59,10 @@ public class CompressedReplicationTest {
@Parameterized.Parameters(name = "{0}")
public static Object[] codecs() {
- return new Object[] {
- TLogCompressionType.LZ4, TLogCompressionType.SNAPPY, TLogCompressionType.ZSTD
- };
+ return new Object[] {CompressionType.LZ4, CompressionType.SNAPPY, CompressionType.ZSTD};
}
- @Parameterized.Parameter public TLogCompressionType codec;
+ @Parameterized.Parameter public CompressionType codec;
private static final int COUNT = 32;
private static final int BYTES = 65536;
diff --git a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompressionTest.java b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompressionTest.java
index 4d8ce6103a182..29440af69c1db 100644
--- a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompressionTest.java
+++ b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/logdispatcher/LogCompressionTest.java
@@ -40,6 +40,7 @@
import org.apache.thrift.TDeserializer;
import org.apache.thrift.TSerializer;
+import org.apache.tsfile.file.metadata.enums.CompressionType;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -73,13 +74,19 @@
@RunWith(Parameterized.class)
public class LogCompressionTest {
@Parameterized.Parameters(name = "{0}")
- public static Object[] codecs() {
- return new Object[] {
- TLogCompressionType.LZ4, TLogCompressionType.SNAPPY, TLogCompressionType.ZSTD
+ public static Object[][] codecs() {
+ return new Object[][] {
+ {CompressionType.LZ4, TLogCompressionType.LZ4},
+ {CompressionType.SNAPPY, TLogCompressionType.SNAPPY},
+ {CompressionType.ZSTD, TLogCompressionType.ZSTD}
};
}
- @Parameterized.Parameter public TLogCompressionType codec;
+ @Parameterized.Parameter public CompressionType codec;
+
+ @Parameterized.Parameter(1)
+ public TLogCompressionType expectedWireCodec;
+
private IMemoryBlock previous;
private final IoTConsensusMemoryManager memory = IoTConsensusMemoryManager.getInstance();
private static final int LIMIT = 8 * 1024 * 1024;
@@ -149,7 +156,7 @@ public void testRoundTripThriftAndRetryOwnership() throws Exception {
assertTrue(encoded.getEncodedBytes() < encoded.getRawBytes());
assertTrue(memory.getMemoryBlock().getUsedMemoryInBytes() > 0);
assertEquals(
- codec,
+ expectedWireCodec,
encoded.getRequest().getLogEntries().get(0).getCompressionInfo().get(0).getCodec());
assertEquals(
TLogCompressionType.NONE,
@@ -186,7 +193,7 @@ public void testFixedMinimumSizeBoundary() throws Exception {
} else {
assertNotSame(raw, encoded.getRequest());
assertEquals(
- codec,
+ expectedWireCodec,
encoded.getRequest().getLogEntries().get(0).getCompressionInfo().get(0).getCodec());
}
TSyncLogEntriesReq received = encoded.getRequest().deepCopy();
@@ -202,7 +209,7 @@ public void testFixedMinimumSizeBoundary() throws Exception {
public void testAdaptiveAndMemoryFallback() throws Exception {
TSyncLogEntriesReq raw = request();
try (LogCompression.EncodedRequest result =
- LogCompression.compress(raw, TLogCompressionType.NONE, LIMIT)) {
+ LogCompression.compress(raw, CompressionType.UNCOMPRESSED, LIMIT)) {
assertSame(raw, result.getRequest());
}
byte[] random = new byte[16384];
@@ -389,13 +396,13 @@ public void testMemoryPressureFallsBackPerBlockAndKeepsOtherBlocksCompressed() t
assertNotSame(raw, encoded.getRequest());
assertTrue(encoded.isMemoryLimited());
assertEquals(
- codec,
+ expectedWireCodec,
encoded.getRequest().getLogEntries().get(0).getCompressionInfo().get(0).getCodec());
assertEquals(
TLogCompressionType.NONE,
encoded.getRequest().getLogEntries().get(0).getCompressionInfo().get(1).getCodec());
assertEquals(
- codec,
+ expectedWireCodec,
encoded.getRequest().getLogEntries().get(0).getCompressionInfo().get(2).getCodec());
TSyncLogEntriesReq received = encoded.getRequest().deepCopy();
LogCompression.decompress(received, LIMIT);
diff --git a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcherCompressionTest.java b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcherCompressionTest.java
index 81ae68231c2cc..24d9607167f9b 100644
--- a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcherCompressionTest.java
+++ b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcherCompressionTest.java
@@ -31,7 +31,6 @@
import org.apache.iotdb.consensus.iot.IoTConsensusServerImpl;
import org.apache.iotdb.consensus.iot.client.AsyncIoTConsensusServiceClient;
import org.apache.iotdb.consensus.iot.client.DispatchLogHandler;
-import org.apache.iotdb.consensus.iot.thrift.TLogCompressionType;
import org.apache.iotdb.consensus.iot.thrift.TLogEntry;
import org.apache.iotdb.consensus.iot.thrift.TSyncLogEntriesReq;
import org.apache.iotdb.consensus.iot.thrift.TSyncLogEntriesRes;
@@ -39,6 +38,7 @@
import org.apache.thrift.TException;
import org.apache.thrift.async.AsyncMethodCallback;
+import org.apache.tsfile.file.metadata.enums.CompressionType;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
@@ -86,7 +86,7 @@ public void setUp() throws Exception {
IoTConsensusConfig.newBuilder()
.setReplication(
IoTConsensusConfig.Replication.newBuilder()
- .setLogCompressionType(TLogCompressionType.LZ4)
+ .setLogCompressionType(CompressionType.LZ4)
.build())
.build();
IoTConsensusServerImpl impl = mock(IoTConsensusServerImpl.class);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 41dc78446d863..bd2895f887642 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -30,7 +30,6 @@
import org.apache.iotdb.commons.pipe.config.PipeConfig;
import org.apache.iotdb.commons.utils.FileUtils;
import org.apache.iotdb.consensus.ConsensusFactory;
-import org.apache.iotdb.consensus.iot.thrift.TLogCompressionType;
import org.apache.iotdb.db.exception.LoadConfigurationException;
import org.apache.iotdb.db.i18n.DataNodeMiscMessages;
import org.apache.iotdb.db.protocol.thrift.impl.ClientRPCServiceImpl;
@@ -1131,7 +1130,7 @@ public class IoTDBConfig implements DefaultEncodingProvider {
// IoTConsensus Config
private int maxLogEntriesNumPerBatch = 1024;
private int maxSizePerBatch = 16 * 1024 * 1024;
- private TLogCompressionType iotLogCompressionType = TLogCompressionType.NONE;
+ private CompressionType iotLogCompressionType = CompressionType.UNCOMPRESSED;
private int maxPendingBatchesNum = 5;
private double maxMemoryRatioForQueue = 0.6;
private long regionMigrationSpeedLimitBytesPerSecond = 48 * 1024 * 1024L;
@@ -1311,11 +1310,11 @@ public int getMaxLogEntriesNumPerBatch() {
return maxLogEntriesNumPerBatch;
}
- public TLogCompressionType getIotLogCompressionType() {
+ public CompressionType getIotLogCompressionType() {
return iotLogCompressionType;
}
- public void setIotLogCompressionType(TLogCompressionType type) {
+ public void setIotLogCompressionType(CompressionType type) {
iotLogCompressionType = type;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index ce526ff448f59..968988ca48f8f 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -39,7 +39,6 @@
import org.apache.iotdb.confignode.rpc.thrift.TGlobalConfig;
import org.apache.iotdb.confignode.rpc.thrift.TRatisConfig;
import org.apache.iotdb.consensus.config.IoTConsensusV2Config;
-import org.apache.iotdb.consensus.iot.thrift.TLogCompressionType;
import org.apache.iotdb.db.conf.rest.IoTDBRestServiceDescriptor;
import org.apache.iotdb.db.consensus.DataRegionConsensusImpl;
import org.apache.iotdb.db.i18n.DataNodeMiscMessages;
@@ -1197,8 +1196,19 @@ public void loadProperties(TrimProperties properties) throws BadNodeUrlException
// Sending policy is a startup setting. Decoding is always available for in-flight requests.
String compression = properties.getProperty("data_region_iot_log_compression", "NONE").trim();
try {
- TLogCompressionType type = TLogCompressionType.valueOf(compression.toUpperCase(Locale.ROOT));
- conf.setIotLogCompressionType(type);
+ String normalizedCompression = compression.toUpperCase(Locale.ROOT);
+ switch (normalizedCompression) {
+ case "NONE":
+ conf.setIotLogCompressionType(CompressionType.UNCOMPRESSED);
+ break;
+ case "LZ4":
+ case "SNAPPY":
+ case "ZSTD":
+ conf.setIotLogCompressionType(CompressionType.valueOf(normalizedCompression));
+ break;
+ default:
+ throw new IllegalArgumentException();
+ }
} catch (IllegalArgumentException e) {
throw new IOException(
String.format(
diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
index 9e1bbe1fddab6..7ca2aa736317c 100644
--- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
+++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
@@ -1873,7 +1873,8 @@ enable_wal_compression=true
### IoTConsensus Configuration
####################
-# IoTConsensus V1 log payload codec: NONE, LZ4, SNAPPY, ZSTD (level 1).
+# IoTConsensus V1 log payload codec: NONE, LZ4, SNAPPY, ZSTD.
+# Compression uses the TsFile implementation and its default codec settings.
# Enable only after every peer has the same supporting version and configuration.
# This controls sending; receivers can decode every supported codec even when set to NONE.
# Independent of WAL/TsFile compression and dn_rpc_thrift_compression_enable.