From 12161147ceb9d78721f83a789df6712885b2fa58 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 22 Oct 2026 16:29:42 +0800 Subject: [PATCH 1/4] [Pipe] Unify TsFile parser allocation and resize quotas (#18984) --- .../iotdb/db/i18n/DataNodePipeMessages.java | 4 + .../iotdb/db/i18n/DataNodePipeMessages.java | 4 + ...FileInsertionEventParserMemoryManager.java | 20 +-- .../memory/PipeMemoryBlockCategory.java | 52 ++++++ .../resource/memory/PipeMemoryBlockType.java | 1 + .../resource/memory/PipeMemoryManager.java | 61 +++++++ .../memory/PipeTsFileParserMemoryBlock.java | 27 +++ .../memory/PipeMemoryManagerResizeTest.java | 169 ++++++++++++++++++ 8 files changed, 327 insertions(+), 11 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryBlockCategory.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeTsFileParserMemoryBlock.java diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java index f3e3f5ec09d9a..56f39342242c0 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java @@ -2378,6 +2378,10 @@ private DataNodePipeMessages() {} public static final String PIPE_EXCEPTION_FORCERESIZE_FAILED_TO_ALLOCATE_MEMORY_AFTER_D_RETRIES_TOTAL_8C6948BC = "forceResize: failed to allocate memory after %d retries, total memory size %d bytes, used " + "memory size %d bytes, requested memory size %d bytes"; + public static final String EXCEPTION_FORCERESIZE_FAILED_AFTER_ARG_RETRIES_TOTAL_ARG_BYTES_USED_ARG_BYTES_USEDTABLETS_ARG_BYTES_USEDTSFILES_ARG_BYTES_TABLETLIMIT_ARG_BYTES_TSFILELIMIT_ARG_BYTES_TABLETANDTSFILELIMIT_ARG_BYTES_PROTECTEDPARSERLIMIT_ARG_BYTES_REQUESTED_ARG_BYTES_311C8C7E = + "forceResize failed after %d retries: total=%d bytes, used=%d bytes, " + + "usedTablets=%d bytes, usedTsFiles=%d bytes, tabletLimit=%d bytes, tsFileLimit=%d bytes, " + + "tabletAndTsFileLimit=%d bytes, protectedParserLimit=%d bytes, requested=%d bytes"; public static final String EXCEPTION_UNSUPPORTED_BATCH_TYPE_ARG_WHEN_TRANSFERRING_TABLET_INSERTION_EVENT_66153E12 = "Unsupported batch type %s when transferring tablet insertion event."; diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java index 2b963bb1c0e0b..f2759b8725879 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java @@ -2210,6 +2210,10 @@ private DataNodePipeMessages() {} "forceAllocate:重试 %d 次后仍无法分配内存,总内存大小 %d bytes,已用内存大小 %d bytes,请求内存大小 %d bytes"; public static final String PIPE_EXCEPTION_FORCERESIZE_FAILED_TO_ALLOCATE_MEMORY_AFTER_D_RETRIES_TOTAL_8C6948BC = "forceResize:重试 %d 次后仍无法分配内存,总内存大小 %d bytes,已用内存大小 %d bytes,请求内存大小 %d bytes"; + public static final String EXCEPTION_FORCERESIZE_FAILED_AFTER_ARG_RETRIES_TOTAL_ARG_BYTES_USED_ARG_BYTES_USEDTABLETS_ARG_BYTES_USEDTSFILES_ARG_BYTES_TABLETLIMIT_ARG_BYTES_TSFILELIMIT_ARG_BYTES_TABLETANDTSFILELIMIT_ARG_BYTES_PROTECTEDPARSERLIMIT_ARG_BYTES_REQUESTED_ARG_BYTES_311C8C7E = + "forceResize 重试 %d 次后仍无法分配内存:total=%d bytes,used=%d bytes," + + "usedTablets=%d bytes,usedTsFiles=%d bytes,tabletLimit=%d bytes,tsFileLimit=%d bytes," + + "tabletAndTsFileLimit=%d bytes,protectedParserLimit=%d bytes,requested=%d bytes"; public static final String EXCEPTION_UNSUPPORTED_BATCH_TYPE_ARG_WHEN_TRANSFERRING_TABLET_INSERTION_EVENT_66153E12 = "传输 tablet insertion event 时不支持 batch 类型 %s。"; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/TsFileInsertionEventParserMemoryManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/TsFileInsertionEventParserMemoryManager.java index 11a31ff77aef0..7bb9a153477b8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/TsFileInsertionEventParserMemoryManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/TsFileInsertionEventParserMemoryManager.java @@ -20,9 +20,9 @@ package org.apache.iotdb.db.pipe.event.common.tsfile.parser; import org.apache.iotdb.commons.exception.pipe.PipeRuntimeOutOfMemoryCriticalException; -import org.apache.iotdb.db.i18n.DataNodePipeMessages; import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager; import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryBlock; +import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryManager; /** Allocates parser working memory from the pool owned by the caller. */ public interface TsFileInsertionEventParserMemoryManager { @@ -42,12 +42,15 @@ final class PipeHolder { public TsFileInsertionEventParserMemoryBlock forceAllocateForTabletWithRetry( final long sizeInBytes) { return new PipeBlock( - PipeDataNodeResourceManager.memory().forceAllocateForTabletWithRetry(sizeInBytes)); + PipeDataNodeResourceManager.memory() + .forceAllocateForTsFileParserWithRetry(sizeInBytes)); } @Override public TsFileInsertionEventParserMemoryBlock forceAllocate(final long sizeInBytes) { - return new PipeBlock(PipeDataNodeResourceManager.memory().forceAllocate(sizeInBytes)); + return new PipeBlock( + PipeDataNodeResourceManager.memory() + .forceAllocateForTsFileParserWithRetry(sizeInBytes)); } }; } @@ -67,15 +70,10 @@ public long getMemoryUsageInBytes() { @Override public void forceResize(final long newSizeInBytes) { final long oldSize = delegate.getMemoryUsageInBytes(); - if (!PipeDataNodeResourceManager.memory().tryResize(delegate, newSizeInBytes)) { + final PipeMemoryManager memoryManager = PipeDataNodeResourceManager.memory(); + if (!memoryManager.tryResize(delegate, newSizeInBytes)) { throw new PipeRuntimeOutOfMemoryCriticalException( - String.format( - DataNodePipeMessages - .PIPE_EXCEPTION_FORCERESIZE_FAILED_TO_ALLOCATE_MEMORY_AFTER_D_RETRIES_TOTAL_8C6948BC, - 0, - PipeDataNodeResourceManager.memory().getTotalNonFloatingMemorySizeInBytes(), - PipeDataNodeResourceManager.memory().getUsedMemorySizeInBytes(), - newSizeInBytes - oldSize)); + memoryManager.getResizeFailureMessage(0, newSizeInBytes - oldSize)); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryBlockCategory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryBlockCategory.java new file mode 100644 index 0000000000000..fb1e8df2c591d --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryBlockCategory.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.resource.memory; + +/** The diagnostic category of a Pipe memory block. */ +public enum PipeMemoryBlockCategory { + GLOBAL, + EVENT, + EVENT_CHILD, + PARSER, + TABLET, + TS_FILE, + BATCH, + WAL, + CACHE, + RECEIVER, + SINK, + SUBSCRIPTION, + FLOATING, + OTHER; + + public static PipeMemoryBlockCategory fromType(final PipeMemoryBlockType type) { + if (type == null) { + return OTHER; + } + return switch (type) { + case NORMAL -> OTHER; + case TABLET -> TABLET; + case TS_FILE_PARSER -> PARSER; + case TS_FILE -> TS_FILE; + case BATCH -> BATCH; + case WAL -> WAL; + }; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryBlockType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryBlockType.java index 846fc7dd1ce17..78fd9aff6704a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryBlockType.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryBlockType.java @@ -22,6 +22,7 @@ public enum PipeMemoryBlockType { NORMAL, TABLET, + TS_FILE_PARSER, TS_FILE, BATCH, WAL diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java index 6d89ac0d9d221..32c3eb9ed1158 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java @@ -138,6 +138,34 @@ private double allowedMaxMemorySizeInBytesOfTsTiles( * totalNonFloatingMemorySizeInBytes; } + private boolean isEnoughForTsFileParserMemory( + final long extraMemoryInBytes, final long totalNonFloatingMemorySizeInBytes) { + return (double) usedMemorySizeInBytesOfTablets + + (double) usedMemorySizeInBytesOfTsFiles + + (double) extraMemoryInBytes + < EXCEED_PROTECT_THRESHOLD + * allowedMaxMemorySizeInBytesOfTabletsAndTsFiles(totalNonFloatingMemorySizeInBytes); + } + + public synchronized String getResizeFailureMessage(final int retries, final long requestedSize) { + final long totalNonFloatingMemorySizeInBytes = getTotalNonFloatingMemorySizeInBytes(); + final double combinedLimit = + allowedMaxMemorySizeInBytesOfTabletsAndTsFiles(totalNonFloatingMemorySizeInBytes); + return String.format( + DataNodePipeMessages + .EXCEPTION_FORCERESIZE_FAILED_AFTER_ARG_RETRIES_TOTAL_ARG_BYTES_USED_ARG_BYTES_USEDTABLETS_ARG_BYTES_USEDTSFILES_ARG_BYTES_TABLETLIMIT_ARG_BYTES_TSFILELIMIT_ARG_BYTES_TABLETANDTSFILELIMIT_ARG_BYTES_PROTECTEDPARSERLIMIT_ARG_BYTES_REQUESTED_ARG_BYTES_311C8C7E, + retries, + totalNonFloatingMemorySizeInBytes, + memoryBlock.getUsedMemoryInBytes(), + usedMemorySizeInBytesOfTablets, + usedMemorySizeInBytesOfTsFiles, + Math.round(allowedMaxMemorySizeInBytesOfTablets(totalNonFloatingMemorySizeInBytes)), + Math.round(allowedMaxMemorySizeInBytesOfTsTiles(totalNonFloatingMemorySizeInBytes)), + Math.round(combinedLimit), + Math.round(EXCEED_PROTECT_THRESHOLD * combinedLimit), + requestedSize); + } + private static long getTsFileParserMemorySizeInBytes() { return Math.max( PIPE_CONFIG.getTsFileParserMemory(), PIPE_CONFIG.getPipeMemoryAllocateMinSizeInBytes()); @@ -542,6 +570,10 @@ < allowedMaxMemorySizeInBytesOfTabletsAndTsFiles(totalNonFloatingMemorySizeInByt private boolean isHardEnoughForResizing( final PipeMemoryBlock block, final long extraMemoryInBytes) { + if (block instanceof PipeTsFileParserMemoryBlock) { + return isEnoughForTsFileParserMemory( + extraMemoryInBytes, getTotalNonFloatingMemorySizeInBytes()); + } if (!(block instanceof PipeTabletMemoryBlock) && !(block instanceof PipeTsFileMemoryBlock)) { return true; } @@ -625,6 +657,27 @@ public PipeTabletMemoryBlock forceAllocateForTabletWithRetry(long tabletSizeInBy } } + public synchronized PipeTsFileParserMemoryBlock forceAllocateForTsFileParserWithRetry( + final long parserMemorySizeInBytes) { + if (!PIPE_MEMORY_MANAGEMENT_ENABLED) { + return new PipeTsFileParserMemoryBlock(0); + } + + final PipeTsFileParserMemoryBlock block = + (PipeTsFileParserMemoryBlock) registerMemoryBlock(0, PipeMemoryBlockType.TS_FILE_PARSER); + if (parserMemorySizeInBytes == 0) { + return block; + } + + try { + forceResize(block, parserMemorySizeInBytes); + return block; + } catch (final RuntimeException e) { + release(block); + throw e; + } + } + public PipeTsFileMemoryBlock forceAllocateForTsFileWithRetry(long tsFileSizeInBytes) throws PipeRuntimeOutOfMemoryCriticalException { if (!PIPE_MEMORY_MANAGEMENT_ENABLED) { @@ -712,6 +765,8 @@ private PipeMemoryBlock forceAllocateWithRetry(long sizeInBytes, PipeMemoryBlock switch (type) { case TABLET: return new PipeTabletMemoryBlock(sizeInBytes); + case TS_FILE_PARSER: + return new PipeTsFileParserMemoryBlock(sizeInBytes); case TS_FILE: return new PipeTsFileMemoryBlock(sizeInBytes); case BATCH: @@ -811,6 +866,9 @@ public synchronized void resize( } private boolean tryResizeInternal(final PipeMemoryBlock block, final long targetSize) { + if (block.isReleased()) { + return false; + } if (!PIPE_MEMORY_MANAGEMENT_ENABLED) { block.setMemoryUsageInBytes(targetSize); return true; @@ -983,6 +1041,9 @@ private PipeMemoryBlock registerMemoryBlock(long sizeInBytes, PipeMemoryBlockTyp case TABLET: returnedMemoryBlock = new PipeTabletMemoryBlock(sizeInBytes); break; + case TS_FILE_PARSER: + returnedMemoryBlock = new PipeTsFileParserMemoryBlock(sizeInBytes); + break; case TS_FILE: returnedMemoryBlock = new PipeTsFileMemoryBlock(sizeInBytes); break; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeTsFileParserMemoryBlock.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeTsFileParserMemoryBlock.java new file mode 100644 index 0000000000000..5258d5216b5cd --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeTsFileParserMemoryBlock.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.db.pipe.resource.memory; + +/** Parser working memory charged to TsFiles and admitted against the protected combined quota. */ +public class PipeTsFileParserMemoryBlock extends PipeTsFileMemoryBlock { + + public PipeTsFileParserMemoryBlock(final long memoryUsageInBytes) { + super(memoryUsageInBytes); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManagerResizeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManagerResizeTest.java index ddaf043b969d9..298647b8eb347 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManagerResizeTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManagerResizeTest.java @@ -30,6 +30,13 @@ import org.junit.Before; import org.junit.Test; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; public class PipeMemoryManagerResizeTest { @@ -226,4 +233,166 @@ public void testFloatingAndNonFloatingMemoryShareTheSamePool() { manager.release(nonFloatingMemory); } } + + @Test + public void testTsFileParserAllocationAndResizeShareProtectedCombinedLimit() { + final AtomicLong floatingMemoryUsageInBytes = new AtomicLong(0); + final PipeMemoryManager manager = + new PipeMemoryManager( + new AtomicLongMemoryBlock( + "PipeMemoryManagerResizeTest", + null, + TOTAL_MEMORY_SIZE_IN_BYTES, + MemoryBlockType.DYNAMIC), + floatingMemoryUsageInBytes::get); + final PipeTsFileParserMemoryBlock parser = manager.forceAllocateForTsFileParserWithRetry(1000); + + try { + Assert.assertTrue(manager.tryResize(parser, 1100)); + Assert.assertFalse(manager.tryResize(parser, 1140)); + Assert.assertEquals(1100, parser.getMemoryUsageInBytes()); + Assert.assertEquals(0, manager.getUsedMemorySizeInBytesOfTablets()); + Assert.assertEquals(1100, manager.getUsedMemorySizeInBytesOfTsFiles()); + Assert.assertEquals(1100, manager.getUsedMemorySizeInBytes()); + + final PipeRuntimeOutOfMemoryCriticalException allocationException = + Assert.assertThrows( + PipeRuntimeOutOfMemoryCriticalException.class, + () -> manager.forceAllocateForTsFileParserWithRetry(40)); + Assert.assertTrue(allocationException.getMessage().contains("usedTsFiles=1100 bytes")); + Assert.assertTrue(allocationException.getMessage().contains("tabletLimit=900 bytes")); + Assert.assertTrue( + allocationException.getMessage().contains("tabletAndTsFileLimit=1200 bytes")); + Assert.assertTrue( + allocationException.getMessage().contains("protectedParserLimit=1140 bytes")); + + // Parser working memory must not consume the tablet-only quota: the generated tablet + // still needs to be retained and sent while the parser owns its decoded pages/chunks. + final PipeTabletMemoryBlock output = manager.forceAllocateForTabletWithRetry(50); + try { + Assert.assertTrue(manager.tryResize(output, 90)); + Assert.assertEquals(90, manager.getUsedMemorySizeInBytesOfTablets()); + Assert.assertEquals(1100, manager.getUsedMemorySizeInBytesOfTsFiles()); + } finally { + manager.release(output); + } + } finally { + manager.release(parser); + } + + Assert.assertEquals(0, manager.getUsedMemorySizeInBytes()); + } + + @Test + public void testZeroSizedTsFileParserCanGrowBeyondTabletLimitAndRelease() { + final PipeMemoryManager manager = + new PipeMemoryManager( + new AtomicLongMemoryBlock( + "PipeMemoryManagerResizeTest", + null, + TOTAL_MEMORY_SIZE_IN_BYTES, + MemoryBlockType.DYNAMIC), + () -> 0); + final PipeTsFileParserMemoryBlock parser = manager.forceAllocateForTsFileParserWithRetry(0); + try { + Assert.assertTrue(manager.tryResize(parser, 1000)); + Assert.assertEquals(1000, manager.getUsedMemorySizeInBytesOfTsFiles()); + Assert.assertEquals(1000, manager.getUsedMemorySizeInBytes()); + + Assert.assertTrue(manager.tryResize(parser, 100)); + Assert.assertEquals(100, manager.getUsedMemorySizeInBytesOfTsFiles()); + manager.release(parser); + Assert.assertTrue(parser.isReleased()); + Assert.assertEquals(0, manager.getUsedMemorySizeInBytesOfTsFiles()); + Assert.assertEquals(0, manager.getUsedMemorySizeInBytesOfTablets()); + Assert.assertEquals(0, manager.getUsedMemorySizeInBytes()); + } finally { + manager.release(parser); + } + } + + @Test + public void testTsFileParserSharesQuotaWithTabletsAndTsFilesAndHonorsGlobalLimit() { + final PipeMemoryManager manager = + new PipeMemoryManager( + new AtomicLongMemoryBlock( + "PipeMemoryManagerResizeTest", + null, + TOTAL_MEMORY_SIZE_IN_BYTES, + MemoryBlockType.DYNAMIC), + () -> 0); + final PipeTabletMemoryBlock tablet = manager.forceAllocateForTabletWithRetry(200); + final PipeTsFileMemoryBlock tsFile = manager.forceAllocateForTsFileWithRetry(500); + final PipeTsFileParserMemoryBlock parser = manager.forceAllocateForTsFileParserWithRetry(400); + final PipeMemoryBlock sink = manager.forceAllocate(900); + try { + Assert.assertFalse(manager.tryResize(parser, 440)); + Assert.assertFalse(manager.tryResize(parser, 401)); + Assert.assertEquals(200, manager.getUsedMemorySizeInBytesOfTablets()); + Assert.assertEquals(900, manager.getUsedMemorySizeInBytesOfTsFiles()); + Assert.assertEquals(2000, manager.getUsedMemorySizeInBytes()); + + manager.release(sink); + Assert.assertTrue(manager.tryResize(parser, 439)); + Assert.assertFalse(manager.tryResize(parser, 440)); + Assert.assertEquals(200, manager.getUsedMemorySizeInBytesOfTablets()); + Assert.assertEquals(939, manager.getUsedMemorySizeInBytesOfTsFiles()); + Assert.assertEquals(1139, manager.getUsedMemorySizeInBytes()); + } finally { + manager.release(tablet); + manager.release(tsFile); + manager.release(parser); + manager.release(sink); + } + Assert.assertEquals(0, manager.getUsedMemorySizeInBytesOfTablets()); + Assert.assertEquals(0, manager.getUsedMemorySizeInBytesOfTsFiles()); + Assert.assertEquals(0, manager.getUsedMemorySizeInBytes()); + } + + @Test + public void testConcurrentTsFileParserAllocationsCannotOvershootQuota() throws Exception { + final PipeMemoryManager manager = + new PipeMemoryManager( + new AtomicLongMemoryBlock( + "PipeMemoryManagerResizeTest", + null, + TOTAL_MEMORY_SIZE_IN_BYTES, + MemoryBlockType.DYNAMIC), + () -> 0); + final ExecutorService executor = Executors.newFixedThreadPool(4); + final CountDownLatch start = new CountDownLatch(1); + final List> futures = new ArrayList<>(); + final List allocated = new ArrayList<>(); + try { + for (int i = 0; i < 20; i++) { + futures.add( + executor.submit( + () -> { + start.await(); + try { + return manager.forceAllocateForTsFileParserWithRetry(300); + } catch (final PipeRuntimeOutOfMemoryCriticalException e) { + return null; + } + })); + } + start.countDown(); + for (final Future future : futures) { + final PipeTsFileParserMemoryBlock block = future.get(10, TimeUnit.SECONDS); + if (block != null) { + allocated.add(block); + } + } + Assert.assertEquals(3, allocated.size()); + Assert.assertEquals(900, manager.getUsedMemorySizeInBytes()); + Assert.assertEquals(0, manager.getUsedMemorySizeInBytesOfTablets()); + Assert.assertEquals(900, manager.getUsedMemorySizeInBytesOfTsFiles()); + } finally { + executor.shutdownNow(); + allocated.forEach(manager::release); + } + Assert.assertEquals(0, manager.getUsedMemorySizeInBytes()); + Assert.assertEquals(0, manager.getUsedMemorySizeInBytesOfTsFiles()); + Assert.assertEquals(0, manager.getUsedMemorySizeInBytesOfTablets()); + } } From 093be83c6369224ce7bc46d0afbab858978eef12 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+caideyipi@users.noreply.github.com> Date: Mon, 26 Oct 2026 19:30:49 +0800 Subject: [PATCH 2/4] Fix subscription provider recovery after broker loss (#19018) * Fix subscription provider recovery after broker loss * Preserve subscription state during provider recovery Close stale sessions without dropping subscriptions, keep provider failures terminal, ignore retired heartbeat results, and stop recovery after consumer close. Recognize file and tablet broker-missing poll diagnostics in both server locales and cover the recovery and commit paths with regression tests. (cherry picked from commit 495f04f7c179089aa261d19143c8fdd5b0601e94) --- .../base/AbstractSubscriptionConsumer.java | 8 ++ .../base/AbstractSubscriptionProvider.java | 55 +++++++- .../base/AbstractSubscriptionProviders.java | 111 +++++++++------ ...criptionProviderConnectionTimeoutTest.java | 130 ++++++++++++++++++ 4 files changed, 254 insertions(+), 50 deletions(-) diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumer.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumer.java index 061fd948de9b6..122b4138ee77f 100644 --- a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumer.java +++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumer.java @@ -1448,6 +1448,7 @@ private List pollFileInternal( if (isClosed()) { return Collections.emptyList(); } + providers.syncIfUnavailable(this); throw new SubscriptionConnectionException( String.format( SubscriptionMessages @@ -1459,6 +1460,7 @@ private List pollFileInternal( try { return provider.pollFile(commitContext, writingOffset, timeoutMs); } catch (final SubscriptionConnectionException ignored) { + providers.syncIfUnavailable(this); return Collections.emptyList(); } } finally { @@ -1477,6 +1479,7 @@ private List pollTabletsInternal( if (isClosed()) { return Collections.emptyList(); } + providers.syncIfUnavailable(this); throw new SubscriptionConnectionException( String.format( SubscriptionMessages @@ -1488,6 +1491,7 @@ private List pollTabletsInternal( try { return provider.pollTablets(commitContext, offset, timeoutMs); } catch (final SubscriptionConnectionException ignored) { + providers.syncIfUnavailable(this); return Collections.emptyList(); } } finally { @@ -1679,6 +1683,7 @@ private AbstractSubscriptionProvider.CommitResult commitInternal( if (isClosed()) { return AbstractSubscriptionProvider.CommitResult.empty(); } + providers.syncIfUnavailable(this); throw new SubscriptionConnectionException( String.format( SubscriptionMessages @@ -1692,6 +1697,9 @@ private AbstractSubscriptionProvider.CommitResult commitInternal( } catch (final SubscriptionConsumerFencedException e) { fence(e); throw e; + } catch (final SubscriptionConnectionException e) { + providers.syncIfUnavailable(this); + throw e; } } finally { providers.releaseReadLock(); diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProvider.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProvider.java index 0beabf8d6145f..56f7324910318 100644 --- a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProvider.java +++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProvider.java @@ -78,6 +78,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.ReentrantLock; public abstract class AbstractSubscriptionProvider { @@ -107,7 +108,15 @@ public abstract class AbstractSubscriptionProvider { private final AtomicBoolean isClosed = new AtomicBoolean(true); private final AtomicBoolean isClosing = new AtomicBoolean(false); - private final AtomicBoolean isAvailable = new AtomicBoolean(false); + + private enum Availability { + NEW, + AVAILABLE, + UNAVAILABLE + } + + private final AtomicReference availability = + new AtomicReference<>(Availability.NEW); private final ReentrantLock rpcLock = new ReentrantLock(); private final int localThriftMaxFrameSize; @@ -243,15 +252,17 @@ private void updateRequestBodySizeLimit() { } boolean isAvailable() { - return isAvailable.get(); + return availability.get() == Availability.AVAILABLE; } void setAvailable() { - isAvailable.set(true); + // A heartbeat that completes after a failed RPC must not revive this connection. Recovery + // constructs and handshakes a new provider instead. + availability.compareAndSet(Availability.NEW, Availability.AVAILABLE); } void setUnavailable() { - isAvailable.set(false); + availability.set(Availability.UNAVAILABLE); } int getDataNodeId() { @@ -413,10 +424,17 @@ void close() throws SubscriptionException, IoTDBConnectionException { } synchronized void closeSession() throws IoTDBConnectionException { + setUnavailable(); try { + try { + // Recovery must preserve subscriptions and interrupt any heartbeat still using this + // session. + getSessionConnection().forceClose(); + } catch (final IoTDBConnectionException ignored) { + // A failed handshake may not have opened a session. + } session.close(); } finally { - setUnavailable(); isClosed.set(true); } } @@ -736,11 +754,34 @@ List poll(final SubscriptionPollRequest pollMessage) setUnavailable(); throw new SubscriptionConnectionException(e.getMessage(), e); } - verifyPipeSubscribeSuccess(resp.status); + try { + verifyPipeSubscribeSuccess(resp.status); + } catch (final SubscriptionRuntimeNonCriticalException e) { + // A missing broker is reported as a poll error instead of a transport failure. Treat it as a + // provider failure so the consumer can recreate this provider before the next poll. + if (isMissingBrokerPollError(resp.status)) { + setUnavailable(); + throw new SubscriptionConnectionException(e.getMessage(), e); + } + throw e; + } final PipeSubscribePollResp pollResp = PipeSubscribePollResp.fromTPipeSubscribeResp(resp); return pollResp.getResponses(); } + private boolean isMissingBrokerPollError(final TSStatus status) { + if (status.code != TSStatusCode.SUBSCRIPTION_POLL_ERROR.getStatusCode() + || status.message == null) { + return false; + } + // These are server protocol diagnostics. Match both server locales independently of the client + // locale, including the distinct messages returned by file and tablet continuation polls. + return status.message.contains("no broker bound to consumer group [" + consumerGroupId + "]") + || status.message.contains( + "broker bound to consumer group [" + consumerGroupId + "] does not exist") + || status.message.contains("绑定到消费者组 [" + consumerGroupId + "] 的 Broker 不存在"); + } + CommitResult commit( final List subscriptionCommitContexts, final boolean nack) throws SubscriptionException { @@ -948,7 +989,7 @@ public String toString() { + ", consumerGroupId=" + consumerGroupId + ", isAvailable=" - + isAvailable + + isAvailable() + ", isClosed=" + isClosed + "}"; diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProviders.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProviders.java index 565c719149d0a..afb5285b5f577 100644 --- a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProviders.java +++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProviders.java @@ -90,11 +90,14 @@ void releaseWriteLock() { /** Caller should ensure that the method is called in the lock {@link #acquireWriteLock()}. */ void openProviders(final AbstractSubscriptionConsumer consumer) throws SubscriptionException { // close stale providers - closeProviders(); + closeProviders(false); final Map connectionFailures = new LinkedHashMap<>(); final List connectionFailureCauses = new ArrayList<>(); for (final TEndPoint endPoint : initialEndpoints) { + if (isClosing.get() || consumer.isFenced()) { + return; + } final AbstractSubscriptionProvider defaultProvider; final int defaultDataNodeId; @@ -116,6 +119,9 @@ void openProviders(final AbstractSubscriptionConsumer consumer) throws Subscript } defaultDataNodeId = defaultProvider.getDataNodeId(); addProvider(defaultDataNodeId, defaultProvider); + if (isClosing.get() || consumer.isFenced()) { + return; + } final Map allEndPoints; try { @@ -134,6 +140,9 @@ void openProviders(final AbstractSubscriptionConsumer consumer) throws Subscript } for (final Map.Entry entry : allEndPoints.entrySet()) { + if (isClosing.get() || consumer.isFenced()) { + return; + } if (defaultDataNodeId == entry.getKey()) { continue; } @@ -243,7 +252,8 @@ void closeAndRemoveProvider(final int dataNodeId) } final AbstractSubscriptionProvider provider = subscriptionProviders.get(dataNodeId); try { - provider.close(); + // Replacing a connection must not unsubscribe topics or drop the shared consumer. + provider.closeSession(); } finally { LOGGER.info(SubscriptionMessages.CLOSE_STALE_PROVIDER, provider); subscriptionProviders.remove(dataNodeId); @@ -338,13 +348,19 @@ void heartbeat(final AbstractSubscriptionConsumer consumer) { private void heartbeatProvider( final AbstractSubscriptionConsumer consumer, final AbstractSubscriptionProvider provider) { try { - if (consumer.isClosed() || consumer.isFenced() || !containsProvider(provider)) { + if (consumer.isClosed() + || consumer.isFenced() + || !containsProvider(provider) + || !provider.isAvailable()) { return; } final List processorBufferedCommitContexts = consumer.getProcessorBufferedCommitContexts(provider.getDataNodeId()); final PipeSubscribeHeartbeatResp resp = provider.heartbeat(processorBufferedCommitContexts); - if (consumer.isClosed() || consumer.isFenced() || !containsProvider(provider)) { + if (consumer.isClosed() + || consumer.isFenced() + || !containsProvider(provider) + || !provider.isAvailable()) { return; } provider.setAvailable(); @@ -360,7 +376,9 @@ private void heartbeatProvider( } } } catch (final SubscriptionConsumerFencedException e) { - consumer.fence(e); + if (containsProvider(provider)) { + consumer.fence(e); + } provider.setUnavailable(); } catch (final Exception e) { LOGGER.warn( @@ -398,7 +416,7 @@ void sync(final AbstractSubscriptionConsumer consumer) { } } - /** Reconnects providers immediately after a failed RPC when every provider is unavailable. */ + /** Reconnects unavailable providers immediately after a failed RPC. */ void syncIfUnavailable(final AbstractSubscriptionConsumer consumer) { if (consumer.isClosed() || consumer.isFenced() @@ -415,7 +433,7 @@ void syncIfUnavailable(final AbstractSubscriptionConsumer consumer) { } acquireWriteLock(); try { - if (consumer.isClosed() || consumer.isFenced() || !hasNoAvailableProviders()) { + if (consumer.isClosed() || consumer.isFenced()) { return; } final long now = System.currentTimeMillis(); @@ -436,7 +454,7 @@ void syncIfUnavailable(final AbstractSubscriptionConsumer consumer) { } private void syncInternal(final AbstractSubscriptionConsumer consumer) { - if (consumer.isFenced()) { + if (consumer.isClosed() || consumer.isFenced()) { return; } if (hasNoAvailableProviders()) { @@ -462,64 +480,71 @@ private void syncInternal(final AbstractSubscriptionConsumer consumer) { return; } - // add new providers or handshake existing providers + // Add new providers, replace unavailable providers, or heartbeat existing providers. for (final Map.Entry entry : allEndPoints.entrySet()) { - if (consumer.isFenced()) { + if (consumer.isClosed() || consumer.isFenced()) { return; } - final AbstractSubscriptionProvider provider = getProvider(entry.getKey()); - if (Objects.isNull(provider)) { - // new provider - final TEndPoint endPoint = entry.getValue(); - final AbstractSubscriptionProvider newProvider; + AbstractSubscriptionProvider provider = getProvider(entry.getKey()); + if (Objects.nonNull(provider) && provider.isAvailable()) { + // Existing provider try { - newProvider = consumer.constructProviderAndHandshake(endPoint); + consumer.subscribedTopics = provider.heartbeat().getTopics(); + provider.setAvailable(); } catch (final SubscriptionConsumerFencedException e) { consumer.fence(e); + provider.setUnavailable(); return; } catch (final Exception e) { LOGGER.warn( - SubscriptionMessages.LOG_ARG_FAILED_CREATE_CONNECTION_ARG_BECAUSE_ARG_E536E22A, + SubscriptionMessages + .LOG_ARG_FAILED_SENDING_HEARTBEAT_SUBSCRIPTION_PROVIDER_ARG_BECAUSE_ARG_SET_0B38FB1F, consumer, - endPoint, + provider, e, e); - continue; + provider.setUnavailable(); } - addProvider(entry.getKey(), newProvider); - } else { - // existing provider + } + + if (Objects.nonNull(provider) && !provider.isAvailable()) { + // Remove unavailable providers before creating replacements in this same sync round. try { - consumer.subscribedTopics = provider.heartbeat().getTopics(); - provider.setAvailable(); - } catch (final SubscriptionConsumerFencedException e) { - consumer.fence(e); - provider.setUnavailable(); - return; + closeAndRemoveProvider(entry.getKey()); } catch (final Exception e) { LOGGER.warn( SubscriptionMessages - .LOG_ARG_FAILED_SENDING_HEARTBEAT_SUBSCRIPTION_PROVIDER_ARG_BECAUSE_ARG_SET_0B38FB1F, + .LOG_EXCEPTION_OCCURRED_ARG_CLOSING_REMOVING_SUBSCRIPTION_PROVIDER_ARG_BECAUSE_ARG_2EC38739, consumer, provider, e, e); - provider.setUnavailable(); } - // close and remove unavailable provider (reset the connection as much as possible) - if (!provider.isAvailable()) { - try { - closeAndRemoveProvider(entry.getKey()); - } catch (final Exception e) { - LOGGER.warn( - SubscriptionMessages - .LOG_EXCEPTION_OCCURRED_ARG_CLOSING_REMOVING_SUBSCRIPTION_PROVIDER_ARG_BECAUSE_ARG_2EC38739, - consumer, - provider, - e, - e); - } + provider = null; + } + + if (Objects.isNull(provider)) { + // New or replacement provider + if (consumer.isClosed() || consumer.isFenced()) { + return; } + final TEndPoint endPoint = entry.getValue(); + final AbstractSubscriptionProvider newProvider; + try { + newProvider = consumer.constructProviderAndHandshake(endPoint); + } catch (final SubscriptionConsumerFencedException e) { + consumer.fence(e); + return; + } catch (final Exception e) { + LOGGER.warn( + SubscriptionMessages.LOG_ARG_FAILED_CREATE_CONNECTION_ARG_BECAUSE_ARG_E536E22A, + consumer, + endPoint, + e, + e); + continue; + } + addProvider(entry.getKey(), newProvider); } } diff --git a/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionProviderConnectionTimeoutTest.java b/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionProviderConnectionTimeoutTest.java index ab7a741b07dc7..46dbd1f8e9f7c 100644 --- a/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionProviderConnectionTimeoutTest.java +++ b/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionProviderConnectionTimeoutTest.java @@ -22,10 +22,13 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.iotdb.rpc.subscription.exception.SubscriptionConnectionException; +import org.apache.iotdb.rpc.subscription.exception.SubscriptionRuntimeNonCriticalException; import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionCommitContext; +import org.apache.iotdb.rpc.subscription.payload.request.PipeSubscribeRequestType; import org.apache.iotdb.rpc.subscription.payload.response.PipeSubscribeHandshakeResp; import org.apache.iotdb.service.rpc.thrift.IClientRPCService; import org.apache.iotdb.service.rpc.thrift.ServerProperties; +import org.apache.iotdb.service.rpc.thrift.TPipeSubscribeReq; import org.apache.iotdb.service.rpc.thrift.TPipeSubscribeResp; import org.apache.iotdb.service.rpc.thrift.TSOpenSessionResp; import org.apache.iotdb.service.rpc.thrift.TSProtocolVersion; @@ -128,6 +131,133 @@ public void testCommitReadTimeoutClosesProviderTransport() throws Exception { } } + @Test + public void testMissingBrokerPollErrorMarksProviderUnavailable() throws Exception { + assertPollError( + "Subscription: no broker bound to consumer group [group]", true /* broker missing */); + } + + @Test + public void testMissingFileBrokerPollErrorMarksProviderUnavailable() throws Exception { + assertPollError( + "Subscription: pipe broker bound to consumer group [group] does not exist", true); + } + + @Test + public void testMissingTabletBrokerPollErrorMarksProviderUnavailable() throws Exception { + assertPollError("Subscription: broker bound to consumer group [group] does not exist", true); + } + + @Test + public void testMissingTabletBrokerPollErrorFromChineseServerMarksProviderUnavailable() + throws Exception { + assertPollError("订阅:绑定到消费者组 [group] 的 Broker 不存在", true); + } + + @Test + public void testOtherPollErrorRemainsNonCritical() throws Exception { + assertPollError("other poll error", false /* broker missing */); + } + + @Test + public void testNullPollErrorRemainsNonCritical() throws Exception { + assertPollError(null, false /* broker missing */); + } + + private void assertPollError(final String message, final boolean brokerMissing) throws Exception { + final AtomicInteger subscriptionCloseCount = new AtomicInteger(); + final AtomicInteger sessionCloseCount = new AtomicInteger(); + final ServerSocket socket = new ServerSocket(0, 50, InetAddress.getByName("127.0.0.1")); + final TServerSocket serverSocket = new TServerSocket(socket); + final IClientRPCService.Iface service = + (IClientRPCService.Iface) + Proxy.newProxyInstance( + IClientRPCService.Iface.class.getClassLoader(), + new Class[] {IClientRPCService.Iface.class}, + (proxy, method, args) -> { + switch (method.getName()) { + case "openSession": + return new TSOpenSessionResp( + successStatus(), TSProtocolVersion.IOTDB_SERVICE_PROTOCOL_V3) + .setSessionId(1L); + case "requestStatementId": + return 1L; + case "getProperties": + return new ServerProperties( + "test", Arrays.asList("max_time", "min_time"), "ms", 1); + case "pipeSubscribe": + final TPipeSubscribeReq request = (TPipeSubscribeReq) args[0]; + if (request.getType() == PipeSubscribeRequestType.HANDSHAKE.getType()) { + return PipeSubscribeHandshakeResp.toTPipeSubscribeResp( + successStatus(), 1, "consumer", "group"); + } + if (request.getType() != PipeSubscribeRequestType.POLL.getType()) { + if (request.getType() == PipeSubscribeRequestType.CLOSE.getType()) { + subscriptionCloseCount.incrementAndGet(); + } + return new TPipeSubscribeResp(successStatus(), (byte) 1, (short) 6); + } + return new TPipeSubscribeResp( + new TSStatus(TSStatusCode.SUBSCRIPTION_POLL_ERROR.getStatusCode()) + .setMessage(message), + (byte) 1, + (short) 6); + case "closeSession": + sessionCloseCount.incrementAndGet(); + return successStatus(); + default: + return defaultValue(method.getReturnType()); + } + }); + final TSimpleServer server = + new TSimpleServer( + new TSimpleServer.Args(serverSocket) + .processor(new IClientRPCService.Processor<>(service)) + .transportFactory(new org.apache.iotdb.rpc.TElasticFramedTransport.Factory())); + final Thread serverThread = + new Thread(server::serve, "subscription-provider-poll-error-server"); + serverThread.setDaemon(true); + serverThread.start(); + + final TestSubscriptionProvider provider = new TestSubscriptionProvider(socket.getLocalPort()); + try { + provider.handshake(); + final TTransport transport = getTransport(provider.getSessionConnection()); + final SubscriptionCommitContext context = + new SubscriptionCommitContext(1, 0, "topic", "group", 1L); + + if (brokerMissing) { + Assert.assertThrows( + SubscriptionConnectionException.class, + () -> provider.poll(Collections.singleton("topic"), 1_000L)); + Assert.assertFalse(provider.isAvailable()); + Assert.assertThrows( + SubscriptionConnectionException.class, () -> provider.pollFile(context, 0L, 1_000L)); + Assert.assertThrows( + SubscriptionConnectionException.class, () -> provider.pollTablets(context, 0, 1_000L)); + provider.closeSession(); + Assert.assertFalse(transport.isOpen()); + Assert.assertEquals(0, subscriptionCloseCount.get()); + Assert.assertEquals(0, sessionCloseCount.get()); + } else { + Assert.assertThrows( + SubscriptionRuntimeNonCriticalException.class, + () -> provider.poll(Collections.singleton("topic"), 1_000L)); + Assert.assertTrue(provider.isAvailable()); + Assert.assertThrows( + SubscriptionRuntimeNonCriticalException.class, + () -> provider.pollFile(context, 0L, 1_000L)); + Assert.assertThrows( + SubscriptionRuntimeNonCriticalException.class, + () -> provider.pollTablets(context, 0, 1_000L)); + } + } finally { + provider.close(); + server.stop(); + serverThread.join(2_000); + } + } + private static TTransport getTransport(final SubscriptionSessionConnection connection) throws Exception { final Field field = SessionConnection.class.getDeclaredField("transport"); From b46646a60111fbb7387239272ad370f8f2d6d8bd Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 27 Oct 2026 10:36:11 +0800 Subject: [PATCH 3/4] Repair missing subscription brokers from consumer metadata (#19021) * Repair missing subscription brokers from consumer metadata * Fix subscription broker recovery rollback * Serialize subscription broker repair with region creation * Add cluster coverage for subscription broker recovery (cherry picked from commit 389e374abc15e876e68a1a413c1b5fb999bb8553) --- ...ConsensusSubscriptionBrokerRecoveryIT.java | 428 ++++++++++++++++++ .../iotdb/consensus/iot/IoTConsensus.java | 36 +- .../iot/IoTConsensusRecoveryTest.java | 22 + .../agent/SubscriptionBrokerAgent.java | 27 ++ .../agent/SubscriptionConsumerAgent.java | 87 +++- .../broker/ConsensusSubscriptionBroker.java | 46 ++ .../ConsensusSubscriptionCommitManager.java | 27 +- .../ConsensusSubscriptionSetupHandler.java | 267 ++++++----- .../receiver/SubscriptionReceiverV1.java | 6 + ...iptionConsumerAgentBrokerRecoveryTest.java | 279 ++++++++++++ ...ConsensusSubscriptionSetupHandlerTest.java | 207 --------- .../receiver/SubscriptionReceiverV1Test.java | 33 ++ 12 files changed, 1124 insertions(+), 341 deletions(-) create mode 100644 integration-test/src/test/java/org/apache/iotdb/subscription/it/consensus/local/IoTDBConsensusSubscriptionBrokerRecoveryIT.java create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/agent/SubscriptionConsumerAgentBrokerRecoveryTest.java delete mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionSetupHandlerTest.java diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/consensus/local/IoTDBConsensusSubscriptionBrokerRecoveryIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/consensus/local/IoTDBConsensusSubscriptionBrokerRecoveryIT.java new file mode 100644 index 0000000000000..b46d699072244 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/consensus/local/IoTDBConsensusSubscriptionBrokerRecoveryIT.java @@ -0,0 +1,428 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.consensus.local; + +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.commons.client.ClientPoolFactory; +import org.apache.iotdb.commons.client.IClientManager; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.commons.client.sync.SyncDataNodeInternalServiceClient; +import org.apache.iotdb.commons.consensus.ConsensusGroupId; +import org.apache.iotdb.commons.subscription.meta.consumer.ConsumerGroupMeta; +import org.apache.iotdb.confignode.rpc.thrift.TGetAllSubscriptionInfoResp; +import org.apache.iotdb.confignode.rpc.thrift.TGetCommitProgressReq; +import org.apache.iotdb.confignode.rpc.thrift.TGetCommitProgressResp; +import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.env.cluster.config.MppBaseConfig; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.ClusterIT; +import org.apache.iotdb.mpp.rpc.thrift.TPushConsumerGroupMetaResp; +import org.apache.iotdb.mpp.rpc.thrift.TPushSingleConsumerGroupMetaReq; +import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.iotdb.rpc.subscription.config.ConsumerConfig; +import org.apache.iotdb.rpc.subscription.config.ConsumerConstant; +import org.apache.iotdb.rpc.subscription.payload.poll.PollPayload; +import org.apache.iotdb.rpc.subscription.payload.poll.RegionProgress; +import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionPollRequest; +import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionPollRequestType; +import org.apache.iotdb.rpc.subscription.payload.poll.TopicProgress; +import org.apache.iotdb.rpc.subscription.payload.request.PipeSubscribeHandshakeReq; +import org.apache.iotdb.rpc.subscription.payload.request.PipeSubscribePollReq; +import org.apache.iotdb.rpc.subscription.payload.request.SubscriptionHeartbeatReq; +import org.apache.iotdb.rpc.subscription.payload.response.PipeSubscribeHeartbeatResp; +import org.apache.iotdb.service.rpc.thrift.TPipeSubscribeResp; +import org.apache.iotdb.session.subscription.SubscriptionSessionWrapper; +import org.apache.iotdb.session.subscription.SubscriptionTreeSession; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumer; +import org.apache.iotdb.subscription.it.AbstractSubscriptionIT; + +import org.awaitility.Awaitility; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.lang.reflect.Field; +import java.nio.ByteBuffer; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +@RunWith(IoTDBTestRunner.class) +@Category({ClusterIT.class}) +public class IoTDBConsensusSubscriptionBrokerRecoveryIT extends AbstractSubscriptionIT { + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS) + .setSchemaReplicationFactor(1) + .setDataReplicationFactor(3) + .setAutoCreateSchemaEnabled(true) + .setPipeMemoryManagementEnabled(false) + .setIsPipeEnableMemoryCheck(false); + // Recovery is driven explicitly below; a periodic metadata push must not mask a broken + // handshake. + final Properties config = new Properties(); + config.setProperty("subscription_enabled", "true"); + ((MppBaseConfig) EnvFactory.getEnv().getConfig().getDataNodeCommonConfig()) + .updateProperties(config); + config.setProperty("subscription_meta_syncer_initial_sync_delay_minutes", "60"); + config.setProperty("subscription_meta_syncer_sync_interval_minutes", "60"); + ((MppBaseConfig) EnvFactory.getEnv().getConfig().getConfigNodeCommonConfig()) + .updateProperties(config); + EnvFactory.getEnv().initClusterEnvironment(1, 3); + } + + @Override + @After + public void tearDown() throws Exception { + try { + EnvFactory.getEnv().cleanClusterEnvironment(); + } finally { + super.tearDown(); + } + } + + @Test + public void testHandshakeRepairsMissingBrokersWithoutResubscribing() throws Exception { + testMissingBrokerRecovery(true); + } + + @Test + public void testMetadataPushRepairsMissingBrokersWithoutRestoringCanceledTopics() + throws Exception { + testMissingBrokerRecovery(false); + } + + private void testMissingBrokerRecovery(final boolean repairByHandshake) throws Exception { + final ConsensusSubscriptionITSupport.TestIdentifiers ids = + ConsensusSubscriptionITSupport.newIdentifiers( + repairByHandshake ? "broker_handshake" : "broker_metadata"); + final String canceledTopic = ids.getTopic() + "_canceled"; + final List probes = new ArrayList<>(); + SubscriptionTreePullConsumer consumer = null; + final IClientManager clients = + new IClientManager.Factory() + .createClientManager( + new ClientPoolFactory.SyncDataNodeInternalServiceClientPoolFactory()); + try { + ConsensusSubscriptionITSupport.bootstrapDatabase(ids.getDatabase()); + ConsensusSubscriptionITSupport.createConsensusTopic( + ids.getTopic(), ids.getDatabase() + ".**"); + ConsensusSubscriptionITSupport.createConsensusTopic(canceledTopic, ids.getDatabase() + ".**"); + consumer = createConsumer(ids, TimeUnit.HOURS.toMillis(1)); + consumer.subscribe(ids.getTopic()); + final Set committedRows = + ConsensusSubscriptionITSupport.insertRows(ids.getDatabase(), "d0", 100L, 8, true); + assertRowsConsumed(consumer, committedRows); + // Creating a new subscription runs ConfigNode's commit-progress synchronization without + // background metadata pushes, then this topic is left unsubscribed. + consumer.subscribe(canceledTopic); + consumer.unsubscribe(canceledTopic); + awaitConfigNodeCheckpoint(ids, consumer.committedPositions(ids.getTopic())); + final ConsumerGroupMeta authoritativeMeta = getConsumerGroupMeta(ids.getConsumerGroupId()); + Assert.assertEquals( + Collections.singleton(ids.getTopic()), authoritativeMeta.getSubscribedTopicNames()); + final ConsumerConfig consumerConfig = createConsumerConfig(authoritativeMeta, consumer); + + for (final DataNodeWrapper node : EnvFactory.getEnv().getDataNodeWrapperList()) { + final SubscriptionSessionWrapper probe = getProviderSession(consumer, node); + probes.add(probe); + // A replacement-generation push can remove the old broker and then fail while resolving + // topic metadata. Exercise that real RPC failure without changing ConfigNode's metadata. + final ConsumerGroupMeta replacement = + new ConsumerGroupMeta( + ids.getConsumerGroupId(), + authoritativeMeta.getCreationTime() + 1, + authoritativeMeta.getConsumerMeta(ids.getConsumerId())); + replacement.addSubscription( + ids.getConsumerId(), Collections.singleton(ids.getTopic() + "_missing")); + final TPushConsumerGroupMetaResp failure = pushMeta(clients, node, replacement); + Assert.assertEquals( + failure.toString(), + TSStatusCode.CONSUMER_PUSH_META_ERROR.getStatusCode(), + failure.getStatus().getCode()); + Assert.assertTrue(failure.isSetExceptionMessages()); + Assert.assertFalse(failure.getExceptionMessages().isEmpty()); + final TPipeSubscribeResp missingBroker = + probe + .getSessionConnection() + .pipeSubscribe( + PipeSubscribePollReq.toTPipeSubscribeReq( + new SubscriptionPollRequest( + SubscriptionPollRequestType.POLL.getType(), + new PollPayload(Collections.singleton(ids.getTopic())), + 1_000L, + 1_048_576L))); + Assert.assertEquals( + missingBroker.getStatus().toString(), + TSStatusCode.SUBSCRIPTION_POLL_ERROR.getStatusCode(), + missingBroker.getStatus().getCode()); + } + Assert.assertEquals(authoritativeMeta, getConsumerGroupMeta(ids.getConsumerGroupId())); + final Set pendingRows = + ConsensusSubscriptionITSupport.insertRows(ids.getDatabase(), "d0", 200L, 8, true); + + for (int index = 0; index < probes.size(); index++) { + final SubscriptionSessionWrapper probe = probes.get(index); + if (repairByHandshake) { + assertSuccess( + probe + .getSessionConnection() + .pipeSubscribe(PipeSubscribeHandshakeReq.toTPipeSubscribeReq(consumerConfig))); + } else { + final TPushConsumerGroupMetaResp repaired = + pushMeta(clients, EnvFactory.getEnv().getDataNodeWrapper(index), authoritativeMeta); + Assert.assertEquals( + repaired.toString(), + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + repaired.getStatus().getCode()); + } + final TPipeSubscribeResp heartbeat = + probe.getSessionConnection().pipeSubscribe(SubscriptionHeartbeatReq.toThriftReq()); + assertSuccess(heartbeat); + Assert.assertEquals( + Collections.singleton(ids.getTopic()), + PipeSubscribeHeartbeatResp.fromTPipeSubscribeResp(heartbeat).getTopics().keySet()); + } + Assert.assertEquals(authoritativeMeta, getConsumerGroupMeta(ids.getConsumerGroupId())); + assertRowsConsumed(consumer, pendingRows); + assertRowsConsumed( + consumer, + ConsensusSubscriptionITSupport.insertRows(ids.getDatabase(), "d0", 300L, 8, true)); + } finally { + try { + clients.close(); + } finally { + ConsensusSubscriptionITSupport.cleanup(consumer, ids.getTopic(), ids.getDatabase()); + try (final SubscriptionTreeSession session = + new SubscriptionTreeSession( + EnvFactory.getEnv().getIP(), Integer.parseInt(EnvFactory.getEnv().getPort()))) { + session.open(); + session.dropTopicIfExists(canceledTopic); + } + } + } + } + + @Test + public void testProviderReplacementPreservesUncommittedBatch() throws Exception { + final ConsensusSubscriptionITSupport.TestIdentifiers ids = + ConsensusSubscriptionITSupport.newIdentifiers("broker_provider_replacement"); + SubscriptionTreePullConsumer consumer = null; + try { + ConsensusSubscriptionITSupport.bootstrapDatabase(ids.getDatabase()); + ConsensusSubscriptionITSupport.createConsensusTopic( + ids.getTopic(), ids.getDatabase() + ".**"); + consumer = createConsumer(ids, 1_000L); + consumer.subscribe(ids.getTopic()); + final Set rows = + ConsensusSubscriptionITSupport.insertRows(ids.getDatabase(), "d0", 100L, 8, true); + final ConsensusSubscriptionITSupport.PolledMessageBatch batch = + ConsensusSubscriptionITSupport.pollFirstNonEmptyBatchWithoutCommit( + consumer, 60, Duration.ofSeconds(1)); + Assert.assertFalse(batch.getMessages().isEmpty()); + final TopicProgress progressBeforeReplacement = consumer.committedPositions(ids.getTopic()); + final ConsumerGroupMeta metaBeforeReplacement = + getConsumerGroupMeta(ids.getConsumerGroupId()); + final Map providers = getProviders(consumer); + final Map oldProviders = new HashMap<>(providers); + Assert.assertEquals(3, oldProviders.size()); + for (final Object provider : oldProviders.values()) { + ((SubscriptionSessionWrapper) getField(provider, "session")) + .getSessionConnection() + .forceClose(); + } + Awaitility.await() + .pollInSameThread() + .atMost(Duration.ofSeconds(30)) + .untilAsserted( + () -> { + Assert.assertEquals(oldProviders.keySet(), providers.keySet()); + for (final Object nodeId : oldProviders.keySet()) { + Assert.assertNotSame(oldProviders.get(nodeId), providers.get(nodeId)); + } + }); + Assert.assertEquals(metaBeforeReplacement, getConsumerGroupMeta(ids.getConsumerGroupId())); + Assert.assertEquals(progressBeforeReplacement, consumer.committedPositions(ids.getTopic())); + consumer.commitSync(batch.getMessages()); + final ConsensusSubscriptionITSupport.ConsumedRecords consumed = batch.getConsumedRecords(); + consumed.merge( + ConsensusSubscriptionITSupport.pollAndCommitUntilAtLeast( + consumer, rows.size() - consumed.getUniqueRowCount(), 20, Duration.ofSeconds(1))); + ConsensusSubscriptionITSupport.assertExactRowKeys(rows, consumed); + assertRowsConsumed( + consumer, + ConsensusSubscriptionITSupport.insertRows(ids.getDatabase(), "d0", 200L, 8, true)); + } finally { + ConsensusSubscriptionITSupport.cleanup(consumer, ids.getTopic(), ids.getDatabase()); + } + } + + private static SubscriptionTreePullConsumer createConsumer( + final ConsensusSubscriptionITSupport.TestIdentifiers ids, final long controlIntervalMs) + throws Exception { + final SubscriptionTreePullConsumer consumer = + new SubscriptionTreePullConsumer.Builder() + .host(EnvFactory.getEnv().getIP()) + .port(Integer.parseInt(EnvFactory.getEnv().getPort())) + .consumerId(ids.getConsumerId()) + .consumerGroupId(ids.getConsumerGroupId()) + .heartbeatIntervalMs(controlIntervalMs) + .endpointsSyncIntervalMs(controlIntervalMs) + .autoCommit(false) + .buildPullConsumer(); + consumer.open(); + return consumer; + } + + private static ConsumerConfig createConsumerConfig( + final ConsumerGroupMeta meta, final SubscriptionTreePullConsumer consumer) throws Exception { + final Map attributes = + new HashMap<>(meta.getConsumerMeta(consumer.getConsumerId()).getConfig().getAttribute()); + attributes.put( + ConsumerConstant.CONSUMER_INSTANCE_ID_KEY, + (String) getField(consumer, "consumerInstanceId")); + return new ConsumerConfig(attributes); + } + + private static SubscriptionSessionWrapper getProviderSession( + final SubscriptionTreePullConsumer consumer, final DataNodeWrapper node) throws Exception { + final TEndPoint endpoint = new TEndPoint(node.getIp(), node.getPort()); + for (final Object provider : getProviders(consumer).values()) { + if (endpoint.equals(getField(provider, "endPoint"))) { + return (SubscriptionSessionWrapper) getField(provider, "session"); + } + } + throw new AssertionError("Missing subscription provider for " + endpoint); + } + + private static ConsumerGroupMeta getConsumerGroupMeta(final String consumerGroupId) + throws Exception { + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) { + final TGetAllSubscriptionInfoResp resp = client.getAllSubscriptionInfo(); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), resp.getStatus().getCode()); + for (final ByteBuffer buffer : resp.getAllSubscriptionInfo()) { + final ConsumerGroupMeta meta = ConsumerGroupMeta.deserialize(buffer); + if (consumerGroupId.equals(meta.getConsumerGroupId())) { + return meta; + } + } + throw new AssertionError("Missing consumer group " + consumerGroupId); + } + } + + private static TPushConsumerGroupMetaResp pushMeta( + final IClientManager clients, + final DataNodeWrapper node, + final ConsumerGroupMeta meta) + throws Exception { + try (final SyncDataNodeInternalServiceClient client = + clients.borrowClient(new TEndPoint(node.getInternalAddress(), node.getInternalPort()))) { + return client.pushSingleConsumerGroupMeta( + new TPushSingleConsumerGroupMetaReq().setConsumerGroupMeta(meta.serialize())); + } + } + + private static void awaitConfigNodeCheckpoint( + final ConsensusSubscriptionITSupport.TestIdentifiers ids, final TopicProgress checkpoint) { + Assert.assertNotNull(checkpoint); + Assert.assertFalse(checkpoint.getRegionProgress().isEmpty()); + Awaitility.await() + .pollInSameThread() + .atMost(Duration.ofSeconds(30)) + .untilAsserted( + () -> { + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) + EnvFactory.getEnv().getLeaderConfigNodeConnection()) { + for (final Map.Entry entry : + checkpoint.getRegionProgress().entrySet()) { + final TGetCommitProgressResp resp = + client.getCommitProgress( + new TGetCommitProgressReq( + ids.getConsumerGroupId(), + ids.getTopic(), + ConsensusGroupId.Factory.createFromString(entry.getKey()).getId(), + 0)); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), resp.getStatus().getCode()); + Assert.assertTrue(resp.isSetCommittedRegionProgress()); + Assert.assertEquals( + entry.getValue(), + RegionProgress.deserialize( + ByteBuffer.wrap(resp.getCommittedRegionProgress()))); + } + } + }); + } + + private static void assertRowsConsumed( + final SubscriptionTreePullConsumer consumer, final Set expectedRows) + throws Exception { + ConsensusSubscriptionITSupport.assertExactRowKeys( + expectedRows, + ConsensusSubscriptionITSupport.pollAndCommitUntilAtLeast( + consumer, expectedRows.size(), 60, Duration.ofSeconds(1))); + } + + private static void assertSuccess(final TPipeSubscribeResp resp) { + Assert.assertEquals( + resp.getStatus().toString(), + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + resp.getStatus().getCode()); + } + + private static Map getProviders(final SubscriptionTreePullConsumer consumer) + throws Exception { + return (Map) getField(getField(consumer, "providers"), "subscriptionProviders"); + } + + private static Object getField(final Object target, final String name) throws Exception { + for (Class type = target.getClass(); type != null; type = type.getSuperclass()) { + try { + final Field field = type.getDeclaredField(name); + field.setAccessible(true); + return field.get(target); + } catch (final NoSuchFieldException ignored) { + // Client state is declared in the consumer/provider base classes. + } + } + throw new NoSuchFieldException(name); + } +} diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensus.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensus.java index 63ddcc6eba861..28b0458c8b6f4 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensus.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensus.java @@ -82,6 +82,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; @@ -104,6 +105,7 @@ public class IoTConsensus implements IConsensus { private final IStateMachine.Registry registry; private final Map stateMachineMap = new ConcurrentHashMap<>(); + private final Set pendingPeerCreatedCallbacks = ConcurrentHashMap.newKeySet(); private final IoTConsensusRPCService service; private final RegisterManager registerManager = new RegisterManager(); private final UserDataTransferAuditHandler userDataTransferAuditHandler; @@ -223,6 +225,7 @@ private void initAndRecover() throws IOException { config, userDataTransferAuditHandler, userDataTransferAuditClassifier); + pendingPeerCreatedCallbacks.add(consensusGroupId); stateMachineMap.put(consensusGroupId, consensus); } } catch (DiskSpaceInsufficientException e) { @@ -253,15 +256,19 @@ private void initAndRecover() throws IOException { } stateMachineMap.forEach( (consensusGroupId, consensus) -> { - final BiConsumer callback = onPeerRecovering; - if (callback != null) { - callback.accept(consensusGroupId, consensus); + try { + final BiConsumer callback = onPeerRecovering; + if (callback != null) { + callback.accept(consensusGroupId, consensus); + } + consensus.start(); + // Subscription metadata may have been loaded before any local consensus peers existed. + // Recovered peers must trigger the same binding as newly created peers, after startup + // has restored their indices and installed the pre-start WAL-retention guards. + notifyPeerCreated(consensusGroupId, consensus); + } finally { + pendingPeerCreatedCallbacks.remove(consensusGroupId); } - consensus.start(); - // Subscription metadata may have been loaded before any local consensus peers existed. - // Recovered peers must trigger the same binding as newly created peers, after startup - // has restored their indices and installed the pre-start WAL-retention guards. - notifyPeerCreated(consensusGroupId, consensus); }); } @@ -355,6 +362,7 @@ public void createLocalPeer(ConsensusGroupId groupId, List peers) throw new RuntimeException(e); } impl.start(); + pendingPeerCreatedCallbacks.add(groupId); return impl; })) .orElseThrow( @@ -367,7 +375,16 @@ public void createLocalPeer(ConsensusGroupId groupId, List peers) throw new ConsensusGroupAlreadyExistException(groupId); } - notifyPeerCreated(groupId, stateMachineMap.get(groupId)); + try { + notifyPeerCreated(groupId, stateMachineMap.get(groupId)); + } finally { + pendingPeerCreatedCallbacks.remove(groupId); + } + } + + /** Returns whether a new or recovered local peer is still waiting for its creation callback. */ + public boolean isPeerCreationCallbackPending(final ConsensusGroupId groupId) { + return pendingPeerCreatedCallbacks.contains(groupId); } private void notifyPeerCreated( @@ -412,6 +429,7 @@ public void deleteLocalPeer(ConsensusGroupId groupId) throws ConsensusException if (!exist.get()) { throw new ConsensusGroupNotExistException(groupId); } + pendingPeerCreatedCallbacks.remove(groupId); FileUtils.deleteFileOrDirectoryWithRateLimiter( new File(buildPeerDir(storageDir, groupId)), RegionMigrationFileRemoveRateLimiter.getInstance()::acquire); diff --git a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/IoTConsensusRecoveryTest.java b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/IoTConsensusRecoveryTest.java index fcbb2772a62d5..4e7131f1a5c03 100644 --- a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/IoTConsensusRecoveryTest.java +++ b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/IoTConsensusRecoveryTest.java @@ -101,12 +101,14 @@ public void testRecoveredPeersNotifyAfterStarting() throws Exception { IoTConsensus.onPeerRecovering = (groupId, server) -> { assertSame(server, consensus.getImpl(groupId)); + assertTrue(consensus.isPeerCreationCallbackPending(groupId)); assertFalse(stateMachines.get(groupId).started); notifications.computeIfAbsent(groupId, ignored -> new ArrayList<>()).add("recovering"); }; IoTConsensus.onNewPeerCreated = (groupId, server) -> { assertSame(server, consensus.getImpl(groupId)); + assertTrue(consensus.isPeerCreationCallbackPending(groupId)); assertTrue(stateMachines.get(groupId).started); notifications.computeIfAbsent(groupId, ignored -> new ArrayList<>()).add("available"); }; @@ -116,6 +118,8 @@ public void testRecoveredPeersNotifyAfterStarting() throws Exception { assertEquals(2, notifications.size()); assertEquals(Arrays.asList("recovering", "available"), notifications.get(firstRegion)); assertEquals(Arrays.asList("recovering", "available"), notifications.get(secondRegion)); + assertFalse(consensus.isPeerCreationCallbackPending(firstRegion)); + assertFalse(consensus.isPeerCreationCallbackPending(secondRegion)); } @Test @@ -141,6 +145,24 @@ public void testRecoveryContinuesAfterPeerNotificationFailure() throws Exception assertTrue(notifiedRegions.containsAll(Arrays.asList(firstRegion, secondRegion))); assertTrue(stateMachines.get(firstRegion).started); assertTrue(stateMachines.get(secondRegion).started); + assertFalse(consensus.isPeerCreationCallbackPending(firstRegion)); + assertFalse(consensus.isPeerCreationCallbackPending(secondRegion)); + } + + @Test + public void testNewPeerRemainsPendingUntilCallbackReturns() throws Exception { + final DataRegionId regionId = new DataRegionId(3); + IoTConsensus.onNewPeerCreated = + (groupId, server) -> { + assertSame(server, consensus.getImpl(groupId)); + assertTrue(consensus.getAllConsensusGroupIds().contains(groupId)); + assertTrue(consensus.isPeerCreationCallbackPending(groupId)); + throw new IllegalStateException("subscription binding failed"); + }; + + createPeer(regionId); + + assertFalse(consensus.isPeerCreationCallbackPending(regionId)); } private IoTConsensus newConsensus() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionBrokerAgent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionBrokerAgent.java index 0528bff3125c5..981069d9b73d6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionBrokerAgent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionBrokerAgent.java @@ -799,6 +799,33 @@ public void unbindConsensusPrefetchingQueue( prefetchingQueueCount.invalidate(); } + public Map> getConsensusQueueRegionIds( + final String consumerGroupId, final Set topicNames) { + final Map> regionIdsByTopic = new LinkedHashMap<>(); + final ConsensusSubscriptionBroker broker = getConsensusBroker(consumerGroupId); + if (Objects.nonNull(broker)) { + for (final String topicName : topicNames) { + regionIdsByTopic.put(topicName, broker.getQueueRegionIds(topicName)); + } + } + return regionIdsByTopic; + } + + public boolean hasConsensusQueueForRegion( + final String consumerGroupId, final String topicName, final String regionId) { + final ConsensusSubscriptionBroker broker = getConsensusBroker(consumerGroupId); + return Objects.nonNull(broker) && broker.hasQueueForRegion(topicName, regionId); + } + + public void rollbackConsensusPrefetchingQueues( + final String consumerGroupId, final String topicName, final Set existingRegionIds) { + final ConsensusSubscriptionBroker broker = getConsensusBroker(consumerGroupId); + if (Objects.nonNull(broker)) { + broker.rollbackConsensusPrefetchingQueues(topicName, existingRegionIds); + prefetchingQueueCount.invalidate(); + } + } + public void unbindByRegion(final ConsensusGroupId regionId) { int totalClosed = 0; for (final ConsensusSubscriptionBroker broker : getBrokers(ConsensusSubscriptionBroker.class)) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionConsumerAgent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionConsumerAgent.java index 79f0c59ec7e01..b3c00f287a41a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionConsumerAgent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionConsumerAgent.java @@ -30,6 +30,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Collections; import java.util.LinkedHashSet; import java.util.List; import java.util.Objects; @@ -106,11 +107,7 @@ private void handleSingleConsumerGroupMetaChangesInternal( // if consumer group meta does not exist on local agent if (Objects.isNull(metaInAgent)) { - SubscriptionAgent.broker().createPipeBrokerIfNotExist(consumerGroupId); - ConsensusSubscriptionSetupHandler.setupConsensusSubscriptions( - consumerGroupId, - metaFromCoordinator.getSubscribedTopicNames(), - metaFromCoordinator.visibleUnder(true)); + ensureConsumerGroupBrokerInternal(metaFromCoordinator); replaceConsumerGroupMeta(consumerGroupId, metaFromCoordinator); return; } @@ -134,12 +131,8 @@ private void handleSingleConsumerGroupMetaChangesInternal( } } - ConsensusSubscriptionSetupHandler.setupConsensusSubscriptions( - consumerGroupId, - metaFromCoordinator.getSubscribedTopicNames(), - metaFromCoordinator.visibleUnder(true)); + ensureConsumerGroupBrokerInternal(metaFromCoordinator); replaceConsumerGroupMeta(consumerGroupId, metaFromCoordinator); - // no need to create broker manually return; } @@ -168,12 +161,7 @@ private void handleSingleConsumerGroupMetaChangesInternal( newlySubscribedTopics); applyTopicDiff( - () -> { - if (!newlySubscribedTopics.isEmpty()) { - ConsensusSubscriptionSetupHandler.handleNewSubscriptions( - consumerGroupId, newlySubscribedTopics); - } - }, + () -> ensureConsumerGroupBrokerInternal(metaFromCoordinator), () -> { for (final String topicName : pipeTopicsUnsubByGroup) { SubscriptionAgent.broker().removePrefetchingQueue(consumerGroupId, topicName); @@ -259,6 +247,48 @@ private void handleDropConsumerGroupInternal(final String consumerGroupId) { removeConsumerGroupMeta(consumerGroupId); } + /** + * Ensures that a non-empty consumer group has the local broker and its consensus subscriptions. + * + *

The coordinator metadata is authoritative here. In particular, an empty metadata must not + * recreate a broker or restore topics that were explicitly removed from the group. + */ + public void ensureConsumerGroupBroker(final String consumerGroupId) { + executeWithConsumerGroupLock( + consumerGroupId, + () -> { + final ConsumerGroupMeta consumerGroupMeta = getConsumerGroupMeta(consumerGroupId); + if (Objects.nonNull(consumerGroupMeta)) { + ensureConsumerGroupBrokerInternal(consumerGroupMeta); + } + }); + } + + private void ensureConsumerGroupBrokerInternal(final ConsumerGroupMeta consumerGroupMeta) { + if (consumerGroupMeta.isEmpty()) { + return; + } + + final String consumerGroupId = consumerGroupMeta.getConsumerGroupId(); + final boolean brokerExisted = SubscriptionAgent.broker().isBrokerExist(consumerGroupId); + SubscriptionAgent.broker().createPipeBrokerIfNotExist(consumerGroupId); + try { + // setupConsensusSubscriptions is idempotent and also repairs a missing consensus broker or + // queue when a pipe broker still exists. + ConsensusSubscriptionSetupHandler.setupConsensusSubscriptions( + consumerGroupId, + consumerGroupMeta.getSubscribedTopicNames(), + consumerGroupMeta.visibleUnder(true)); + } catch (final RuntimeException e) { + // Allow a later metadata push or handshake to retry after a partial setup failure. Preserve + // an existing broker because it may still contain usable queues for the current metadata. + if (!brokerExisted) { + SubscriptionAgent.broker().dropBroker(consumerGroupId); + } + throw e; + } + } + private ConsumerGroupMeta getConsumerGroupMeta(final String consumerGroupId) { acquireReadLock(); try { @@ -287,7 +317,7 @@ private void removeConsumerGroupMeta(final String consumerGroupId) { } } - void executeWithConsumerGroupLock(final String consumerGroupId, final Runnable operation) { + public void executeWithConsumerGroupLock(final String consumerGroupId, final Runnable operation) { final ConsumerGroupOperationLock operationLock = consumerGroupOperationLocks.compute( consumerGroupId, @@ -352,6 +382,29 @@ public boolean isTableModel(final String consumerGroupId) { } } + public boolean isTopicSubscribedByConsumerGroup( + final String consumerGroupId, final String topicName) { + acquireReadLock(); + try { + return consumerGroupMetaKeeper.isTopicSubscribedByConsumerGroup(topicName, consumerGroupId); + } finally { + releaseReadLock(); + } + } + + public Set getTopicNamesSubscribedByConsumerGroup(final String consumerGroupId) { + acquireReadLock(); + try { + final ConsumerGroupMeta consumerGroupMeta = + consumerGroupMetaKeeper.getConsumerGroupMeta(consumerGroupId); + return Objects.isNull(consumerGroupMeta) + ? Collections.emptySet() + : new LinkedHashSet<>(consumerGroupMeta.getSubscribedTopicNames()); + } finally { + releaseReadLock(); + } + } + /** * Get all active subscriptions: consumerGroupId → set of subscribed topic names. Used by * consensus subscription auto-binding when a new DataRegion is created. diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/ConsensusSubscriptionBroker.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/ConsensusSubscriptionBroker.java index d5c140d0fe7f5..8bb3b3ad084ed 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/ConsensusSubscriptionBroker.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/ConsensusSubscriptionBroker.java @@ -749,6 +749,52 @@ public void unbindConsensusPrefetchingQueue(final String topicName) { closeAndRemoveConsensusPrefetchingQueues(topicName, true, false); } + public Set getQueueRegionIds(final String topicName) { + synchronized (queueLifecycleLock) { + return topicNameToConsensusPrefetchingQueues + .getOrDefault(topicName, Collections.emptyList()) + .stream() + .filter(queue -> !queue.isClosed()) + .map(queue -> queue.getConsensusGroupId().toString()) + .collect(Collectors.toSet()); + } + } + + public boolean hasQueueForRegion(final String topicName, final String regionId) { + synchronized (queueLifecycleLock) { + return topicNameToConsensusPrefetchingQueues + .getOrDefault(topicName, Collections.emptyList()) + .stream() + .anyMatch( + queue -> + !queue.isClosed() && regionId.equals(queue.getConsensusGroupId().toString())); + } + } + + public void rollbackConsensusPrefetchingQueues( + final String topicName, final Set existingRegionIds) { + final List queuesToClose = new ArrayList<>(); + synchronized (queueLifecycleLock) { + final List queues = + topicNameToConsensusPrefetchingQueues.get(topicName); + if (Objects.isNull(queues)) { + return; + } + for (final ConsensusPrefetchingQueue queue : queues) { + if (!existingRegionIds.contains(queue.getConsensusGroupId().toString())) { + queuesToClose.add(queue); + } + } + queues.removeAll(queuesToClose); + if (queues.isEmpty()) { + topicNameToConsensusPrefetchingQueues.remove(topicName, queues); + topicConsumerLastPollMs.remove(topicName); + topicOwnershipSnapshots.remove(topicName); + } + } + queuesToClose.forEach(queue -> queue.closeAsync(false, false)); + } + public void wakeUpPrefetchingQueues(final String topicName, final boolean isTableModel) { final List queues = topicNameToConsensusPrefetchingQueues.get(topicName); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionCommitManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionCommitManager.java index 14e50ae44d0a4..53280e8d636da 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionCommitManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionCommitManager.java @@ -273,17 +273,36 @@ SetupSnapshot captureSetupSnapshot(final String consumerGroupId, final Set attemptedTopicNames) { + restoreSetupSnapshot(setupSnapshot, attemptedTopicNames, Collections.emptyMap()); + } + + void restoreSetupSnapshot( + final SetupSnapshot setupSnapshot, + final Set attemptedTopicNames, + final Map> existingQueueRegionIds) { for (final String topicName : attemptedTopicNames) { final TopicSetupSnapshot topicSnapshot = setupSnapshot.topicSnapshots.get(topicName); if (Objects.isNull(topicSnapshot)) { continue; } synchronized (getTopicPersistLock(topicSnapshot.topicFileKey)) { - removeTopicStateFromMemory(setupSnapshot.consumerGroupId, topicName); + final Set preservedRegionIds = + existingQueueRegionIds.getOrDefault(topicName, Collections.emptySet()); + // Existing queues can commit while setup is running. Keep their state objects, progress, + // and initialization markers instead of rewinding them to the setup snapshot. + removeTopicStateFromMemory(setupSnapshot.consumerGroupId, topicName, preservedRegionIds); CommitStateKey stateKeyForRewrite = null; + if (!preservedRegionIds.isEmpty()) { + stateKeyForRewrite = + getCommitStateKey( + setupSnapshot.consumerGroupId, topicName, preservedRegionIds.iterator().next()); + } for (final Map.Entry entry : topicSnapshot.stateSnapshots.entrySet()) { + if (preservedRegionIds.contains(entry.getKey())) { + continue; + } final CommitStateKey stateKey = getCommitStateKey(setupSnapshot.consumerGroupId, topicName, entry.getKey()); final StateSetupSnapshot stateSnapshot = entry.getValue(); @@ -333,12 +352,14 @@ void restoreSetupSnapshot( } } - private void removeTopicStateFromMemory(final String consumerGroupId, final String topicName) { + private void removeTopicStateFromMemory( + final String consumerGroupId, final String topicName, final Set preservedRegionIds) { final Iterator> keyIterator = commitStateKeys.entrySet().iterator(); while (keyIterator.hasNext()) { final Map.Entry entry = keyIterator.next(); - if (!entry.getValue().sameTopic(consumerGroupId, topicName)) { + if (!entry.getValue().sameTopic(consumerGroupId, topicName) + || preservedRegionIds.contains(entry.getValue().regionIdStr)) { continue; } commitStates.remove(entry.getKey()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionSetupHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionSetupHandler.java index 87b3c66f698f1..c41260fb997df 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionSetupHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionSetupHandler.java @@ -138,7 +138,7 @@ public static void ensureNewRegionListenerRegistered() { * existing subscription metadata to find all active consensus subscriptions and binds prefetching * queues to the new region. */ - private static void onNewRegionCreated( + static void onNewRegionCreated( final ConsensusGroupId groupId, final IoTConsensusServerImpl serverImpl) { if (!(groupId instanceof DataRegionId)) { return; @@ -161,110 +161,128 @@ private static void onNewRegionCreated( allSubscriptions.size(), serverImpl.getSearchIndex()); - for (final Map.Entry> groupEntry : allSubscriptions.entrySet()) { - final String consumerGroupId = groupEntry.getKey(); - final boolean isTableModel = SubscriptionAgent.consumer().isTableModel(consumerGroupId); - for (final String topicName : groupEntry.getValue()) { - if (!isConsensusBasedTopic(topicName, isTableModel)) { + for (final String consumerGroupId : allSubscriptions.keySet()) { + // Serialize binding with metadata repair and its rollback. Re-read the group's topics after + // acquiring the lock so an unsubscribe while this callback was waiting cannot be undone. + SubscriptionAgent.consumer() + .executeWithConsumerGroupLock( + consumerGroupId, + () -> + bindNewRegionForConsumerGroup( + consumerGroupId, groupId, serverImpl, commitManager)); + } + } + + private static void bindNewRegionForConsumerGroup( + final String consumerGroupId, + final ConsensusGroupId groupId, + final IoTConsensusServerImpl serverImpl, + final ConsensusSubscriptionCommitManager commitManager) { + final boolean isTableModel = SubscriptionAgent.consumer().isTableModel(consumerGroupId); + for (final String topicName : + SubscriptionAgent.consumer().getTopicNamesSubscribedByConsumerGroup(consumerGroupId)) { + if (!isConsensusBasedTopic(topicName, isTableModel)) { + continue; + } + if (SubscriptionAgent.broker() + .hasConsensusQueueForRegion(consumerGroupId, topicName, groupId.toString())) { + continue; + } + try { + final Map topicConfigs = + SubscriptionAgent.topic() + .getTopicConfigs(java.util.Collections.singleton(topicName), isTableModel); + final TopicConfig topicConfig = topicConfigs.get(topicName); + if (topicConfig == null) { continue; } - try { - final Map topicConfigs = - SubscriptionAgent.topic() - .getTopicConfigs(java.util.Collections.singleton(topicName), isTableModel); - final TopicConfig topicConfig = topicConfigs.get(topicName); - if (topicConfig == null) { - continue; - } - - // Resolve the new DataRegion's actual database name - final DataRegion dataRegion = - StorageEngine.getInstance().getDataRegion((DataRegionId) groupId); - if (dataRegion == null) { - continue; - } - final String dbRaw = dataRegion.getDatabaseName(); - final String dbTableModel = dbRaw.startsWith("root.") ? dbRaw.substring(5) : dbRaw; - - if (!matchesTopicDataRegion(dbRaw, topicConfig, isTableModel)) { - continue; - } - - final String actualDbName = topicConfig.isTableTopic() ? dbTableModel : null; - final ConsensusLogToTabletConverter converter = - buildConverter(topicName, topicConfig, actualDbName); - final SubscriptionWalRetentionPolicy retentionPolicy = - buildSubscriptionWalRetentionPolicy(topicName, topicConfig, serverImpl); - - // Recover from persisted per-writer region progress when available. The queue will - // resolve a replay start from that progress on first poll via the region-level locator. - // A newly-created region starts from its first consensus index and never proposes its - // current local tail as an initial committed frontier. - commitManager.initializeStateWithoutTailProposal(consumerGroupId, topicName, groupId); - final RegionProgress committedRegionProgress = - resolveFallbackCommittedRegionProgress( - commitManager, consumerGroupId, topicName, groupId); - final boolean hasLocalPersistedState = - commitManager.hasPersistedState(consumerGroupId, topicName, groupId); - // This region is created after the subscription already exists. Its first WAL entry may - // already be visible by the time this callback runs, so start from the beginning of this - // new region instead of tail+1 to avoid skipping the first write. - final long tailStartSearchIndex = FIRST_CONSENSUS_SEARCH_INDEX; - final long initialRuntimeVersion = - regionRuntimeVersion.getOrDefault(groupId.convertToTConsensusGroupId(), 0L); - final boolean initialActive = - lastKnownPreferredWriter.getOrDefault(groupId.convertToTConsensusGroupId(), -1) - == IOTDB_CONFIG.getDataNodeId(); - final Set initialActiveWriterNodeIds = - regionActiveWriterNodeIds.getOrDefault( - groupId.convertToTConsensusGroupId(), - initialActive - ? Collections.singleton(IOTDB_CONFIG.getDataNodeId()) - : Collections.emptySet()); - final ConsensusRegionRuntimeState initialRuntimeState = - new ConsensusRegionRuntimeState( - initialRuntimeVersion, - lastKnownPreferredWriter.getOrDefault(groupId.convertToTConsensusGroupId(), -1), - initialActive, - initialActiveWriterNodeIds); - LOGGER.info( - DataNodePipeMessages - .PIPE_LOG_AUTO_BINDING_CONSENSUS_QUEUE_FOR_TOPIC_IN_GROUP_TO_NEW_REGION_86F21649, - topicName, - consumerGroupId, - groupId, - dbTableModel, - tailStartSearchIndex, - hasLocalPersistedState, - committedRegionProgress, - initialRuntimeVersion, - initialActive); + // Resolve the new DataRegion's actual database name + final DataRegion dataRegion = + StorageEngine.getInstance().getDataRegion((DataRegionId) groupId); + if (dataRegion == null) { + continue; + } + final String dbRaw = dataRegion.getDatabaseName(); + final String dbTableModel = dbRaw.startsWith("root.") ? dbRaw.substring(5) : dbRaw; - SubscriptionAgent.broker() - .bindConsensusPrefetchingQueue( - consumerGroupId, - topicName, - topicConfig.getOrderMode(), - groupId, - serverImpl, - retentionPolicy, - converter, - commitManager, - committedRegionProgress, - tailStartSearchIndex, - initialRuntimeVersion, - initialActive); - SubscriptionAgent.broker().applyRuntimeStateForRegion(groupId, initialRuntimeState); - } catch (final Exception e) { - LOGGER.error( - DataNodePipeMessages - .PIPE_LOG_FAILED_TO_AUTO_BIND_TOPIC_IN_GROUP_TO_NEW_REGION_5BFD0E7D, - topicName, - consumerGroupId, - groupId, - e); + if (!matchesTopicDataRegion(dbRaw, topicConfig, isTableModel)) { + continue; } + + final String actualDbName = topicConfig.isTableTopic() ? dbTableModel : null; + final ConsensusLogToTabletConverter converter = + buildConverter(topicName, topicConfig, actualDbName); + final SubscriptionWalRetentionPolicy retentionPolicy = + buildSubscriptionWalRetentionPolicy(topicName, topicConfig, serverImpl); + + // Recover from persisted per-writer region progress when available. The queue will + // resolve a replay start from that progress on first poll via the region-level locator. + // A newly-created region starts from its first consensus index and never proposes its + // current local tail as an initial committed frontier. + commitManager.initializeStateWithoutTailProposal(consumerGroupId, topicName, groupId); + final RegionProgress committedRegionProgress = + resolveFallbackCommittedRegionProgress( + commitManager, consumerGroupId, topicName, groupId); + final boolean hasLocalPersistedState = + commitManager.hasPersistedState(consumerGroupId, topicName, groupId); + // This region is created after the subscription already exists. Its first WAL entry may + // already be visible by the time this callback runs, so start from the beginning of this + // new region instead of tail+1 to avoid skipping the first write. + final long tailStartSearchIndex = FIRST_CONSENSUS_SEARCH_INDEX; + final long initialRuntimeVersion = + regionRuntimeVersion.getOrDefault(groupId.convertToTConsensusGroupId(), 0L); + final boolean initialActive = + lastKnownPreferredWriter.getOrDefault(groupId.convertToTConsensusGroupId(), -1) + == IOTDB_CONFIG.getDataNodeId(); + final Set initialActiveWriterNodeIds = + regionActiveWriterNodeIds.getOrDefault( + groupId.convertToTConsensusGroupId(), + initialActive + ? Collections.singleton(IOTDB_CONFIG.getDataNodeId()) + : Collections.emptySet()); + final ConsensusRegionRuntimeState initialRuntimeState = + new ConsensusRegionRuntimeState( + initialRuntimeVersion, + lastKnownPreferredWriter.getOrDefault(groupId.convertToTConsensusGroupId(), -1), + initialActive, + initialActiveWriterNodeIds); + + LOGGER.info( + DataNodePipeMessages + .PIPE_LOG_AUTO_BINDING_CONSENSUS_QUEUE_FOR_TOPIC_IN_GROUP_TO_NEW_REGION_86F21649, + topicName, + consumerGroupId, + groupId, + dbTableModel, + tailStartSearchIndex, + hasLocalPersistedState, + committedRegionProgress, + initialRuntimeVersion, + initialActive); + + SubscriptionAgent.broker() + .bindConsensusPrefetchingQueue( + consumerGroupId, + topicName, + topicConfig.getOrderMode(), + groupId, + serverImpl, + retentionPolicy, + converter, + commitManager, + committedRegionProgress, + tailStartSearchIndex, + initialRuntimeVersion, + initialActive); + SubscriptionAgent.broker().applyRuntimeStateForRegion(groupId, initialRuntimeState); + } catch (final Exception e) { + LOGGER.error( + DataNodePipeMessages.PIPE_LOG_FAILED_TO_AUTO_BIND_TOPIC_IN_GROUP_TO_NEW_REGION_5BFD0E7D, + topicName, + consumerGroupId, + groupId, + e); } } } @@ -341,6 +359,14 @@ public static void setupConsensusSubscriptions( public static void setupConsensusSubscriptions( final String consumerGroupId, final Set topicNames, final boolean isTableModel) { + SubscriptionAgent.consumer() + .executeWithConsumerGroupLock( + consumerGroupId, + () -> setupConsensusSubscriptionsInternal(consumerGroupId, topicNames, isTableModel)); + } + + private static void setupConsensusSubscriptionsInternal( + final String consumerGroupId, final Set topicNames, final boolean isTableModel) { final IConsensus dataRegionConsensus = DataRegionConsensusImpl.getInstance(); if (!(dataRegionConsensus instanceof IoTConsensus)) { final String configuredProtocol = IOTDB_CONFIG.getDataRegionConsensusProtocolClass(); @@ -370,6 +396,10 @@ public static void setupConsensusSubscriptions( topicNames, ioTConsensus.getAllConsensusGroupIds().size()); + // Recovery also visits topics with live queues. Roll back only newly bound regions so a + // failure on another topic cannot discard their in-flight events or committed progress. + final Map> existingQueueRegionIds = + SubscriptionAgent.broker().getConsensusQueueRegionIds(consumerGroupId, topicNames); final ConsensusSubscriptionCommitManager.SetupSnapshot setupSnapshot = commitManager.captureSetupSnapshot(consumerGroupId, topicNames); setupConsensusTopics( @@ -381,18 +411,27 @@ public static void setupConsensusSubscriptions( consumerGroupId, topicName, isTableModel, ioTConsensus, commitManager), attemptedTopicNames -> rollbackConsensusSubscriptionSetup( - consumerGroupId, attemptedTopicNames, commitManager, setupSnapshot)); + consumerGroupId, + attemptedTopicNames, + commitManager, + setupSnapshot, + existingQueueRegionIds)); } private static void rollbackConsensusSubscriptionSetup( final String consumerGroupId, final Set attemptedTopicNames, final ConsensusSubscriptionCommitManager commitManager, - final ConsensusSubscriptionCommitManager.SetupSnapshot setupSnapshot) { + final ConsensusSubscriptionCommitManager.SetupSnapshot setupSnapshot, + final Map> existingQueueRegionIds) { RuntimeException rollbackFailure = null; for (final String topicName : attemptedTopicNames) { try { - SubscriptionAgent.broker().unbindConsensusPrefetchingQueue(consumerGroupId, topicName); + SubscriptionAgent.broker() + .rollbackConsensusPrefetchingQueues( + consumerGroupId, + topicName, + existingQueueRegionIds.getOrDefault(topicName, Collections.emptySet())); } catch (final RuntimeException e) { if (Objects.isNull(rollbackFailure)) { rollbackFailure = e; @@ -402,7 +441,8 @@ private static void rollbackConsensusSubscriptionSetup( } } try { - commitManager.restoreSetupSnapshot(setupSnapshot, attemptedTopicNames); + commitManager.restoreSetupSnapshot( + setupSnapshot, attemptedTopicNames, existingQueueRegionIds); } catch (final RuntimeException e) { if (Objects.isNull(rollbackFailure)) { rollbackFailure = e; @@ -471,7 +511,7 @@ interface ConsensusTopicSetup { * DATABASE_KEY} filter. Additionally, the {@link #onNewRegionCreated} callback ensures that * regions created after this method runs are also automatically bound. */ - private static void setupConsensusQueueForTopic( + static void setupConsensusQueueForTopic( final String consumerGroupId, final String topicName, final boolean isTableModel, @@ -515,6 +555,23 @@ private static void setupConsensusQueueForTopic( if (!(groupId instanceof DataRegionId)) { continue; } + if (ioTConsensus.isPeerCreationCallbackPending(groupId) + && SubscriptionAgent.consumer() + .isTopicSubscribedByConsumerGroup(consumerGroupId, topicName)) { + // The new-region callback binds this peer from its first WAL index. Binding it here from + // the current tail could skip writes replicated before the callback finishes. + bound = true; + continue; + } + if (SubscriptionAgent.broker() + .hasConsensusQueueForRegion(consumerGroupId, topicName, groupId.toString())) { + // In particular, a queue created by onNewRegionCreated must keep its initialization from + // the beginning of the region instead of being reinitialized from a local tail proposal. + // Check the live broker as well because a region callback can finish after the setup + // snapshot was captured. + bound = true; + continue; + } final IoTConsensusServerImpl serverImpl = ioTConsensus.getImpl(groupId); if (serverImpl == null) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1.java index cd53d9376ab77..93d309cc5db80 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1.java @@ -454,6 +454,12 @@ private TPipeSubscribeResp handlePipeSubscribeHandshakeInternal( consumerConfig); } + // A previous inactivity cleanup or metadata race may have removed the local broker while the + // coordinator still has a non-empty consumer-group metadata record. Reconcile that state before + // exposing the consumer to subsequent requests. The operation is idempotent and serialized with + // metadata pushes by SubscriptionConsumerAgent. + SubscriptionAgent.consumer().ensureConsumerGroupBroker(consumerGroupId); + activateConsumer(consumerConfig); final int dataNodeId = IoTDBDescriptor.getInstance().getConfig().getDataNodeId(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/agent/SubscriptionConsumerAgentBrokerRecoveryTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/agent/SubscriptionConsumerAgentBrokerRecoveryTest.java new file mode 100644 index 0000000000000..d281c85d5ccc8 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/agent/SubscriptionConsumerAgentBrokerRecoveryTest.java @@ -0,0 +1,279 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.subscription.agent; + +import org.apache.iotdb.commons.pipe.agent.task.connection.UnboundedBlockingPendingQueue; +import org.apache.iotdb.commons.subscription.meta.consumer.ConsumerGroupMeta; +import org.apache.iotdb.commons.subscription.meta.consumer.ConsumerMeta; +import org.apache.iotdb.commons.subscription.meta.topic.TopicMeta; +import org.apache.iotdb.consensus.IConsensus; +import org.apache.iotdb.consensus.iot.IoTConsensus; +import org.apache.iotdb.db.consensus.DataRegionConsensusImpl; +import org.apache.iotdb.db.pipe.metric.source.PipeDataRegionEventCounter; +import org.apache.iotdb.db.subscription.broker.ISubscriptionBroker; +import org.apache.iotdb.db.subscription.broker.SubscriptionBroker; +import org.apache.iotdb.db.subscription.task.subtask.SubscriptionSinkSubtask; +import org.apache.iotdb.pipe.api.event.Event; +import org.apache.iotdb.rpc.subscription.config.ConsumerConfig; +import org.apache.iotdb.rpc.subscription.config.ConsumerConstant; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; + +import org.junit.Test; + +import java.lang.reflect.Field; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.clearInvocations; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class SubscriptionConsumerAgentBrokerRecoveryTest { + + private static final long TEST_TIMEOUT_SECONDS = 5L; + + @Test + public void testNonEmptyMetadataRecreatesMissingBroker() { + final String consumerGroupId = unique("group"); + final String consumerId = unique("consumer"); + final SubscriptionConsumerAgent agent = SubscriptionAgent.consumer(); + try { + final ConsumerGroupMeta meta = createMeta(consumerGroupId, consumerId); + assertNull(agent.handleSingleConsumerGroupMetaChanges(meta)); + assertTrue(SubscriptionAgent.broker().isBrokerExist(consumerGroupId)); + + assertTrue(SubscriptionAgent.broker().dropBroker(consumerGroupId)); + assertFalse(SubscriptionAgent.broker().isBrokerExist(consumerGroupId)); + + agent.ensureConsumerGroupBroker(consumerGroupId); + + assertTrue(SubscriptionAgent.broker().isBrokerExist(consumerGroupId)); + assertTrue(agent.isConsumerExisted(consumerGroupId, consumerId)); + } finally { + agent.handleDropConsumerGroup(consumerGroupId); + } + } + + @Test + public void testMetadataPushSetsUpNewConsensusTopicOnce() { + final String consumerGroupId = unique("group"); + final String consumerId = unique("consumer"); + final String topicName = unique("topic"); + final SubscriptionConsumerAgent agent = SubscriptionAgent.consumer(); + final IConsensus originalConsensus = DataRegionConsensusImpl.getInstance(); + final IoTConsensus consensus = mock(IoTConsensus.class); + when(consensus.getAllConsensusGroupIds()).thenReturn(Collections.emptyList()); + DataRegionConsensusImpl.setInstance(consensus); + try { + assertNull( + SubscriptionAgent.topic() + .handleSingleTopicMetaChanges( + new TopicMeta( + topicName, + 1L, + Collections.singletonMap( + TopicConstant.MODE_KEY, TopicConstant.MODE_INCREMENTAL_VALUE)))); + final ConsumerGroupMeta meta = createMeta(consumerGroupId, consumerId); + assertNull(agent.handleSingleConsumerGroupMetaChanges(meta)); + clearInvocations(consensus); + + final ConsumerGroupMeta subscribedMeta = meta.deepCopy(); + subscribedMeta.addSubscription(consumerId, Collections.singleton(topicName)); + assertNull(agent.handleSingleConsumerGroupMetaChanges(subscribedMeta)); + + // One discovery for setup logging and one for binding the topic's regions. + verify(consensus, times(2)).getAllConsensusGroupIds(); + assertEquals( + Collections.singleton(topicName), + agent.getTopicNamesSubscribedByConsumer(consumerGroupId, consumerId)); + } finally { + agent.handleDropConsumerGroup(consumerGroupId); + SubscriptionAgent.topic().handleDropTopic(topicName, false); + DataRegionConsensusImpl.setInstance(originalConsensus); + } + } + + @Test + public void testEmptyMetadataDoesNotRecreateBroker() { + final String consumerGroupId = unique("group"); + final String consumerId = unique("consumer"); + final SubscriptionConsumerAgent agent = SubscriptionAgent.consumer(); + try { + final ConsumerGroupMeta meta = createMeta(consumerGroupId, consumerId); + assertNull(agent.handleSingleConsumerGroupMetaChanges(meta)); + assertTrue(SubscriptionAgent.broker().isBrokerExist(consumerGroupId)); + + final ConsumerGroupMeta emptyMeta = meta.deepCopy(); + emptyMeta.removeConsumer(consumerId); + assertTrue(emptyMeta.isEmpty()); + assertNull(agent.handleSingleConsumerGroupMetaChanges(emptyMeta)); + + assertFalse(SubscriptionAgent.broker().isBrokerExist(consumerGroupId)); + agent.ensureConsumerGroupBroker(consumerGroupId); + assertFalse(SubscriptionAgent.broker().isBrokerExist(consumerGroupId)); + } finally { + agent.handleDropConsumerGroup(consumerGroupId); + } + } + + @Test + public void testConcurrentMetadataPushAndBrokerEnsureCreateOneBroker() throws Exception { + final String consumerGroupId = unique("group"); + final String consumerId = unique("consumer"); + final SubscriptionConsumerAgent agent = SubscriptionAgent.consumer(); + final ExecutorService executor = Executors.newFixedThreadPool(2); + try { + final ConsumerGroupMeta meta = createMeta(consumerGroupId, consumerId); + assertNull(agent.handleSingleConsumerGroupMetaChanges(meta)); + assertTrue(SubscriptionAgent.broker().dropBroker(consumerGroupId)); + + final Future metadataPush = + executor.submit(() -> agent.handleSingleConsumerGroupMetaChanges(meta.deepCopy())); + final Future handshakeRepair = + executor.submit(() -> agent.ensureConsumerGroupBroker(consumerGroupId)); + metadataPush.get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS); + handshakeRepair.get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS); + + assertEquals(1, getBrokers(consumerGroupId).size()); + } finally { + executor.shutdownNow(); + executor.awaitTermination(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS); + agent.handleDropConsumerGroup(consumerGroupId); + } + } + + @Test + public void testBrokerAndExistingQueueSurviveProviderReplacementRepair() throws Exception { + final String consumerGroupId = unique("group"); + final String consumerId = unique("consumer"); + final String topicName = unique("topic"); + final SubscriptionConsumerAgent agent = SubscriptionAgent.consumer(); + try { + assertNull( + SubscriptionAgent.topic() + .handleSingleTopicMetaChanges(new TopicMeta(topicName, 1L, Collections.emptyMap()))); + final ConsumerGroupMeta meta = createMeta(consumerGroupId, consumerId); + meta.addSubscription(consumerId, Collections.singleton(topicName)); + assertNull(agent.handleSingleConsumerGroupMetaChanges(meta)); + + bindPipeQueue(consumerGroupId, topicName); + final SubscriptionBroker brokerBeforeRepair = getPipeBroker(consumerGroupId); + assertTrue(brokerBeforeRepair.hasQueue(topicName)); + + // A provider replacement closes only its session. The server-side repair must preserve the + // existing broker and queue when it reconciles the same non-empty metadata. + agent.ensureConsumerGroupBroker(consumerGroupId); + + assertSame(brokerBeforeRepair, getPipeBroker(consumerGroupId)); + assertTrue(getPipeBroker(consumerGroupId).hasQueue(topicName)); + } finally { + agent.handleDropConsumerGroup(consumerGroupId); + SubscriptionAgent.topic().handleDropTopic(topicName, false); + } + } + + @Test + public void testCanceledTopicIsNotRestoredWhenBrokerIsRecreated() throws Exception { + final String consumerGroupId = unique("group"); + final String consumerId = unique("consumer"); + final String topicName = unique("topic"); + final SubscriptionConsumerAgent agent = SubscriptionAgent.consumer(); + try { + assertNull( + SubscriptionAgent.topic() + .handleSingleTopicMetaChanges(new TopicMeta(topicName, 1L, Collections.emptyMap()))); + final ConsumerGroupMeta meta = createMeta(consumerGroupId, consumerId); + meta.addSubscription(consumerId, Collections.singleton(topicName)); + assertNull(agent.handleSingleConsumerGroupMetaChanges(meta)); + + bindPipeQueue(consumerGroupId, topicName); + assertTrue(getPipeBroker(consumerGroupId).hasQueue(topicName)); + + final ConsumerGroupMeta unsubscribedMeta = meta.deepCopy(); + unsubscribedMeta.removeSubscription(consumerId, Collections.singleton(topicName)); + assertNull(agent.handleSingleConsumerGroupMetaChanges(unsubscribedMeta)); + assertFalse(getPipeBroker(consumerGroupId).hasQueue(topicName)); + + assertTrue(SubscriptionAgent.broker().dropBroker(consumerGroupId)); + agent.ensureConsumerGroupBroker(consumerGroupId); + + assertFalse(getPipeBroker(consumerGroupId).hasQueue(topicName)); + assertFalse(agent.getAllSubscriptions().containsKey(consumerGroupId)); + } finally { + agent.handleDropConsumerGroup(consumerGroupId); + SubscriptionAgent.topic().handleDropTopic(topicName, false); + } + } + + private static ConsumerGroupMeta createMeta( + final String consumerGroupId, final String consumerId) { + final Map attributes = new HashMap<>(); + attributes.put(ConsumerConstant.CONSUMER_ID_KEY, consumerId); + attributes.put(ConsumerConstant.CONSUMER_GROUP_ID_KEY, consumerGroupId); + final ConsumerConfig consumerConfig = new ConsumerConfig(attributes); + return new ConsumerGroupMeta( + consumerGroupId, 1L, new ConsumerMeta(consumerId, 1L, consumerConfig.getAttribute())); + } + + private static void bindPipeQueue(final String consumerGroupId, final String topicName) { + final UnboundedBlockingPendingQueue inputQueue = + new UnboundedBlockingPendingQueue<>(new PipeDataRegionEventCounter()); + final SubscriptionSinkSubtask subtask = + new SubscriptionSinkSubtask( + unique("task"), 1L, "", 0, inputQueue, null, topicName, consumerGroupId); + SubscriptionAgent.broker().bindPrefetchingQueue(subtask); + } + + @SuppressWarnings("unchecked") + private static List getBrokers(final String consumerGroupId) + throws Exception { + final Field field = SubscriptionBrokerAgent.class.getDeclaredField("consumerGroupIdToBrokers"); + field.setAccessible(true); + final Map> brokersByConsumerGroup = + (Map>) field.get(SubscriptionAgent.broker()); + return brokersByConsumerGroup.getOrDefault(consumerGroupId, Collections.emptyList()); + } + + private static SubscriptionBroker getPipeBroker(final String consumerGroupId) throws Exception { + return getBrokers(consumerGroupId).stream() + .filter(SubscriptionBroker.class::isInstance) + .map(SubscriptionBroker.class::cast) + .findFirst() + .orElseThrow(AssertionError::new); + } + + private static String unique(final String prefix) { + return prefix + '-' + UUID.randomUUID(); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionSetupHandlerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionSetupHandlerTest.java deleted file mode 100644 index 30cbd487668a7..0000000000000 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionSetupHandlerTest.java +++ /dev/null @@ -1,207 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.subscription.broker.consensus; - -import org.apache.iotdb.commons.consensus.DataRegionId; -import org.apache.iotdb.commons.pipe.config.constant.SystemConstant; -import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.rpc.subscription.config.TopicConfig; -import org.apache.iotdb.rpc.subscription.config.TopicConstant; -import org.apache.iotdb.rpc.subscription.exception.SubscriptionException; - -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -import java.io.File; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedHashSet; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; - -public class ConsensusSubscriptionSetupHandlerTest { - - @Rule public final TemporaryFolder temporaryFolder = new TemporaryFolder(); - - @Test - public void testRuntimeUpdatesAreIgnoredWhenSubscriptionIsDisabled() { - ConsensusSubscriptionSetupHandler.applyRuntimeState(null, null); - ConsensusSubscriptionSetupHandler.onRegionRouteChanged(null, 0); - } - - @Test - public void testSingleTopicSetupFailurePropagates() { - SubscriptionException failure = null; - try { - ConsensusSubscriptionSetupHandler.setupConsensusTopics( - "consumerGroup", - Collections.singleton("topic"), - topicName -> true, - topicName -> { - throw new IllegalStateException("setup failed"); - }, - ignored -> {}); - } catch (final SubscriptionException e) { - failure = e; - } - - assertNotNull(failure); - assertTrue(failure.getMessage().contains("topic")); - assertTrue(failure.getMessage().contains("consumerGroup")); - assertTrue(failure.getCause() instanceof IllegalStateException); - } - - @Test - public void testTopicModeLookupFailurePropagates() { - SubscriptionException failure = null; - try { - ConsensusSubscriptionSetupHandler.setupConsensusTopics( - "consumerGroup", - Collections.singleton("topic"), - topicName -> { - throw new IllegalStateException("topic metadata missing"); - }, - topicName -> {}, - ignored -> {}); - } catch (final SubscriptionException e) { - failure = e; - } - - assertNotNull(failure); - assertTrue(failure.getCause() instanceof IllegalStateException); - } - - @Test - public void testMultiTopicSetupFailureRollsBackAllAttemptedTopics() { - final Set attemptedTopicNames = new LinkedHashSet<>(); - final Set rolledBackTopicNames = new LinkedHashSet<>(); - final Set topicNames = new LinkedHashSet<>(Arrays.asList("first", "second", "third")); - SubscriptionException failure = null; - - try { - ConsensusSubscriptionSetupHandler.setupConsensusTopics( - "consumerGroup", - topicNames, - topicName -> true, - topicName -> failOnSecondTopic(topicName, attemptedTopicNames), - attemptedTopics -> rolledBackTopicNames.addAll(attemptedTopics)); - } catch (final SubscriptionException e) { - failure = e; - } - - assertNotNull(failure); - assertEquals(new LinkedHashSet<>(Arrays.asList("first", "second")), attemptedTopicNames); - assertEquals(attemptedTopicNames, rolledBackTopicNames); - } - - @Test - public void testFallbackLookupDoesNotQueryConfigNodeWithoutLocalPersistence() throws Exception { - final String originalSystemDir = IoTDBDescriptor.getInstance().getConfig().getSystemDir(); - final File systemDir = temporaryFolder.newFolder("noLocalPersistence"); - try { - IoTDBDescriptor.getInstance().getConfig().setSystemDir(systemDir.getAbsolutePath()); - final AtomicInteger queryCount = new AtomicInteger(); - final ConsensusSubscriptionCommitManager commitManager = - new ConsensusSubscriptionCommitManager( - (consumerGroupId, topicName, regionId) -> { - queryCount.incrementAndGet(); - return ConsensusSubscriptionCommitManager.ConfigNodeProgressQueryResult.absent(); - }); - - assertNull( - ConsensusSubscriptionSetupHandler.resolveFallbackCommittedRegionProgress( - commitManager, "consumerGroup", "topic", new DataRegionId(1))); - assertEquals(0, queryCount.get()); - } finally { - IoTDBDescriptor.getInstance().getConfig().setSystemDir(originalSystemDir); - } - } - - @Test - public void testAuditDatabaseNeverMatchesTopic() { - final Map tableTopicAttributes = new HashMap<>(); - tableTopicAttributes.put( - SystemConstant.SQL_DIALECT_KEY, SystemConstant.SQL_DIALECT_TABLE_VALUE); - tableTopicAttributes.put(TopicConstant.DATABASE_KEY, ".*"); - final TopicConfig tableTopicConfig = new TopicConfig(tableTopicAttributes); - - assertFalse( - ConsensusSubscriptionSetupHandler.matchesTopicDatabase(tableTopicConfig, "__audit")); - assertFalse( - ConsensusSubscriptionSetupHandler.matchesTopicDatabase(tableTopicConfig, "root.__audit")); - assertTrue(ConsensusSubscriptionSetupHandler.matchesTopicDatabase(tableTopicConfig, "user_db")); - - final TopicConfig treeTopicConfig = new TopicConfig(Collections.emptyMap()); - assertFalse(ConsensusSubscriptionSetupHandler.matchesTopicDatabase(treeTopicConfig, "__audit")); - assertTrue(ConsensusSubscriptionSetupHandler.matchesTopicDatabase(treeTopicConfig, "user_db")); - } - - @Test - public void testTopicDataRegionModelIsolation() { - final TopicConfig treeTopicConfig = new TopicConfig(Collections.emptyMap()); - final Map tableTopicAttributes = new HashMap<>(); - tableTopicAttributes.put( - SystemConstant.SQL_DIALECT_KEY, SystemConstant.SQL_DIALECT_TABLE_VALUE); - tableTopicAttributes.put(TopicConstant.DATABASE_KEY, "table_db"); - final TopicConfig tableTopicConfig = new TopicConfig(tableTopicAttributes); - - assertTrue( - ConsensusSubscriptionSetupHandler.matchesTopicDataRegion( - "root.tree_db", treeTopicConfig, false)); - assertFalse( - ConsensusSubscriptionSetupHandler.matchesTopicDataRegion( - "table_db", treeTopicConfig, false)); - assertFalse( - ConsensusSubscriptionSetupHandler.matchesTopicDataRegion( - "table_db", treeTopicConfig, true)); - assertFalse( - ConsensusSubscriptionSetupHandler.matchesTopicDataRegion( - "root.table_db", tableTopicConfig, true)); - assertTrue( - ConsensusSubscriptionSetupHandler.matchesTopicDataRegion( - "table_db", tableTopicConfig, true)); - assertFalse( - ConsensusSubscriptionSetupHandler.matchesTopicDataRegion( - "other_table_db", tableTopicConfig, true)); - assertFalse( - ConsensusSubscriptionSetupHandler.matchesTopicDataRegion( - "table_db", tableTopicConfig, false)); - assertFalse( - ConsensusSubscriptionSetupHandler.matchesTopicDataRegion( - "root.table_db", tableTopicConfig, false)); - } - - private static void failOnSecondTopic( - final String topicName, final Set attemptedTopicNames) { - attemptedTopicNames.add(topicName); - if ("second".equals(topicName)) { - throw new IllegalStateException("setup failed"); - } - } -} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1Test.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1Test.java index 40482fe1b20a2..05eccb978f90a 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1Test.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1Test.java @@ -221,6 +221,39 @@ public void testFencedConsumerCannotHandshakeAgain() throws Exception { Assert.assertTrue((boolean) getField(receiver, "consumerFenced")); } + @Test + public void testHandshakeRepairsMissingBrokerWithoutChangingConsumerMetadata() throws Exception { + final String consumerGroupId = "group-" + UUID.randomUUID(); + final String consumerId = "consumer-" + UUID.randomUUID(); + final ConsumerConfig consumerConfig = + createConsumerConfig(1_000L, null, null, consumerId, consumerGroupId); + final ConsumerGroupMeta consumerGroupMeta = + new ConsumerGroupMeta( + consumerGroupId, + System.currentTimeMillis(), + new ConsumerMeta( + consumerId, System.currentTimeMillis(), consumerConfig.getAttribute())); + final SubscriptionReceiverV1 receiver = new SubscriptionReceiverV1(); + Assert.assertNull( + SubscriptionAgent.consumer().handleSingleConsumerGroupMetaChanges(consumerGroupMeta)); + try { + Assert.assertTrue(SubscriptionAgent.broker().dropBroker(consumerGroupId)); + Assert.assertFalse(SubscriptionAgent.broker().isBrokerExist(consumerGroupId)); + + final TSStatus status = + receiver + .handle(PipeSubscribeHandshakeReq.toTPipeSubscribeReq(consumerConfig)) + .getStatus(); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertTrue(SubscriptionAgent.broker().isBrokerExist(consumerGroupId)); + Assert.assertTrue( + SubscriptionAgent.consumer().isConsumerExisted(consumerGroupId, consumerId)); + } finally { + SubscriptionAgent.consumer().handleDropConsumerGroup(consumerGroupId); + } + } + @Test public void testNeverHandshakenConsumerStillReturnsMissingConsumerStatus() { final SubscriptionReceiverV1 receiver = new SubscriptionReceiverV1(); From d87e9e202d59b46ce393d9135ab3054d76963a8b Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 27 Oct 2026 15:44:35 +0800 Subject: [PATCH 4/4] Fix subscription commit recovery after RPC timeouts (#19023) Update ConsensusSubscriptionBroker.java * Retry explicit subscription commits after connection failures * Preserve completed subscription commit batches during retries * Preserve newer subscription checkpoints across commit retries * Retry transient subscription poll cleanup NACKs * Retry only unconfirmed subscription acknowledgements * Preserve subscription ACK progress across concurrent commits and server errors (cherry picked from commit 07dc28102b8840d82a7b7d7e47530f0660b4005f) --- .../base/AbstractSubscriptionConsumer.java | 182 ++++++++++-- .../base/AbstractSubscriptionProvider.java | 40 ++- .../SubscriptionConsumerLifecycleTest.java | 2 +- ...criptionConsumerMultiProviderPollTest.java | 281 ------------------ .../resource/memory/PipeMemoryManager.java | 8 +- .../dataregion/memtable/TsFileProcessor.java | 3 +- .../broker/ConsensusSubscriptionBroker.java | 2 +- .../memory/PipeMemoryManagerResizeTest.java | 4 +- .../dataregion/DataRegionTest.java | 13 +- .../rescon/memory/SystemInfoTest.java | 24 +- 10 files changed, 219 insertions(+), 340 deletions(-) delete mode 100644 iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionConsumerMultiProviderPollTest.java diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumer.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumer.java index 122b4138ee77f..391c4d56e2e2c 100644 --- a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumer.java +++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumer.java @@ -110,6 +110,9 @@ abstract class AbstractSubscriptionConsumer implements AutoCloseable { private static final long SLEEP_MS = 100L; private static final long SLEEP_DELTA_MS = 50L; private static final long TIMER_DELTA_MS = 250L; + private static final int MAX_COMMIT_RETRIES = 3; + private static final long COMMIT_RETRY_INTERVAL_MS = + ConsumerConstant.ENDPOINTS_SYNC_INTERVAL_MS_MIN_VALUE; private static final AtomicLong LAST_CONSUMER_INSTANCE_EPOCH = new AtomicLong(); private final String username; @@ -1060,9 +1063,7 @@ private Optional pollFile( inFlightFilesCommitContextSet.remove(commitContext); } // construct temporary message to nack - nack( - Collections.singletonList( - new SubscriptionMessage(commitContext, file.getAbsolutePath(), null))); + nackAfterPollFailure(new SubscriptionMessage(commitContext, file.getAbsolutePath(), null), e); throw new SubscriptionRuntimeNonCriticalException(e.getMessage(), e); } } @@ -1271,13 +1272,25 @@ private Optional pollTablets( throw (SubscriptionConsumerFencedException) e; } // construct temporary message to nack - nack( - Collections.singletonList( - new SubscriptionMessage(response.getCommitContext(), Collections.emptyMap()))); + nackAfterPollFailure( + new SubscriptionMessage(response.getCommitContext(), Collections.emptyMap()), e); throw new SubscriptionRuntimeNonCriticalException(e.getMessage(), e); } } + private void nackAfterPollFailure(final SubscriptionMessage message, final Exception pollFailure) + throws SubscriptionException { + try { + nack(Collections.singletonList(message)); + } catch (final SubscriptionConnectionException nackFailure) { + // Recovery was scheduled by commitInternal. A NACK replay can increment the server's poison + // message counter again. Preserve the poll failure and let the server recycle the event. + if (nackFailure != pollFailure) { + pollFailure.addSuppressed(nackFailure); + } + } + } + private Optional pollTabletsInternal( final SubscriptionPollResponse initialResponse, final PollTimer timer) { final Map> tablets = @@ -1536,15 +1549,9 @@ private void commit(final Iterable commitContexts, fi dataNodeIdToSubscriptionCommitContexts.entrySet()) { final List groupedCommitContexts = entry.getValue(); requestedCount += groupedCommitContexts.size(); - final AbstractSubscriptionProvider.CommitResult commitResp = - commitInternal(entry.getKey(), groupedCommitContexts, nack); final List acceptedCommitContexts = - commitResp.getAcceptedCommitContexts(); + commitWithRetry(entry.getKey(), groupedCommitContexts, nack); acceptedCount += acceptedCommitContexts.size(); - if (!nack) { - overlayCommittedPositions(commitResp.getCommittedProgressByTopic()); - onAckedCommitContexts(acceptedCommitContexts); - } if (acceptedCommitContexts.size() != groupedCommitContexts.size()) { final List failedInGroup = new ArrayList<>(groupedCommitContexts); @@ -1565,6 +1572,88 @@ private void commit(final Iterable commitContexts, fi } } + private List commitWithRetry( + final int dataNodeId, + final List commitContexts, + final boolean nack) + throws SubscriptionException { + final List remainingCommitContexts = new ArrayList<>(commitContexts); + final List acceptedCommitContexts = new ArrayList<>(); + for (int retries = 0; ; retries++) { + checkIfFenced(); + AbstractSubscriptionProvider.CommitResult result = + AbstractSubscriptionProvider.CommitResult.empty(); + SubscriptionConnectionException connectionFailure = null; + SubscriptionException failure = null; + try { + result = commitInternal(dataNodeId, remainingCommitContexts, nack); + } catch (final SubscriptionException e) { + failure = getCommitFailure(e); + result = getCompletedCommitResult(e); + if (failure instanceof SubscriptionConnectionException) { + connectionFailure = (SubscriptionConnectionException) failure; + } + } + + final List acceptedInAttempt = result.getAcceptedCommitContexts(); + acceptedCommitContexts.addAll(acceptedInAttempt); + if (!nack) { + // Apply each response once, before waiting. Reapplying older progress at retry completion + // could overwrite a concurrent commit; accepted contexts must also leave the buffer now. + overlayCommittedPositions(result.getCommittedProgressByTopic()); + onAckedCommitContexts(acceptedInAttempt); + } + if (failure != null && connectionFailure == null) { + throw failure; + } + remainingCommitContexts.removeAll(new HashSet<>(acceptedInAttempt)); + if (nack + || remainingCommitContexts.isEmpty() + || retries >= MAX_COMMIT_RETRIES + || isClosed() + || Thread.currentThread().isInterrupted()) { + if (connectionFailure != null) { + throw connectionFailure; + } + return acceptedCommitContexts; + } + checkIfFenced(); + // A partial response has no permanent-rejection reason. Retry only its rejected contexts + // and any batches whose response was lost, using the same budget as connection recovery. + // commitInternal has released the provider read lock before we wait for recovery. + try { + Thread.sleep(COMMIT_RETRY_INTERVAL_MS); + } catch (final InterruptedException interruptedException) { + Thread.currentThread().interrupt(); + if (connectionFailure != null) { + connectionFailure.addSuppressed(interruptedException); + throw connectionFailure; + } + return acceptedCommitContexts; + } + checkIfFenced(); + if (isClosed()) { + if (connectionFailure != null) { + throw connectionFailure; + } + return acceptedCommitContexts; + } + } + } + + private SubscriptionException getCommitFailure(final SubscriptionException e) { + return e instanceof AbstractSubscriptionProvider.CommitException + ? ((AbstractSubscriptionProvider.CommitException) e).getFailure() + : e; + } + + private AbstractSubscriptionProvider.CommitResult getCompletedCommitResult( + final SubscriptionException e) { + return e instanceof AbstractSubscriptionProvider.CommitException + ? ((AbstractSubscriptionProvider.CommitException) e).getCompletedResult() + : AbstractSubscriptionProvider.CommitResult.empty(); + } + /** Invoked after the server accepts commit contexts from an acknowledgement request. */ protected void onAckedCommitContexts( final Collection acceptedCommitContexts) { @@ -1610,10 +1699,26 @@ protected Set ackCommitContextsWithPartialProgress( commitInternal(entry.getKey(), groupedCommitContexts, false); overlayCommittedPositions(commitResp.getCommittedProgressByTopic()); removableCommitContexts.addAll(commitResp.getAcceptedCommitContexts()); - } catch (final SubscriptionConnectionException e) { + } catch (final SubscriptionException e) { + final SubscriptionException failure = getCommitFailure(e); + final AbstractSubscriptionProvider.CommitResult completedResult = + getCompletedCommitResult(e); + overlayCommittedPositions(completedResult.getCommittedProgressByTopic()); + final Set acceptedBeforeFailure = + new HashSet<>(completedResult.getAcceptedCommitContexts()); + removableCommitContexts.addAll(acceptedBeforeFailure); + if (!(failure instanceof SubscriptionConnectionException)) { + // The caller cannot receive the removable set when an error propagates. Remove confirmed + // contexts from every completed group before leaving the automatic/close-time ACK path. + onAckedCommitContexts(removableCommitContexts); + throw failure; + } int stagedCount = 0; int retainedCount = 0; for (final SubscriptionCommitContext commitContext : groupedCommitContexts) { + if (acceptedBeforeFailure.contains(commitContext)) { + continue; + } if (isConsensusCommitContext(commitContext)) { stagePendingRedirectAck(commitContext); removableCommitContexts.add(commitContext); @@ -1637,7 +1742,7 @@ protected Set ackCommitContextsWithPartialProgress( this, retainedCount, entry.getKey(), - e); + failure); } } } @@ -1676,6 +1781,7 @@ private AbstractSubscriptionProvider.CommitResult commitInternal( final List subscriptionCommitContexts, final boolean nack) throws SubscriptionException { + checkIfFenced(); providers.acquireReadLock(); try { final AbstractSubscriptionProvider provider = providers.getProvider(dataNodeId); @@ -1694,11 +1800,13 @@ private AbstractSubscriptionProvider.CommitResult commitInternal( } try { return provider.commit(subscriptionCommitContexts, nack); - } catch (final SubscriptionConsumerFencedException e) { - fence(e); - throw e; - } catch (final SubscriptionConnectionException e) { - providers.syncIfUnavailable(this); + } catch (final SubscriptionException e) { + final SubscriptionException failure = getCommitFailure(e); + if (failure instanceof SubscriptionConsumerFencedException) { + fence((SubscriptionConsumerFencedException) failure); + } else if (failure instanceof SubscriptionConnectionException) { + providers.syncIfUnavailable(this); + } throw e; } } finally { @@ -2090,15 +2198,11 @@ private void flushPendingRedirectAcks(final List currentMes } final List contextsToRedirect = new ArrayList<>(pendingContexts); + AbstractSubscriptionProvider.CommitResult commitResp; try { - final AbstractSubscriptionProvider.CommitResult commitResp = - commitInternal(entry.getValue(), contextsToRedirect, false); - overlayCommittedPositions(commitResp.getCommittedProgressByTopic()); - commitResp.getAcceptedCommitContexts().forEach(pendingContexts::remove); - if (pendingContexts.isEmpty()) { - pendingRedirectAcksByTopicRegion.remove(entry.getKey(), pendingContexts); - } + commitResp = commitInternal(entry.getValue(), contextsToRedirect, false); } catch (final SubscriptionException e) { + commitResp = getCompletedCommitResult(e); LOGGER.warn( SubscriptionMessages .LOG_ARG_FAILED_REDIRECT_ARG_PENDING_CONSENSUS_ACK_S_ARG_VIA_E6A10AC5, @@ -2108,6 +2212,11 @@ private void flushPendingRedirectAcks(final List currentMes entry.getValue(), e); } + overlayCommittedPositions(commitResp.getCommittedProgressByTopic()); + commitResp.getAcceptedCommitContexts().forEach(pendingContexts::remove); + if (pendingContexts.isEmpty()) { + pendingRedirectAcksByTopicRegion.remove(entry.getKey(), pendingContexts); + } } } @@ -2162,7 +2271,24 @@ private void overlayCommittedPositions(final Map progress if (Objects.isNull(progressByTopic) || progressByTopic.isEmpty()) { return; } - progressByTopic.forEach(this::overlayCommittedPositions); + // ACK responses from different providers can arrive out of order, including completed batches + // carried by a failure. Preserve the newest position for each writer. Explicit seek operations + // use the topic-scoped overlay above and can still move a checkpoint backward. + for (final Entry entry : progressByTopic.entrySet()) { + if (Objects.isNull(entry.getValue())) { + continue; + } + for (final RegionProgress regionProgress : entry.getValue().getRegionProgress().values()) { + if (Objects.nonNull(regionProgress)) { + regionProgress + .getWriterPositions() + .forEach( + (writerId, writerProgress) -> + mergeTopicProgress( + committedPositionsByTopic, entry.getKey(), writerId, writerProgress)); + } + } + } } private void overlayTopicProgress( diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProvider.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProvider.java index 56f7324910318..125005763dc53 100644 --- a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProvider.java +++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProvider.java @@ -804,20 +804,32 @@ CommitResult commit( final List acceptedCommitContexts = new ArrayList<>(); final Map committedProgressByTopic = new LinkedHashMap<>(); + int completedCommitContextCount = 0; rpcLock.lock(); try { for (final List batch : batches) { final CommitResult result = commitOnce(batch, nack); + completedCommitContextCount += batch.size(); acceptedCommitContexts.addAll(result.getAcceptedCommitContexts()); mergeTopicProgress(committedProgressByTopic, result.getCommittedProgressByTopic()); } + } catch (final SubscriptionException e) { + if (nack || completedCommitContextCount == 0) { + throw e; + } + // Preserve every completed response even if a later batch fails with a server error. The + // consumer applies these results before propagating the original failure or retrying. + throw new CommitException( + e, + completedCommitContextCount, + new CommitResult(acceptedCommitContexts, committedProgressByTopic)); } finally { rpcLock.unlock(); } return new CommitResult(acceptedCommitContexts, committedProgressByTopic); } - private CommitResult commitOnce( + CommitResult commitOnce( final List subscriptionCommitContexts, final boolean nack) throws SubscriptionException { final PipeSubscribeCommitReq req; @@ -898,6 +910,32 @@ interface PipeSubscribeRequestSender { TPipeSubscribeResp send(TPipeSubscribeReq req) throws TException; } + static final class CommitException extends SubscriptionException { + private final int completedCommitContextCount; + private final CommitResult completedResult; + + private CommitException( + final SubscriptionException cause, + final int completedCommitContextCount, + final CommitResult completedResult) { + super(cause.getMessage(), cause); + this.completedCommitContextCount = completedCommitContextCount; + this.completedResult = completedResult; + } + + int getCompletedCommitContextCount() { + return completedCommitContextCount; + } + + CommitResult getCompletedResult() { + return completedResult; + } + + SubscriptionException getFailure() { + return (SubscriptionException) getCause(); + } + } + static final class CommitResult { private static final CommitResult EMPTY = new CommitResult(Collections.emptyList(), Collections.emptyMap()); diff --git a/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionConsumerLifecycleTest.java b/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionConsumerLifecycleTest.java index 51f35c5d8205f..7a05318acf80e 100644 --- a/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionConsumerLifecycleTest.java +++ b/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionConsumerLifecycleTest.java @@ -436,7 +436,7 @@ public void testPartialCommitKeepsRejectedContextsInAutoCommitBuffer() throws Ex Assert.assertEquals(1, uncommittedCommitContexts.size()); Assert.assertEquals( Collections.singleton(rejectedCommitContext), uncommittedCommitContexts.get(0L)); - Assert.assertEquals(1, consumer.commitRequestCount); + Assert.assertEquals(4, consumer.commitRequestCount); } finally { consumer.close(); } diff --git a/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionConsumerMultiProviderPollTest.java b/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionConsumerMultiProviderPollTest.java deleted file mode 100644 index 34ef7a8fc4840..0000000000000 --- a/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionConsumerMultiProviderPollTest.java +++ /dev/null @@ -1,281 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.session.subscription.consumer.base; - -import org.apache.iotdb.common.rpc.thrift.TEndPoint; -import org.apache.iotdb.rpc.subscription.config.TopicConfig; -import org.apache.iotdb.rpc.subscription.exception.SubscriptionException; -import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionCommitContext; -import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionPollResponse; -import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionPollResponseType; -import org.apache.iotdb.rpc.subscription.payload.poll.TabletsPayload; -import org.apache.iotdb.rpc.subscription.payload.poll.TopicProgress; -import org.apache.iotdb.rpc.subscription.payload.response.PipeSubscribeHeartbeatResp; -import org.apache.iotdb.session.AbstractSessionBuilder; -import org.apache.iotdb.session.subscription.SubscriptionTreeSessionBuilder; -import org.apache.iotdb.session.subscription.payload.SubscriptionMessage; - -import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.write.record.Tablet; -import org.apache.tsfile.write.schema.IMeasurementSchema; -import org.apache.tsfile.write.schema.MeasurementSchema; -import org.junit.Assert; -import org.junit.Test; - -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; - -public class SubscriptionConsumerMultiProviderPollTest { - - private static final String HOST = "127.0.0.1"; - private static final int FIRST_PORT = 10_001; - private static final String TOPIC = "topic1"; - private static final String CONSUMER_ID = "test_consumer"; - private static final String CONSUMER_GROUP_ID = "test_consumer_group"; - private static final long LONG_INTERVAL_MS = 86_400_000L; - - @Test - public void testPollTriesNextProviderBeforeBackingOff() throws SubscriptionException { - final TestPullConsumer consumer = new TestPullConsumer(2, 0); - try { - consumer.open(); - consumer.subscribeTopic(); - - final List messages = consumer.pollForTest(1_000L); - - Assert.assertEquals(1, messages.size()); - Assert.assertEquals(1, consumer.getPollCount(1)); - Assert.assertEquals(1, consumer.getPollCount(2)); - Assert.assertEquals(0, consumer.getPollCount(3)); - Assert.assertEquals(0, consumer.getBackoffCount()); - } finally { - consumer.close(); - } - } - - @Test - public void testPollBacksOffAfterAllProvidersAreEmpty() throws SubscriptionException { - final TestPullConsumer consumer = new TestPullConsumer(1, 1); - try { - consumer.open(); - consumer.subscribeTopic(); - - final List messages = consumer.pollForTest(1_000L); - - Assert.assertEquals(1, messages.size()); - Assert.assertEquals(2, consumer.getPollCount(1)); - Assert.assertEquals(1, consumer.getPollCount(2)); - Assert.assertEquals(1, consumer.getPollCount(3)); - Assert.assertEquals(1, consumer.getBackoffCount()); - } finally { - consumer.close(); - } - } - - private static class TestPullConsumer extends AbstractSubscriptionPullConsumer { - - private final Map pollCounts = new HashMap<>(); - private final int dataProviderId; - private final int emptyPollsBeforeData; - private int backoffCount; - - private TestPullConsumer(final int dataProviderId, final int emptyPollsBeforeData) { - super( - new AbstractSubscriptionPullConsumerBuilder() - .host(HOST) - .port(FIRST_PORT) - .consumerId(CONSUMER_ID) - .consumerGroupId(CONSUMER_GROUP_ID) - .heartbeatIntervalMs(LONG_INTERVAL_MS) - .endpointsSyncIntervalMs(LONG_INTERVAL_MS) - .autoCommit(false)); - this.dataProviderId = dataProviderId; - this.emptyPollsBeforeData = emptyPollsBeforeData; - } - - @Override - protected AbstractSubscriptionProvider constructSubscriptionProvider( - final TEndPoint endPoint, - final String username, - final String password, - final String encryptedPassword, - final String consumerId, - final String consumerGroupId, - final String ownerId, - final Long ownerEpoch, - final int thriftMaxFrameSize, - final long heartbeatIntervalMs, - final int connectionTimeoutInMs) { - return new TestSubscriptionProvider( - endPoint, - username, - password, - encryptedPassword, - consumerId, - consumerGroupId, - ownerId, - ownerEpoch, - thriftMaxFrameSize, - heartbeatIntervalMs, - connectionTimeoutInMs, - pollCounts, - dataProviderId, - emptyPollsBeforeData); - } - - private void subscribeTopic() { - subscribedTopics = Collections.singletonMap(TOPIC, new TopicConfig()); - } - - private List pollForTest(final long timeoutMs) - throws SubscriptionException { - return poll(timeoutMs); - } - - private int getPollCount(final int dataNodeId) { - return pollCounts.getOrDefault(dataNodeId, 0); - } - - private int getBackoffCount() { - return backoffCount; - } - - @Override - void sleepAfterEmptyPollRound() { - backoffCount++; - } - } - - private static class TestSubscriptionProvider extends AbstractSubscriptionProvider { - - private final int dataNodeId; - private final Map pollCounts; - private final int dataProviderId; - private final int emptyPollsBeforeData; - - private TestSubscriptionProvider( - final TEndPoint endPoint, - final String username, - final String password, - final String encryptedPassword, - final String consumerId, - final String consumerGroupId, - final String ownerId, - final Long ownerEpoch, - final int thriftMaxFrameSize, - final long heartbeatIntervalMs, - final int connectionTimeoutInMs, - final Map pollCounts, - final int dataProviderId, - final int emptyPollsBeforeData) { - super( - endPoint, - username, - password, - encryptedPassword, - consumerId, - consumerGroupId, - ownerId, - ownerEpoch, - thriftMaxFrameSize, - heartbeatIntervalMs, - connectionTimeoutInMs); - this.dataNodeId = endPoint.port - FIRST_PORT + 1; - this.pollCounts = pollCounts; - this.dataProviderId = dataProviderId; - this.emptyPollsBeforeData = emptyPollsBeforeData; - } - - @Override - protected AbstractSessionBuilder constructSubscriptionSessionBuilder( - final String host, - final int port, - final String username, - final String password, - final String encryptedPassword, - final int thriftMaxFrameSize, - final int connectionTimeoutInMs) { - final boolean useEncryptedPassword = Objects.nonNull(encryptedPassword); - return new SubscriptionTreeSessionBuilder() - .host(host) - .port(port) - .username(username) - .password(useEncryptedPassword ? encryptedPassword : password) - .useEncryptedPassword(useEncryptedPassword) - .thriftMaxFrameSize(thriftMaxFrameSize) - .connectionTimeoutInMs(connectionTimeoutInMs); - } - - @Override - synchronized void handshake() { - setAvailable(); - } - - @Override - synchronized void close() { - setUnavailable(); - } - - @Override - int getDataNodeId() { - return dataNodeId; - } - - @Override - PipeSubscribeHeartbeatResp heartbeat( - final List processorBufferedCommitContexts) { - final PipeSubscribeHeartbeatResp response = new PipeSubscribeHeartbeatResp(); - response.getTopics().put(TOPIC, new TopicConfig()); - response.getEndPoints().put(1, new TEndPoint(HOST, FIRST_PORT)); - response.getEndPoints().put(2, new TEndPoint(HOST, FIRST_PORT + 1)); - response.getEndPoints().put(3, new TEndPoint(HOST, FIRST_PORT + 2)); - return response; - } - - @Override - List poll( - final Set topicNames, - final long timeoutMs, - final Map progressByTopic) - throws SubscriptionException { - pollCounts.merge(dataNodeId, 1, Integer::sum); - if (dataNodeId != dataProviderId || pollCounts.get(dataNodeId) <= emptyPollsBeforeData) { - return Collections.emptyList(); - } - final SubscriptionCommitContext commitContext = - new SubscriptionCommitContext(dataNodeId, 0, TOPIC, CONSUMER_GROUP_ID, 0L); - final List schemas = - Collections.singletonList(new MeasurementSchema("s1", TSDataType.INT64)); - final Tablet tablet = new Tablet("root.sg.d1", schemas, 1); - tablet.setTimestamps(new long[] {1L}); - ((long[]) tablet.getValues()[0])[0] = 1L; - tablet.setRowSize(1); - return Collections.singletonList( - new SubscriptionPollResponse( - SubscriptionPollResponseType.TABLETS.getType(), - new TabletsPayload(Collections.singletonList(tablet), -1), - commitContext)); - } - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java index 32c3eb9ed1158..11c5d14aaae5a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java @@ -855,13 +855,7 @@ public synchronized void resize( if (force) { throw new PipeRuntimeOutOfMemoryCriticalException( - String.format( - DataNodePipeMessages - .PIPE_EXCEPTION_FORCERESIZE_FAILED_TO_ALLOCATE_MEMORY_AFTER_D_RETRIES_TOTAL_8C6948BC, - memoryAllocateMaxRetries, - getTotalNonFloatingMemorySizeInBytes(), - memoryBlock.getUsedMemoryInBytes(), - sizeInBytes)); + getResizeFailureMessage(memoryAllocateMaxRetries, sizeInBytes)); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java index 3601f433c25af..1c0dd008dfe65 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java @@ -1475,7 +1475,8 @@ private void updateMemoryInfo( break; } // The asynchronous submitter needs the region write lock held by this insertion. - // Submit other already-selected MemTables here so they can release memory while we wait. + // Submit other already-selected MemTables here so they can release memory while we + // wait. submitOtherSelectedMemTablesToFlush(); try { TimeUnit.MILLISECONDS.sleep(config.getCheckPeriodWhenInsertBlocked()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/ConsensusSubscriptionBroker.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/ConsensusSubscriptionBroker.java index 8bb3b3ad084ed..2d9d7a58c02ef 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/ConsensusSubscriptionBroker.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/ConsensusSubscriptionBroker.java @@ -792,7 +792,7 @@ public void rollbackConsensusPrefetchingQueues( topicOwnershipSnapshots.remove(topicName); } } - queuesToClose.forEach(queue -> queue.closeAsync(false, false)); + queuesToClose.forEach(queue -> queue.closeAsync(false)); } public void wakeUpPrefetchingQueues(final String topicName, final boolean isTableModel) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManagerResizeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManagerResizeTest.java index 298647b8eb347..6de1776e8be95 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManagerResizeTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManagerResizeTest.java @@ -259,7 +259,9 @@ public void testTsFileParserAllocationAndResizeShareProtectedCombinedLimit() { Assert.assertThrows( PipeRuntimeOutOfMemoryCriticalException.class, () -> manager.forceAllocateForTsFileParserWithRetry(40)); - Assert.assertTrue(allocationException.getMessage().contains("usedTsFiles=1100 bytes")); + Assert.assertTrue( + allocationException.getMessage(), + allocationException.getMessage().contains("usedTsFiles=1100 bytes")); Assert.assertTrue(allocationException.getMessage().contains("tabletLimit=900 bytes")); Assert.assertTrue( allocationException.getMessage().contains("tabletAndTsFileLimit=1200 bytes")); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java index 70810de95faf3..d652b831b1dd1 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java @@ -67,6 +67,7 @@ import org.apache.iotdb.db.storageengine.dataregion.compaction.utils.CompactionConfigRestorer; import org.apache.iotdb.db.storageengine.dataregion.flush.FlushManager; import org.apache.iotdb.db.storageengine.dataregion.flush.TsFileFlushPolicy; +import org.apache.iotdb.db.storageengine.dataregion.memtable.IMemTable; import org.apache.iotdb.db.storageengine.dataregion.memtable.ReadOnlyMemChunk; import org.apache.iotdb.db.storageengine.dataregion.memtable.TsFileProcessor; import org.apache.iotdb.db.storageengine.dataregion.modification.DeletionPredicate; @@ -79,7 +80,6 @@ import org.apache.iotdb.db.storageengine.dataregion.tsfile.generator.TsFileNameGenerator; import org.apache.iotdb.db.storageengine.rescon.memory.MemTableManager; import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo; -import org.apache.iotdb.db.storageengine.rescon.memory.WriteMemoryBlock; import org.apache.iotdb.db.utils.EnvironmentUtils; import org.apache.iotdb.db.utils.constant.TestConstant; import org.apache.iotdb.rpc.RpcUtils; @@ -241,6 +241,8 @@ private void assertRejectedInsertionFlushesOnlySelectedOtherMemtable(boolean tab final boolean originalSeparateData = config.isEnableSeparateData(); final long partitionInterval = TimePartitionUtils.getTimePartitionInterval(); try { + systemInfo.close(); + systemInfo.loadWriteMemory(); config.setEnableSeparateData(true); for (int i = 1; i <= 384; i++) { insertMemoryAdmissionTestRow(i, i); @@ -275,8 +277,7 @@ private void assertRejectedInsertionFlushesOnlySelectedOtherMemtable(boolean tab final DataRegionInfo regionInfo = (DataRegionInfo) regionInfoField.get(dataRegion); final long memoryBefore = regionInfo.getMemCost(); setMemoryAdmissionTestThreshold("memorySizeForMemtable", 2 * memoryBefore); - setMemoryAdmissionTestThreshold( - "FLUSH_THRESHOLD", (double) memoryBefore - selected.getWorkMemTableRamCost() / 2); + setMemoryAdmissionTestThreshold("FLUSH_THRESHOLD", (double) memoryBefore - 1); setMemoryAdmissionTestThreshold("REJECT_THRESHOLD", (double) memoryBefore - 1); config.setMaxWaitingTimeWhenInsertBlocked(5_000); config.setCheckPeriodWhenInsertBlocked(10); @@ -309,9 +310,6 @@ private void assertRejectedInsertionFlushesOnlySelectedOtherMemtable(boolean tab Assert.assertFalse(currentMemTable.shouldFlush()); Assert.assertFalse(unselectedMemTable.shouldFlush()); Assert.assertEquals(129, currentMemTable.getTotalPointsNum()); - for (WriteMemoryBlock block : regionInfo.getWriteMemoryManager().getMemoryBlocks()) { - Assert.assertEquals(0, block.getReservedMemoryInBytes()); - } Assert.assertFalse(systemInfo.isRejected()); } finally { dataRegion.writeUnlock(); @@ -321,7 +319,7 @@ private void assertRejectedInsertionFlushesOnlySelectedOtherMemtable(boolean tab insertMemoryAdmissionTestRow(partitionInterval + 130, 130); Assert.assertSame(currentMemTable, current.getWorkMemTable()); Assert.assertEquals(3, dataRegion.getTsFileManager().getTsFileList(true).size()); - dataRegion.syncCloseAllWorkingAndClosingTsFileProcessors(); + dataRegion.syncCloseAllWorkingTsFileProcessors(); int points = 0; for (TsFileResource resource : dataRegion.getTsFileManager().getTsFileList(true)) { try (TsFileSequenceReader reader = new TsFileSequenceReader(resource.getTsFilePath()); @@ -340,6 +338,7 @@ private void assertRejectedInsertionFlushesOnlySelectedOtherMemtable(boolean tab } Assert.assertEquals(515, points); } finally { + systemInfo.close(); systemInfo.loadWriteMemory(); config.setMaxWaitingTimeWhenInsertBlocked(originalWait); config.setCheckPeriodWhenInsertBlocked(originalPeriod); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/memory/SystemInfoTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/memory/SystemInfoTest.java index d330af1777e3a..8c5e3e362d56c 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/memory/SystemInfoTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/memory/SystemInfoTest.java @@ -86,7 +86,7 @@ public void tearDown() throws Exception { @Test public void testRejectedWriterCannotFlushSmallerCurrentProcessor() throws Exception { - dataRegionInfo.getWriteMemoryManager().addMemory(900); + dataRegionInfo.addStorageGroupMemCost(900); Assert.assertFalse(systemInfo.reportStorageGroupStatus(dataRegionInfo, currentProcessor)); Assert.assertTrue(systemInfo.isRejected()); @@ -97,7 +97,7 @@ public void testRejectedWriterCannotFlushSmallerCurrentProcessor() throws Except @Test public void testRejectedWriterFirstTabletCannotBypassSelection() throws Exception { when(currentProcessor.getWorkMemTableRamCost()).thenReturn(0L); - dataRegionInfo.getWriteMemoryManager().addMemory(900); + dataRegionInfo.addStorageGroupMemCost(900); Assert.assertFalse(systemInfo.reportStorageGroupStatus(dataRegionInfo, currentProcessor)); verify(currentProcessor, never()).setWorkMemTableShouldFlush(); @@ -106,7 +106,7 @@ public void testRejectedWriterFirstTabletCannotBypassSelection() throws Exceptio @Test public void testRepeatedReportsSubmitEachMemtableOnce() throws Exception { - dataRegionInfo.getWriteMemoryManager().addMemory(900); + dataRegionInfo.addStorageGroupMemCost(900); for (int i = 0; i < 10; i++) { Assert.assertFalse(systemInfo.reportStorageGroupStatus(dataRegionInfo, currentProcessor)); @@ -120,7 +120,7 @@ public void testRepeatedReportsSubmitEachMemtableOnce() throws Exception { @Test public void testRepeatedReportsForFirstTabletSubmitOnce() throws Exception { when(currentProcessor.getWorkMemTableRamCost()).thenReturn(0L); - dataRegionInfo.getWriteMemoryManager().addMemory(900); + dataRegionInfo.addStorageGroupMemCost(900); for (int i = 0; i < 10; i++) { Assert.assertFalse(systemInfo.reportStorageGroupStatus(dataRegionInfo, currentProcessor)); @@ -134,7 +134,7 @@ public void testRepeatedReportsForFirstTabletSubmitOnce() throws Exception { public void testNewMemtableCanBeScheduledAfterPreviousFlush() throws Exception { when(currentProcessor.getWorkMemTableRamCost()).thenReturn(700L); when(largerProcessor.getWorkMemTableRamCost()).thenReturn(100L); - dataRegionInfo.getWriteMemoryManager().addMemory(900); + dataRegionInfo.addStorageGroupMemCost(900); Assert.assertTrue(systemInfo.reportStorageGroupStatus(dataRegionInfo, currentProcessor)); mockWorkMemTable(currentProcessor); @@ -148,9 +148,9 @@ public void testNewMemtableCanBeScheduledAfterPreviousFlush() throws Exception { public void testMarkedCurrentProcessorCannotReachHardMemoryLimit() throws Exception { when(currentProcessor.getWorkMemTableRamCost()).thenReturn(700L); when(largerProcessor.getWorkMemTableRamCost()).thenReturn(100L); - dataRegionInfo.getWriteMemoryManager().addMemory(900); + dataRegionInfo.addStorageGroupMemCost(900); Assert.assertTrue(systemInfo.reportStorageGroupStatus(dataRegionInfo, currentProcessor)); - dataRegionInfo.getWriteMemoryManager().addMemory(100); + dataRegionInfo.addStorageGroupMemCost(100); try { systemInfo.reportStorageGroupStatus(dataRegionInfo, currentProcessor); @@ -166,7 +166,7 @@ public void testMarkedCurrentProcessorCannotReachHardMemoryLimit() throws Except public void testNoCurrentMemtableCannotBypassRejection() throws Exception { when(currentProcessor.getWorkMemTable()).thenReturn(null); when(currentProcessor.getWorkMemTableRamCost()).thenReturn(0L); - dataRegionInfo.getWriteMemoryManager().addMemory(900); + dataRegionInfo.addStorageGroupMemCost(900); Assert.assertFalse(systemInfo.reportStorageGroupStatus(dataRegionInfo, currentProcessor)); Assert.assertTrue(systemInfo.isRejected()); @@ -176,10 +176,10 @@ public void testNoCurrentMemtableCannotBypassRejection() throws Exception { @Test public void testRejectionClearsAfterMemoryIsReleased() throws Exception { - dataRegionInfo.getWriteMemoryManager().addMemory(900); + dataRegionInfo.addStorageGroupMemCost(900); Assert.assertFalse(systemInfo.reportStorageGroupStatus(dataRegionInfo, currentProcessor)); - dataRegionInfo.getWriteMemoryManager().releaseMemory(500); + dataRegionInfo.releaseStorageGroupMemCost(500); systemInfo.resetStorageGroupStatus(dataRegionInfo); Assert.assertFalse(systemInfo.isRejected()); Assert.assertTrue(systemInfo.reportStorageGroupStatus(dataRegionInfo, currentProcessor)); @@ -190,7 +190,7 @@ public void testRejectionClearsAfterMemoryIsReleased() throws Exception { public void testCurrentProcessorCannotExceedHardMemoryLimit() throws Exception { when(currentProcessor.getWorkMemTableRamCost()).thenReturn(700L); when(largerProcessor.getWorkMemTableRamCost()).thenReturn(100L); - dataRegionInfo.getWriteMemoryManager().addMemory(1100); + dataRegionInfo.addStorageGroupMemCost(1100); try { systemInfo.reportStorageGroupStatus(dataRegionInfo, currentProcessor); @@ -202,7 +202,7 @@ public void testCurrentProcessorCannotExceedHardMemoryLimit() throws Exception { @Test public void testNormalFlushStillSelectsLargestMemtable() throws Exception { - dataRegionInfo.getWriteMemoryManager().addMemory(700); + dataRegionInfo.addStorageGroupMemCost(700); Assert.assertTrue(systemInfo.reportStorageGroupStatus(dataRegionInfo, currentProcessor)); Assert.assertFalse(systemInfo.isRejected());