diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/consensus/local/IoTDBConsensusSubscriptionBrokerRecoveryIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/consensus/local/IoTDBConsensusSubscriptionBrokerRecoveryIT.java new file mode 100644 index 0000000000000..b46d699072244 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/consensus/local/IoTDBConsensusSubscriptionBrokerRecoveryIT.java @@ -0,0 +1,428 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.consensus.local; + +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.commons.client.ClientPoolFactory; +import org.apache.iotdb.commons.client.IClientManager; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.commons.client.sync.SyncDataNodeInternalServiceClient; +import org.apache.iotdb.commons.consensus.ConsensusGroupId; +import org.apache.iotdb.commons.subscription.meta.consumer.ConsumerGroupMeta; +import org.apache.iotdb.confignode.rpc.thrift.TGetAllSubscriptionInfoResp; +import org.apache.iotdb.confignode.rpc.thrift.TGetCommitProgressReq; +import org.apache.iotdb.confignode.rpc.thrift.TGetCommitProgressResp; +import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.env.cluster.config.MppBaseConfig; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.ClusterIT; +import org.apache.iotdb.mpp.rpc.thrift.TPushConsumerGroupMetaResp; +import org.apache.iotdb.mpp.rpc.thrift.TPushSingleConsumerGroupMetaReq; +import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.iotdb.rpc.subscription.config.ConsumerConfig; +import org.apache.iotdb.rpc.subscription.config.ConsumerConstant; +import org.apache.iotdb.rpc.subscription.payload.poll.PollPayload; +import org.apache.iotdb.rpc.subscription.payload.poll.RegionProgress; +import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionPollRequest; +import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionPollRequestType; +import org.apache.iotdb.rpc.subscription.payload.poll.TopicProgress; +import org.apache.iotdb.rpc.subscription.payload.request.PipeSubscribeHandshakeReq; +import org.apache.iotdb.rpc.subscription.payload.request.PipeSubscribePollReq; +import org.apache.iotdb.rpc.subscription.payload.request.SubscriptionHeartbeatReq; +import org.apache.iotdb.rpc.subscription.payload.response.PipeSubscribeHeartbeatResp; +import org.apache.iotdb.service.rpc.thrift.TPipeSubscribeResp; +import org.apache.iotdb.session.subscription.SubscriptionSessionWrapper; +import org.apache.iotdb.session.subscription.SubscriptionTreeSession; +import org.apache.iotdb.session.subscription.consumer.tree.SubscriptionTreePullConsumer; +import org.apache.iotdb.subscription.it.AbstractSubscriptionIT; + +import org.awaitility.Awaitility; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.lang.reflect.Field; +import java.nio.ByteBuffer; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +@RunWith(IoTDBTestRunner.class) +@Category({ClusterIT.class}) +public class IoTDBConsensusSubscriptionBrokerRecoveryIT extends AbstractSubscriptionIT { + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS) + .setSchemaReplicationFactor(1) + .setDataReplicationFactor(3) + .setAutoCreateSchemaEnabled(true) + .setPipeMemoryManagementEnabled(false) + .setIsPipeEnableMemoryCheck(false); + // Recovery is driven explicitly below; a periodic metadata push must not mask a broken + // handshake. + final Properties config = new Properties(); + config.setProperty("subscription_enabled", "true"); + ((MppBaseConfig) EnvFactory.getEnv().getConfig().getDataNodeCommonConfig()) + .updateProperties(config); + config.setProperty("subscription_meta_syncer_initial_sync_delay_minutes", "60"); + config.setProperty("subscription_meta_syncer_sync_interval_minutes", "60"); + ((MppBaseConfig) EnvFactory.getEnv().getConfig().getConfigNodeCommonConfig()) + .updateProperties(config); + EnvFactory.getEnv().initClusterEnvironment(1, 3); + } + + @Override + @After + public void tearDown() throws Exception { + try { + EnvFactory.getEnv().cleanClusterEnvironment(); + } finally { + super.tearDown(); + } + } + + @Test + public void testHandshakeRepairsMissingBrokersWithoutResubscribing() throws Exception { + testMissingBrokerRecovery(true); + } + + @Test + public void testMetadataPushRepairsMissingBrokersWithoutRestoringCanceledTopics() + throws Exception { + testMissingBrokerRecovery(false); + } + + private void testMissingBrokerRecovery(final boolean repairByHandshake) throws Exception { + final ConsensusSubscriptionITSupport.TestIdentifiers ids = + ConsensusSubscriptionITSupport.newIdentifiers( + repairByHandshake ? "broker_handshake" : "broker_metadata"); + final String canceledTopic = ids.getTopic() + "_canceled"; + final List probes = new ArrayList<>(); + SubscriptionTreePullConsumer consumer = null; + final IClientManager clients = + new IClientManager.Factory() + .createClientManager( + new ClientPoolFactory.SyncDataNodeInternalServiceClientPoolFactory()); + try { + ConsensusSubscriptionITSupport.bootstrapDatabase(ids.getDatabase()); + ConsensusSubscriptionITSupport.createConsensusTopic( + ids.getTopic(), ids.getDatabase() + ".**"); + ConsensusSubscriptionITSupport.createConsensusTopic(canceledTopic, ids.getDatabase() + ".**"); + consumer = createConsumer(ids, TimeUnit.HOURS.toMillis(1)); + consumer.subscribe(ids.getTopic()); + final Set committedRows = + ConsensusSubscriptionITSupport.insertRows(ids.getDatabase(), "d0", 100L, 8, true); + assertRowsConsumed(consumer, committedRows); + // Creating a new subscription runs ConfigNode's commit-progress synchronization without + // background metadata pushes, then this topic is left unsubscribed. + consumer.subscribe(canceledTopic); + consumer.unsubscribe(canceledTopic); + awaitConfigNodeCheckpoint(ids, consumer.committedPositions(ids.getTopic())); + final ConsumerGroupMeta authoritativeMeta = getConsumerGroupMeta(ids.getConsumerGroupId()); + Assert.assertEquals( + Collections.singleton(ids.getTopic()), authoritativeMeta.getSubscribedTopicNames()); + final ConsumerConfig consumerConfig = createConsumerConfig(authoritativeMeta, consumer); + + for (final DataNodeWrapper node : EnvFactory.getEnv().getDataNodeWrapperList()) { + final SubscriptionSessionWrapper probe = getProviderSession(consumer, node); + probes.add(probe); + // A replacement-generation push can remove the old broker and then fail while resolving + // topic metadata. Exercise that real RPC failure without changing ConfigNode's metadata. + final ConsumerGroupMeta replacement = + new ConsumerGroupMeta( + ids.getConsumerGroupId(), + authoritativeMeta.getCreationTime() + 1, + authoritativeMeta.getConsumerMeta(ids.getConsumerId())); + replacement.addSubscription( + ids.getConsumerId(), Collections.singleton(ids.getTopic() + "_missing")); + final TPushConsumerGroupMetaResp failure = pushMeta(clients, node, replacement); + Assert.assertEquals( + failure.toString(), + TSStatusCode.CONSUMER_PUSH_META_ERROR.getStatusCode(), + failure.getStatus().getCode()); + Assert.assertTrue(failure.isSetExceptionMessages()); + Assert.assertFalse(failure.getExceptionMessages().isEmpty()); + final TPipeSubscribeResp missingBroker = + probe + .getSessionConnection() + .pipeSubscribe( + PipeSubscribePollReq.toTPipeSubscribeReq( + new SubscriptionPollRequest( + SubscriptionPollRequestType.POLL.getType(), + new PollPayload(Collections.singleton(ids.getTopic())), + 1_000L, + 1_048_576L))); + Assert.assertEquals( + missingBroker.getStatus().toString(), + TSStatusCode.SUBSCRIPTION_POLL_ERROR.getStatusCode(), + missingBroker.getStatus().getCode()); + } + Assert.assertEquals(authoritativeMeta, getConsumerGroupMeta(ids.getConsumerGroupId())); + final Set pendingRows = + ConsensusSubscriptionITSupport.insertRows(ids.getDatabase(), "d0", 200L, 8, true); + + for (int index = 0; index < probes.size(); index++) { + final SubscriptionSessionWrapper probe = probes.get(index); + if (repairByHandshake) { + assertSuccess( + probe + .getSessionConnection() + .pipeSubscribe(PipeSubscribeHandshakeReq.toTPipeSubscribeReq(consumerConfig))); + } else { + final TPushConsumerGroupMetaResp repaired = + pushMeta(clients, EnvFactory.getEnv().getDataNodeWrapper(index), authoritativeMeta); + Assert.assertEquals( + repaired.toString(), + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + repaired.getStatus().getCode()); + } + final TPipeSubscribeResp heartbeat = + probe.getSessionConnection().pipeSubscribe(SubscriptionHeartbeatReq.toThriftReq()); + assertSuccess(heartbeat); + Assert.assertEquals( + Collections.singleton(ids.getTopic()), + PipeSubscribeHeartbeatResp.fromTPipeSubscribeResp(heartbeat).getTopics().keySet()); + } + Assert.assertEquals(authoritativeMeta, getConsumerGroupMeta(ids.getConsumerGroupId())); + assertRowsConsumed(consumer, pendingRows); + assertRowsConsumed( + consumer, + ConsensusSubscriptionITSupport.insertRows(ids.getDatabase(), "d0", 300L, 8, true)); + } finally { + try { + clients.close(); + } finally { + ConsensusSubscriptionITSupport.cleanup(consumer, ids.getTopic(), ids.getDatabase()); + try (final SubscriptionTreeSession session = + new SubscriptionTreeSession( + EnvFactory.getEnv().getIP(), Integer.parseInt(EnvFactory.getEnv().getPort()))) { + session.open(); + session.dropTopicIfExists(canceledTopic); + } + } + } + } + + @Test + public void testProviderReplacementPreservesUncommittedBatch() throws Exception { + final ConsensusSubscriptionITSupport.TestIdentifiers ids = + ConsensusSubscriptionITSupport.newIdentifiers("broker_provider_replacement"); + SubscriptionTreePullConsumer consumer = null; + try { + ConsensusSubscriptionITSupport.bootstrapDatabase(ids.getDatabase()); + ConsensusSubscriptionITSupport.createConsensusTopic( + ids.getTopic(), ids.getDatabase() + ".**"); + consumer = createConsumer(ids, 1_000L); + consumer.subscribe(ids.getTopic()); + final Set rows = + ConsensusSubscriptionITSupport.insertRows(ids.getDatabase(), "d0", 100L, 8, true); + final ConsensusSubscriptionITSupport.PolledMessageBatch batch = + ConsensusSubscriptionITSupport.pollFirstNonEmptyBatchWithoutCommit( + consumer, 60, Duration.ofSeconds(1)); + Assert.assertFalse(batch.getMessages().isEmpty()); + final TopicProgress progressBeforeReplacement = consumer.committedPositions(ids.getTopic()); + final ConsumerGroupMeta metaBeforeReplacement = + getConsumerGroupMeta(ids.getConsumerGroupId()); + final Map providers = getProviders(consumer); + final Map oldProviders = new HashMap<>(providers); + Assert.assertEquals(3, oldProviders.size()); + for (final Object provider : oldProviders.values()) { + ((SubscriptionSessionWrapper) getField(provider, "session")) + .getSessionConnection() + .forceClose(); + } + Awaitility.await() + .pollInSameThread() + .atMost(Duration.ofSeconds(30)) + .untilAsserted( + () -> { + Assert.assertEquals(oldProviders.keySet(), providers.keySet()); + for (final Object nodeId : oldProviders.keySet()) { + Assert.assertNotSame(oldProviders.get(nodeId), providers.get(nodeId)); + } + }); + Assert.assertEquals(metaBeforeReplacement, getConsumerGroupMeta(ids.getConsumerGroupId())); + Assert.assertEquals(progressBeforeReplacement, consumer.committedPositions(ids.getTopic())); + consumer.commitSync(batch.getMessages()); + final ConsensusSubscriptionITSupport.ConsumedRecords consumed = batch.getConsumedRecords(); + consumed.merge( + ConsensusSubscriptionITSupport.pollAndCommitUntilAtLeast( + consumer, rows.size() - consumed.getUniqueRowCount(), 20, Duration.ofSeconds(1))); + ConsensusSubscriptionITSupport.assertExactRowKeys(rows, consumed); + assertRowsConsumed( + consumer, + ConsensusSubscriptionITSupport.insertRows(ids.getDatabase(), "d0", 200L, 8, true)); + } finally { + ConsensusSubscriptionITSupport.cleanup(consumer, ids.getTopic(), ids.getDatabase()); + } + } + + private static SubscriptionTreePullConsumer createConsumer( + final ConsensusSubscriptionITSupport.TestIdentifiers ids, final long controlIntervalMs) + throws Exception { + final SubscriptionTreePullConsumer consumer = + new SubscriptionTreePullConsumer.Builder() + .host(EnvFactory.getEnv().getIP()) + .port(Integer.parseInt(EnvFactory.getEnv().getPort())) + .consumerId(ids.getConsumerId()) + .consumerGroupId(ids.getConsumerGroupId()) + .heartbeatIntervalMs(controlIntervalMs) + .endpointsSyncIntervalMs(controlIntervalMs) + .autoCommit(false) + .buildPullConsumer(); + consumer.open(); + return consumer; + } + + private static ConsumerConfig createConsumerConfig( + final ConsumerGroupMeta meta, final SubscriptionTreePullConsumer consumer) throws Exception { + final Map attributes = + new HashMap<>(meta.getConsumerMeta(consumer.getConsumerId()).getConfig().getAttribute()); + attributes.put( + ConsumerConstant.CONSUMER_INSTANCE_ID_KEY, + (String) getField(consumer, "consumerInstanceId")); + return new ConsumerConfig(attributes); + } + + private static SubscriptionSessionWrapper getProviderSession( + final SubscriptionTreePullConsumer consumer, final DataNodeWrapper node) throws Exception { + final TEndPoint endpoint = new TEndPoint(node.getIp(), node.getPort()); + for (final Object provider : getProviders(consumer).values()) { + if (endpoint.equals(getField(provider, "endPoint"))) { + return (SubscriptionSessionWrapper) getField(provider, "session"); + } + } + throw new AssertionError("Missing subscription provider for " + endpoint); + } + + private static ConsumerGroupMeta getConsumerGroupMeta(final String consumerGroupId) + throws Exception { + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) { + final TGetAllSubscriptionInfoResp resp = client.getAllSubscriptionInfo(); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), resp.getStatus().getCode()); + for (final ByteBuffer buffer : resp.getAllSubscriptionInfo()) { + final ConsumerGroupMeta meta = ConsumerGroupMeta.deserialize(buffer); + if (consumerGroupId.equals(meta.getConsumerGroupId())) { + return meta; + } + } + throw new AssertionError("Missing consumer group " + consumerGroupId); + } + } + + private static TPushConsumerGroupMetaResp pushMeta( + final IClientManager clients, + final DataNodeWrapper node, + final ConsumerGroupMeta meta) + throws Exception { + try (final SyncDataNodeInternalServiceClient client = + clients.borrowClient(new TEndPoint(node.getInternalAddress(), node.getInternalPort()))) { + return client.pushSingleConsumerGroupMeta( + new TPushSingleConsumerGroupMetaReq().setConsumerGroupMeta(meta.serialize())); + } + } + + private static void awaitConfigNodeCheckpoint( + final ConsensusSubscriptionITSupport.TestIdentifiers ids, final TopicProgress checkpoint) { + Assert.assertNotNull(checkpoint); + Assert.assertFalse(checkpoint.getRegionProgress().isEmpty()); + Awaitility.await() + .pollInSameThread() + .atMost(Duration.ofSeconds(30)) + .untilAsserted( + () -> { + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) + EnvFactory.getEnv().getLeaderConfigNodeConnection()) { + for (final Map.Entry entry : + checkpoint.getRegionProgress().entrySet()) { + final TGetCommitProgressResp resp = + client.getCommitProgress( + new TGetCommitProgressReq( + ids.getConsumerGroupId(), + ids.getTopic(), + ConsensusGroupId.Factory.createFromString(entry.getKey()).getId(), + 0)); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), resp.getStatus().getCode()); + Assert.assertTrue(resp.isSetCommittedRegionProgress()); + Assert.assertEquals( + entry.getValue(), + RegionProgress.deserialize( + ByteBuffer.wrap(resp.getCommittedRegionProgress()))); + } + } + }); + } + + private static void assertRowsConsumed( + final SubscriptionTreePullConsumer consumer, final Set expectedRows) + throws Exception { + ConsensusSubscriptionITSupport.assertExactRowKeys( + expectedRows, + ConsensusSubscriptionITSupport.pollAndCommitUntilAtLeast( + consumer, expectedRows.size(), 60, Duration.ofSeconds(1))); + } + + private static void assertSuccess(final TPipeSubscribeResp resp) { + Assert.assertEquals( + resp.getStatus().toString(), + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + resp.getStatus().getCode()); + } + + private static Map getProviders(final SubscriptionTreePullConsumer consumer) + throws Exception { + return (Map) getField(getField(consumer, "providers"), "subscriptionProviders"); + } + + private static Object getField(final Object target, final String name) throws Exception { + for (Class type = target.getClass(); type != null; type = type.getSuperclass()) { + try { + final Field field = type.getDeclaredField(name); + field.setAccessible(true); + return field.get(target); + } catch (final NoSuchFieldException ignored) { + // Client state is declared in the consumer/provider base classes. + } + } + throw new NoSuchFieldException(name); + } +} diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumer.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumer.java index 061fd948de9b6..391c4d56e2e2c 100644 --- a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumer.java +++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumer.java @@ -110,6 +110,9 @@ abstract class AbstractSubscriptionConsumer implements AutoCloseable { private static final long SLEEP_MS = 100L; private static final long SLEEP_DELTA_MS = 50L; private static final long TIMER_DELTA_MS = 250L; + private static final int MAX_COMMIT_RETRIES = 3; + private static final long COMMIT_RETRY_INTERVAL_MS = + ConsumerConstant.ENDPOINTS_SYNC_INTERVAL_MS_MIN_VALUE; private static final AtomicLong LAST_CONSUMER_INSTANCE_EPOCH = new AtomicLong(); private final String username; @@ -1060,9 +1063,7 @@ private Optional pollFile( inFlightFilesCommitContextSet.remove(commitContext); } // construct temporary message to nack - nack( - Collections.singletonList( - new SubscriptionMessage(commitContext, file.getAbsolutePath(), null))); + nackAfterPollFailure(new SubscriptionMessage(commitContext, file.getAbsolutePath(), null), e); throw new SubscriptionRuntimeNonCriticalException(e.getMessage(), e); } } @@ -1271,13 +1272,25 @@ private Optional pollTablets( throw (SubscriptionConsumerFencedException) e; } // construct temporary message to nack - nack( - Collections.singletonList( - new SubscriptionMessage(response.getCommitContext(), Collections.emptyMap()))); + nackAfterPollFailure( + new SubscriptionMessage(response.getCommitContext(), Collections.emptyMap()), e); throw new SubscriptionRuntimeNonCriticalException(e.getMessage(), e); } } + private void nackAfterPollFailure(final SubscriptionMessage message, final Exception pollFailure) + throws SubscriptionException { + try { + nack(Collections.singletonList(message)); + } catch (final SubscriptionConnectionException nackFailure) { + // Recovery was scheduled by commitInternal. A NACK replay can increment the server's poison + // message counter again. Preserve the poll failure and let the server recycle the event. + if (nackFailure != pollFailure) { + pollFailure.addSuppressed(nackFailure); + } + } + } + private Optional pollTabletsInternal( final SubscriptionPollResponse initialResponse, final PollTimer timer) { final Map> tablets = @@ -1448,6 +1461,7 @@ private List pollFileInternal( if (isClosed()) { return Collections.emptyList(); } + providers.syncIfUnavailable(this); throw new SubscriptionConnectionException( String.format( SubscriptionMessages @@ -1459,6 +1473,7 @@ private List pollFileInternal( try { return provider.pollFile(commitContext, writingOffset, timeoutMs); } catch (final SubscriptionConnectionException ignored) { + providers.syncIfUnavailable(this); return Collections.emptyList(); } } finally { @@ -1477,6 +1492,7 @@ private List pollTabletsInternal( if (isClosed()) { return Collections.emptyList(); } + providers.syncIfUnavailable(this); throw new SubscriptionConnectionException( String.format( SubscriptionMessages @@ -1488,6 +1504,7 @@ private List pollTabletsInternal( try { return provider.pollTablets(commitContext, offset, timeoutMs); } catch (final SubscriptionConnectionException ignored) { + providers.syncIfUnavailable(this); return Collections.emptyList(); } } finally { @@ -1532,15 +1549,9 @@ private void commit(final Iterable commitContexts, fi dataNodeIdToSubscriptionCommitContexts.entrySet()) { final List groupedCommitContexts = entry.getValue(); requestedCount += groupedCommitContexts.size(); - final AbstractSubscriptionProvider.CommitResult commitResp = - commitInternal(entry.getKey(), groupedCommitContexts, nack); final List acceptedCommitContexts = - commitResp.getAcceptedCommitContexts(); + commitWithRetry(entry.getKey(), groupedCommitContexts, nack); acceptedCount += acceptedCommitContexts.size(); - if (!nack) { - overlayCommittedPositions(commitResp.getCommittedProgressByTopic()); - onAckedCommitContexts(acceptedCommitContexts); - } if (acceptedCommitContexts.size() != groupedCommitContexts.size()) { final List failedInGroup = new ArrayList<>(groupedCommitContexts); @@ -1561,6 +1572,88 @@ private void commit(final Iterable commitContexts, fi } } + private List commitWithRetry( + final int dataNodeId, + final List commitContexts, + final boolean nack) + throws SubscriptionException { + final List remainingCommitContexts = new ArrayList<>(commitContexts); + final List acceptedCommitContexts = new ArrayList<>(); + for (int retries = 0; ; retries++) { + checkIfFenced(); + AbstractSubscriptionProvider.CommitResult result = + AbstractSubscriptionProvider.CommitResult.empty(); + SubscriptionConnectionException connectionFailure = null; + SubscriptionException failure = null; + try { + result = commitInternal(dataNodeId, remainingCommitContexts, nack); + } catch (final SubscriptionException e) { + failure = getCommitFailure(e); + result = getCompletedCommitResult(e); + if (failure instanceof SubscriptionConnectionException) { + connectionFailure = (SubscriptionConnectionException) failure; + } + } + + final List acceptedInAttempt = result.getAcceptedCommitContexts(); + acceptedCommitContexts.addAll(acceptedInAttempt); + if (!nack) { + // Apply each response once, before waiting. Reapplying older progress at retry completion + // could overwrite a concurrent commit; accepted contexts must also leave the buffer now. + overlayCommittedPositions(result.getCommittedProgressByTopic()); + onAckedCommitContexts(acceptedInAttempt); + } + if (failure != null && connectionFailure == null) { + throw failure; + } + remainingCommitContexts.removeAll(new HashSet<>(acceptedInAttempt)); + if (nack + || remainingCommitContexts.isEmpty() + || retries >= MAX_COMMIT_RETRIES + || isClosed() + || Thread.currentThread().isInterrupted()) { + if (connectionFailure != null) { + throw connectionFailure; + } + return acceptedCommitContexts; + } + checkIfFenced(); + // A partial response has no permanent-rejection reason. Retry only its rejected contexts + // and any batches whose response was lost, using the same budget as connection recovery. + // commitInternal has released the provider read lock before we wait for recovery. + try { + Thread.sleep(COMMIT_RETRY_INTERVAL_MS); + } catch (final InterruptedException interruptedException) { + Thread.currentThread().interrupt(); + if (connectionFailure != null) { + connectionFailure.addSuppressed(interruptedException); + throw connectionFailure; + } + return acceptedCommitContexts; + } + checkIfFenced(); + if (isClosed()) { + if (connectionFailure != null) { + throw connectionFailure; + } + return acceptedCommitContexts; + } + } + } + + private SubscriptionException getCommitFailure(final SubscriptionException e) { + return e instanceof AbstractSubscriptionProvider.CommitException + ? ((AbstractSubscriptionProvider.CommitException) e).getFailure() + : e; + } + + private AbstractSubscriptionProvider.CommitResult getCompletedCommitResult( + final SubscriptionException e) { + return e instanceof AbstractSubscriptionProvider.CommitException + ? ((AbstractSubscriptionProvider.CommitException) e).getCompletedResult() + : AbstractSubscriptionProvider.CommitResult.empty(); + } + /** Invoked after the server accepts commit contexts from an acknowledgement request. */ protected void onAckedCommitContexts( final Collection acceptedCommitContexts) { @@ -1606,10 +1699,26 @@ protected Set ackCommitContextsWithPartialProgress( commitInternal(entry.getKey(), groupedCommitContexts, false); overlayCommittedPositions(commitResp.getCommittedProgressByTopic()); removableCommitContexts.addAll(commitResp.getAcceptedCommitContexts()); - } catch (final SubscriptionConnectionException e) { + } catch (final SubscriptionException e) { + final SubscriptionException failure = getCommitFailure(e); + final AbstractSubscriptionProvider.CommitResult completedResult = + getCompletedCommitResult(e); + overlayCommittedPositions(completedResult.getCommittedProgressByTopic()); + final Set acceptedBeforeFailure = + new HashSet<>(completedResult.getAcceptedCommitContexts()); + removableCommitContexts.addAll(acceptedBeforeFailure); + if (!(failure instanceof SubscriptionConnectionException)) { + // The caller cannot receive the removable set when an error propagates. Remove confirmed + // contexts from every completed group before leaving the automatic/close-time ACK path. + onAckedCommitContexts(removableCommitContexts); + throw failure; + } int stagedCount = 0; int retainedCount = 0; for (final SubscriptionCommitContext commitContext : groupedCommitContexts) { + if (acceptedBeforeFailure.contains(commitContext)) { + continue; + } if (isConsensusCommitContext(commitContext)) { stagePendingRedirectAck(commitContext); removableCommitContexts.add(commitContext); @@ -1633,7 +1742,7 @@ protected Set ackCommitContextsWithPartialProgress( this, retainedCount, entry.getKey(), - e); + failure); } } } @@ -1672,6 +1781,7 @@ private AbstractSubscriptionProvider.CommitResult commitInternal( final List subscriptionCommitContexts, final boolean nack) throws SubscriptionException { + checkIfFenced(); providers.acquireReadLock(); try { final AbstractSubscriptionProvider provider = providers.getProvider(dataNodeId); @@ -1679,6 +1789,7 @@ private AbstractSubscriptionProvider.CommitResult commitInternal( if (isClosed()) { return AbstractSubscriptionProvider.CommitResult.empty(); } + providers.syncIfUnavailable(this); throw new SubscriptionConnectionException( String.format( SubscriptionMessages @@ -1689,8 +1800,13 @@ private AbstractSubscriptionProvider.CommitResult commitInternal( } try { return provider.commit(subscriptionCommitContexts, nack); - } catch (final SubscriptionConsumerFencedException e) { - fence(e); + } catch (final SubscriptionException e) { + final SubscriptionException failure = getCommitFailure(e); + if (failure instanceof SubscriptionConsumerFencedException) { + fence((SubscriptionConsumerFencedException) failure); + } else if (failure instanceof SubscriptionConnectionException) { + providers.syncIfUnavailable(this); + } throw e; } } finally { @@ -2082,15 +2198,11 @@ private void flushPendingRedirectAcks(final List currentMes } final List contextsToRedirect = new ArrayList<>(pendingContexts); + AbstractSubscriptionProvider.CommitResult commitResp; try { - final AbstractSubscriptionProvider.CommitResult commitResp = - commitInternal(entry.getValue(), contextsToRedirect, false); - overlayCommittedPositions(commitResp.getCommittedProgressByTopic()); - commitResp.getAcceptedCommitContexts().forEach(pendingContexts::remove); - if (pendingContexts.isEmpty()) { - pendingRedirectAcksByTopicRegion.remove(entry.getKey(), pendingContexts); - } + commitResp = commitInternal(entry.getValue(), contextsToRedirect, false); } catch (final SubscriptionException e) { + commitResp = getCompletedCommitResult(e); LOGGER.warn( SubscriptionMessages .LOG_ARG_FAILED_REDIRECT_ARG_PENDING_CONSENSUS_ACK_S_ARG_VIA_E6A10AC5, @@ -2100,6 +2212,11 @@ private void flushPendingRedirectAcks(final List currentMes entry.getValue(), e); } + overlayCommittedPositions(commitResp.getCommittedProgressByTopic()); + commitResp.getAcceptedCommitContexts().forEach(pendingContexts::remove); + if (pendingContexts.isEmpty()) { + pendingRedirectAcksByTopicRegion.remove(entry.getKey(), pendingContexts); + } } } @@ -2154,7 +2271,24 @@ private void overlayCommittedPositions(final Map progress if (Objects.isNull(progressByTopic) || progressByTopic.isEmpty()) { return; } - progressByTopic.forEach(this::overlayCommittedPositions); + // ACK responses from different providers can arrive out of order, including completed batches + // carried by a failure. Preserve the newest position for each writer. Explicit seek operations + // use the topic-scoped overlay above and can still move a checkpoint backward. + for (final Entry entry : progressByTopic.entrySet()) { + if (Objects.isNull(entry.getValue())) { + continue; + } + for (final RegionProgress regionProgress : entry.getValue().getRegionProgress().values()) { + if (Objects.nonNull(regionProgress)) { + regionProgress + .getWriterPositions() + .forEach( + (writerId, writerProgress) -> + mergeTopicProgress( + committedPositionsByTopic, entry.getKey(), writerId, writerProgress)); + } + } + } } private void overlayTopicProgress( diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProvider.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProvider.java index 0beabf8d6145f..125005763dc53 100644 --- a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProvider.java +++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProvider.java @@ -78,6 +78,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.ReentrantLock; public abstract class AbstractSubscriptionProvider { @@ -107,7 +108,15 @@ public abstract class AbstractSubscriptionProvider { private final AtomicBoolean isClosed = new AtomicBoolean(true); private final AtomicBoolean isClosing = new AtomicBoolean(false); - private final AtomicBoolean isAvailable = new AtomicBoolean(false); + + private enum Availability { + NEW, + AVAILABLE, + UNAVAILABLE + } + + private final AtomicReference availability = + new AtomicReference<>(Availability.NEW); private final ReentrantLock rpcLock = new ReentrantLock(); private final int localThriftMaxFrameSize; @@ -243,15 +252,17 @@ private void updateRequestBodySizeLimit() { } boolean isAvailable() { - return isAvailable.get(); + return availability.get() == Availability.AVAILABLE; } void setAvailable() { - isAvailable.set(true); + // A heartbeat that completes after a failed RPC must not revive this connection. Recovery + // constructs and handshakes a new provider instead. + availability.compareAndSet(Availability.NEW, Availability.AVAILABLE); } void setUnavailable() { - isAvailable.set(false); + availability.set(Availability.UNAVAILABLE); } int getDataNodeId() { @@ -413,10 +424,17 @@ void close() throws SubscriptionException, IoTDBConnectionException { } synchronized void closeSession() throws IoTDBConnectionException { + setUnavailable(); try { + try { + // Recovery must preserve subscriptions and interrupt any heartbeat still using this + // session. + getSessionConnection().forceClose(); + } catch (final IoTDBConnectionException ignored) { + // A failed handshake may not have opened a session. + } session.close(); } finally { - setUnavailable(); isClosed.set(true); } } @@ -736,11 +754,34 @@ List poll(final SubscriptionPollRequest pollMessage) setUnavailable(); throw new SubscriptionConnectionException(e.getMessage(), e); } - verifyPipeSubscribeSuccess(resp.status); + try { + verifyPipeSubscribeSuccess(resp.status); + } catch (final SubscriptionRuntimeNonCriticalException e) { + // A missing broker is reported as a poll error instead of a transport failure. Treat it as a + // provider failure so the consumer can recreate this provider before the next poll. + if (isMissingBrokerPollError(resp.status)) { + setUnavailable(); + throw new SubscriptionConnectionException(e.getMessage(), e); + } + throw e; + } final PipeSubscribePollResp pollResp = PipeSubscribePollResp.fromTPipeSubscribeResp(resp); return pollResp.getResponses(); } + private boolean isMissingBrokerPollError(final TSStatus status) { + if (status.code != TSStatusCode.SUBSCRIPTION_POLL_ERROR.getStatusCode() + || status.message == null) { + return false; + } + // These are server protocol diagnostics. Match both server locales independently of the client + // locale, including the distinct messages returned by file and tablet continuation polls. + return status.message.contains("no broker bound to consumer group [" + consumerGroupId + "]") + || status.message.contains( + "broker bound to consumer group [" + consumerGroupId + "] does not exist") + || status.message.contains("绑定到消费者组 [" + consumerGroupId + "] 的 Broker 不存在"); + } + CommitResult commit( final List subscriptionCommitContexts, final boolean nack) throws SubscriptionException { @@ -763,20 +804,32 @@ CommitResult commit( final List acceptedCommitContexts = new ArrayList<>(); final Map committedProgressByTopic = new LinkedHashMap<>(); + int completedCommitContextCount = 0; rpcLock.lock(); try { for (final List batch : batches) { final CommitResult result = commitOnce(batch, nack); + completedCommitContextCount += batch.size(); acceptedCommitContexts.addAll(result.getAcceptedCommitContexts()); mergeTopicProgress(committedProgressByTopic, result.getCommittedProgressByTopic()); } + } catch (final SubscriptionException e) { + if (nack || completedCommitContextCount == 0) { + throw e; + } + // Preserve every completed response even if a later batch fails with a server error. The + // consumer applies these results before propagating the original failure or retrying. + throw new CommitException( + e, + completedCommitContextCount, + new CommitResult(acceptedCommitContexts, committedProgressByTopic)); } finally { rpcLock.unlock(); } return new CommitResult(acceptedCommitContexts, committedProgressByTopic); } - private CommitResult commitOnce( + CommitResult commitOnce( final List subscriptionCommitContexts, final boolean nack) throws SubscriptionException { final PipeSubscribeCommitReq req; @@ -857,6 +910,32 @@ interface PipeSubscribeRequestSender { TPipeSubscribeResp send(TPipeSubscribeReq req) throws TException; } + static final class CommitException extends SubscriptionException { + private final int completedCommitContextCount; + private final CommitResult completedResult; + + private CommitException( + final SubscriptionException cause, + final int completedCommitContextCount, + final CommitResult completedResult) { + super(cause.getMessage(), cause); + this.completedCommitContextCount = completedCommitContextCount; + this.completedResult = completedResult; + } + + int getCompletedCommitContextCount() { + return completedCommitContextCount; + } + + CommitResult getCompletedResult() { + return completedResult; + } + + SubscriptionException getFailure() { + return (SubscriptionException) getCause(); + } + } + static final class CommitResult { private static final CommitResult EMPTY = new CommitResult(Collections.emptyList(), Collections.emptyMap()); @@ -948,7 +1027,7 @@ public String toString() { + ", consumerGroupId=" + consumerGroupId + ", isAvailable=" - + isAvailable + + isAvailable() + ", isClosed=" + isClosed + "}"; diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProviders.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProviders.java index 565c719149d0a..afb5285b5f577 100644 --- a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProviders.java +++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProviders.java @@ -90,11 +90,14 @@ void releaseWriteLock() { /** Caller should ensure that the method is called in the lock {@link #acquireWriteLock()}. */ void openProviders(final AbstractSubscriptionConsumer consumer) throws SubscriptionException { // close stale providers - closeProviders(); + closeProviders(false); final Map connectionFailures = new LinkedHashMap<>(); final List connectionFailureCauses = new ArrayList<>(); for (final TEndPoint endPoint : initialEndpoints) { + if (isClosing.get() || consumer.isFenced()) { + return; + } final AbstractSubscriptionProvider defaultProvider; final int defaultDataNodeId; @@ -116,6 +119,9 @@ void openProviders(final AbstractSubscriptionConsumer consumer) throws Subscript } defaultDataNodeId = defaultProvider.getDataNodeId(); addProvider(defaultDataNodeId, defaultProvider); + if (isClosing.get() || consumer.isFenced()) { + return; + } final Map allEndPoints; try { @@ -134,6 +140,9 @@ void openProviders(final AbstractSubscriptionConsumer consumer) throws Subscript } for (final Map.Entry entry : allEndPoints.entrySet()) { + if (isClosing.get() || consumer.isFenced()) { + return; + } if (defaultDataNodeId == entry.getKey()) { continue; } @@ -243,7 +252,8 @@ void closeAndRemoveProvider(final int dataNodeId) } final AbstractSubscriptionProvider provider = subscriptionProviders.get(dataNodeId); try { - provider.close(); + // Replacing a connection must not unsubscribe topics or drop the shared consumer. + provider.closeSession(); } finally { LOGGER.info(SubscriptionMessages.CLOSE_STALE_PROVIDER, provider); subscriptionProviders.remove(dataNodeId); @@ -338,13 +348,19 @@ void heartbeat(final AbstractSubscriptionConsumer consumer) { private void heartbeatProvider( final AbstractSubscriptionConsumer consumer, final AbstractSubscriptionProvider provider) { try { - if (consumer.isClosed() || consumer.isFenced() || !containsProvider(provider)) { + if (consumer.isClosed() + || consumer.isFenced() + || !containsProvider(provider) + || !provider.isAvailable()) { return; } final List processorBufferedCommitContexts = consumer.getProcessorBufferedCommitContexts(provider.getDataNodeId()); final PipeSubscribeHeartbeatResp resp = provider.heartbeat(processorBufferedCommitContexts); - if (consumer.isClosed() || consumer.isFenced() || !containsProvider(provider)) { + if (consumer.isClosed() + || consumer.isFenced() + || !containsProvider(provider) + || !provider.isAvailable()) { return; } provider.setAvailable(); @@ -360,7 +376,9 @@ private void heartbeatProvider( } } } catch (final SubscriptionConsumerFencedException e) { - consumer.fence(e); + if (containsProvider(provider)) { + consumer.fence(e); + } provider.setUnavailable(); } catch (final Exception e) { LOGGER.warn( @@ -398,7 +416,7 @@ void sync(final AbstractSubscriptionConsumer consumer) { } } - /** Reconnects providers immediately after a failed RPC when every provider is unavailable. */ + /** Reconnects unavailable providers immediately after a failed RPC. */ void syncIfUnavailable(final AbstractSubscriptionConsumer consumer) { if (consumer.isClosed() || consumer.isFenced() @@ -415,7 +433,7 @@ void syncIfUnavailable(final AbstractSubscriptionConsumer consumer) { } acquireWriteLock(); try { - if (consumer.isClosed() || consumer.isFenced() || !hasNoAvailableProviders()) { + if (consumer.isClosed() || consumer.isFenced()) { return; } final long now = System.currentTimeMillis(); @@ -436,7 +454,7 @@ void syncIfUnavailable(final AbstractSubscriptionConsumer consumer) { } private void syncInternal(final AbstractSubscriptionConsumer consumer) { - if (consumer.isFenced()) { + if (consumer.isClosed() || consumer.isFenced()) { return; } if (hasNoAvailableProviders()) { @@ -462,64 +480,71 @@ private void syncInternal(final AbstractSubscriptionConsumer consumer) { return; } - // add new providers or handshake existing providers + // Add new providers, replace unavailable providers, or heartbeat existing providers. for (final Map.Entry entry : allEndPoints.entrySet()) { - if (consumer.isFenced()) { + if (consumer.isClosed() || consumer.isFenced()) { return; } - final AbstractSubscriptionProvider provider = getProvider(entry.getKey()); - if (Objects.isNull(provider)) { - // new provider - final TEndPoint endPoint = entry.getValue(); - final AbstractSubscriptionProvider newProvider; + AbstractSubscriptionProvider provider = getProvider(entry.getKey()); + if (Objects.nonNull(provider) && provider.isAvailable()) { + // Existing provider try { - newProvider = consumer.constructProviderAndHandshake(endPoint); + consumer.subscribedTopics = provider.heartbeat().getTopics(); + provider.setAvailable(); } catch (final SubscriptionConsumerFencedException e) { consumer.fence(e); + provider.setUnavailable(); return; } catch (final Exception e) { LOGGER.warn( - SubscriptionMessages.LOG_ARG_FAILED_CREATE_CONNECTION_ARG_BECAUSE_ARG_E536E22A, + SubscriptionMessages + .LOG_ARG_FAILED_SENDING_HEARTBEAT_SUBSCRIPTION_PROVIDER_ARG_BECAUSE_ARG_SET_0B38FB1F, consumer, - endPoint, + provider, e, e); - continue; + provider.setUnavailable(); } - addProvider(entry.getKey(), newProvider); - } else { - // existing provider + } + + if (Objects.nonNull(provider) && !provider.isAvailable()) { + // Remove unavailable providers before creating replacements in this same sync round. try { - consumer.subscribedTopics = provider.heartbeat().getTopics(); - provider.setAvailable(); - } catch (final SubscriptionConsumerFencedException e) { - consumer.fence(e); - provider.setUnavailable(); - return; + closeAndRemoveProvider(entry.getKey()); } catch (final Exception e) { LOGGER.warn( SubscriptionMessages - .LOG_ARG_FAILED_SENDING_HEARTBEAT_SUBSCRIPTION_PROVIDER_ARG_BECAUSE_ARG_SET_0B38FB1F, + .LOG_EXCEPTION_OCCURRED_ARG_CLOSING_REMOVING_SUBSCRIPTION_PROVIDER_ARG_BECAUSE_ARG_2EC38739, consumer, provider, e, e); - provider.setUnavailable(); } - // close and remove unavailable provider (reset the connection as much as possible) - if (!provider.isAvailable()) { - try { - closeAndRemoveProvider(entry.getKey()); - } catch (final Exception e) { - LOGGER.warn( - SubscriptionMessages - .LOG_EXCEPTION_OCCURRED_ARG_CLOSING_REMOVING_SUBSCRIPTION_PROVIDER_ARG_BECAUSE_ARG_2EC38739, - consumer, - provider, - e, - e); - } + provider = null; + } + + if (Objects.isNull(provider)) { + // New or replacement provider + if (consumer.isClosed() || consumer.isFenced()) { + return; } + final TEndPoint endPoint = entry.getValue(); + final AbstractSubscriptionProvider newProvider; + try { + newProvider = consumer.constructProviderAndHandshake(endPoint); + } catch (final SubscriptionConsumerFencedException e) { + consumer.fence(e); + return; + } catch (final Exception e) { + LOGGER.warn( + SubscriptionMessages.LOG_ARG_FAILED_CREATE_CONNECTION_ARG_BECAUSE_ARG_E536E22A, + consumer, + endPoint, + e, + e); + continue; + } + addProvider(entry.getKey(), newProvider); } } diff --git a/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionConsumerLifecycleTest.java b/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionConsumerLifecycleTest.java index 51f35c5d8205f..7a05318acf80e 100644 --- a/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionConsumerLifecycleTest.java +++ b/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionConsumerLifecycleTest.java @@ -436,7 +436,7 @@ public void testPartialCommitKeepsRejectedContextsInAutoCommitBuffer() throws Ex Assert.assertEquals(1, uncommittedCommitContexts.size()); Assert.assertEquals( Collections.singleton(rejectedCommitContext), uncommittedCommitContexts.get(0L)); - Assert.assertEquals(1, consumer.commitRequestCount); + Assert.assertEquals(4, consumer.commitRequestCount); } finally { consumer.close(); } diff --git a/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionConsumerMultiProviderPollTest.java b/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionConsumerMultiProviderPollTest.java deleted file mode 100644 index 34ef7a8fc4840..0000000000000 --- a/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionConsumerMultiProviderPollTest.java +++ /dev/null @@ -1,281 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.session.subscription.consumer.base; - -import org.apache.iotdb.common.rpc.thrift.TEndPoint; -import org.apache.iotdb.rpc.subscription.config.TopicConfig; -import org.apache.iotdb.rpc.subscription.exception.SubscriptionException; -import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionCommitContext; -import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionPollResponse; -import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionPollResponseType; -import org.apache.iotdb.rpc.subscription.payload.poll.TabletsPayload; -import org.apache.iotdb.rpc.subscription.payload.poll.TopicProgress; -import org.apache.iotdb.rpc.subscription.payload.response.PipeSubscribeHeartbeatResp; -import org.apache.iotdb.session.AbstractSessionBuilder; -import org.apache.iotdb.session.subscription.SubscriptionTreeSessionBuilder; -import org.apache.iotdb.session.subscription.payload.SubscriptionMessage; - -import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.write.record.Tablet; -import org.apache.tsfile.write.schema.IMeasurementSchema; -import org.apache.tsfile.write.schema.MeasurementSchema; -import org.junit.Assert; -import org.junit.Test; - -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; - -public class SubscriptionConsumerMultiProviderPollTest { - - private static final String HOST = "127.0.0.1"; - private static final int FIRST_PORT = 10_001; - private static final String TOPIC = "topic1"; - private static final String CONSUMER_ID = "test_consumer"; - private static final String CONSUMER_GROUP_ID = "test_consumer_group"; - private static final long LONG_INTERVAL_MS = 86_400_000L; - - @Test - public void testPollTriesNextProviderBeforeBackingOff() throws SubscriptionException { - final TestPullConsumer consumer = new TestPullConsumer(2, 0); - try { - consumer.open(); - consumer.subscribeTopic(); - - final List messages = consumer.pollForTest(1_000L); - - Assert.assertEquals(1, messages.size()); - Assert.assertEquals(1, consumer.getPollCount(1)); - Assert.assertEquals(1, consumer.getPollCount(2)); - Assert.assertEquals(0, consumer.getPollCount(3)); - Assert.assertEquals(0, consumer.getBackoffCount()); - } finally { - consumer.close(); - } - } - - @Test - public void testPollBacksOffAfterAllProvidersAreEmpty() throws SubscriptionException { - final TestPullConsumer consumer = new TestPullConsumer(1, 1); - try { - consumer.open(); - consumer.subscribeTopic(); - - final List messages = consumer.pollForTest(1_000L); - - Assert.assertEquals(1, messages.size()); - Assert.assertEquals(2, consumer.getPollCount(1)); - Assert.assertEquals(1, consumer.getPollCount(2)); - Assert.assertEquals(1, consumer.getPollCount(3)); - Assert.assertEquals(1, consumer.getBackoffCount()); - } finally { - consumer.close(); - } - } - - private static class TestPullConsumer extends AbstractSubscriptionPullConsumer { - - private final Map pollCounts = new HashMap<>(); - private final int dataProviderId; - private final int emptyPollsBeforeData; - private int backoffCount; - - private TestPullConsumer(final int dataProviderId, final int emptyPollsBeforeData) { - super( - new AbstractSubscriptionPullConsumerBuilder() - .host(HOST) - .port(FIRST_PORT) - .consumerId(CONSUMER_ID) - .consumerGroupId(CONSUMER_GROUP_ID) - .heartbeatIntervalMs(LONG_INTERVAL_MS) - .endpointsSyncIntervalMs(LONG_INTERVAL_MS) - .autoCommit(false)); - this.dataProviderId = dataProviderId; - this.emptyPollsBeforeData = emptyPollsBeforeData; - } - - @Override - protected AbstractSubscriptionProvider constructSubscriptionProvider( - final TEndPoint endPoint, - final String username, - final String password, - final String encryptedPassword, - final String consumerId, - final String consumerGroupId, - final String ownerId, - final Long ownerEpoch, - final int thriftMaxFrameSize, - final long heartbeatIntervalMs, - final int connectionTimeoutInMs) { - return new TestSubscriptionProvider( - endPoint, - username, - password, - encryptedPassword, - consumerId, - consumerGroupId, - ownerId, - ownerEpoch, - thriftMaxFrameSize, - heartbeatIntervalMs, - connectionTimeoutInMs, - pollCounts, - dataProviderId, - emptyPollsBeforeData); - } - - private void subscribeTopic() { - subscribedTopics = Collections.singletonMap(TOPIC, new TopicConfig()); - } - - private List pollForTest(final long timeoutMs) - throws SubscriptionException { - return poll(timeoutMs); - } - - private int getPollCount(final int dataNodeId) { - return pollCounts.getOrDefault(dataNodeId, 0); - } - - private int getBackoffCount() { - return backoffCount; - } - - @Override - void sleepAfterEmptyPollRound() { - backoffCount++; - } - } - - private static class TestSubscriptionProvider extends AbstractSubscriptionProvider { - - private final int dataNodeId; - private final Map pollCounts; - private final int dataProviderId; - private final int emptyPollsBeforeData; - - private TestSubscriptionProvider( - final TEndPoint endPoint, - final String username, - final String password, - final String encryptedPassword, - final String consumerId, - final String consumerGroupId, - final String ownerId, - final Long ownerEpoch, - final int thriftMaxFrameSize, - final long heartbeatIntervalMs, - final int connectionTimeoutInMs, - final Map pollCounts, - final int dataProviderId, - final int emptyPollsBeforeData) { - super( - endPoint, - username, - password, - encryptedPassword, - consumerId, - consumerGroupId, - ownerId, - ownerEpoch, - thriftMaxFrameSize, - heartbeatIntervalMs, - connectionTimeoutInMs); - this.dataNodeId = endPoint.port - FIRST_PORT + 1; - this.pollCounts = pollCounts; - this.dataProviderId = dataProviderId; - this.emptyPollsBeforeData = emptyPollsBeforeData; - } - - @Override - protected AbstractSessionBuilder constructSubscriptionSessionBuilder( - final String host, - final int port, - final String username, - final String password, - final String encryptedPassword, - final int thriftMaxFrameSize, - final int connectionTimeoutInMs) { - final boolean useEncryptedPassword = Objects.nonNull(encryptedPassword); - return new SubscriptionTreeSessionBuilder() - .host(host) - .port(port) - .username(username) - .password(useEncryptedPassword ? encryptedPassword : password) - .useEncryptedPassword(useEncryptedPassword) - .thriftMaxFrameSize(thriftMaxFrameSize) - .connectionTimeoutInMs(connectionTimeoutInMs); - } - - @Override - synchronized void handshake() { - setAvailable(); - } - - @Override - synchronized void close() { - setUnavailable(); - } - - @Override - int getDataNodeId() { - return dataNodeId; - } - - @Override - PipeSubscribeHeartbeatResp heartbeat( - final List processorBufferedCommitContexts) { - final PipeSubscribeHeartbeatResp response = new PipeSubscribeHeartbeatResp(); - response.getTopics().put(TOPIC, new TopicConfig()); - response.getEndPoints().put(1, new TEndPoint(HOST, FIRST_PORT)); - response.getEndPoints().put(2, new TEndPoint(HOST, FIRST_PORT + 1)); - response.getEndPoints().put(3, new TEndPoint(HOST, FIRST_PORT + 2)); - return response; - } - - @Override - List poll( - final Set topicNames, - final long timeoutMs, - final Map progressByTopic) - throws SubscriptionException { - pollCounts.merge(dataNodeId, 1, Integer::sum); - if (dataNodeId != dataProviderId || pollCounts.get(dataNodeId) <= emptyPollsBeforeData) { - return Collections.emptyList(); - } - final SubscriptionCommitContext commitContext = - new SubscriptionCommitContext(dataNodeId, 0, TOPIC, CONSUMER_GROUP_ID, 0L); - final List schemas = - Collections.singletonList(new MeasurementSchema("s1", TSDataType.INT64)); - final Tablet tablet = new Tablet("root.sg.d1", schemas, 1); - tablet.setTimestamps(new long[] {1L}); - ((long[]) tablet.getValues()[0])[0] = 1L; - tablet.setRowSize(1); - return Collections.singletonList( - new SubscriptionPollResponse( - SubscriptionPollResponseType.TABLETS.getType(), - new TabletsPayload(Collections.singletonList(tablet), -1), - commitContext)); - } - } -} diff --git a/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionProviderConnectionTimeoutTest.java b/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionProviderConnectionTimeoutTest.java index ab7a741b07dc7..46dbd1f8e9f7c 100644 --- a/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionProviderConnectionTimeoutTest.java +++ b/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionProviderConnectionTimeoutTest.java @@ -22,10 +22,13 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.iotdb.rpc.subscription.exception.SubscriptionConnectionException; +import org.apache.iotdb.rpc.subscription.exception.SubscriptionRuntimeNonCriticalException; import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionCommitContext; +import org.apache.iotdb.rpc.subscription.payload.request.PipeSubscribeRequestType; import org.apache.iotdb.rpc.subscription.payload.response.PipeSubscribeHandshakeResp; import org.apache.iotdb.service.rpc.thrift.IClientRPCService; import org.apache.iotdb.service.rpc.thrift.ServerProperties; +import org.apache.iotdb.service.rpc.thrift.TPipeSubscribeReq; import org.apache.iotdb.service.rpc.thrift.TPipeSubscribeResp; import org.apache.iotdb.service.rpc.thrift.TSOpenSessionResp; import org.apache.iotdb.service.rpc.thrift.TSProtocolVersion; @@ -128,6 +131,133 @@ public void testCommitReadTimeoutClosesProviderTransport() throws Exception { } } + @Test + public void testMissingBrokerPollErrorMarksProviderUnavailable() throws Exception { + assertPollError( + "Subscription: no broker bound to consumer group [group]", true /* broker missing */); + } + + @Test + public void testMissingFileBrokerPollErrorMarksProviderUnavailable() throws Exception { + assertPollError( + "Subscription: pipe broker bound to consumer group [group] does not exist", true); + } + + @Test + public void testMissingTabletBrokerPollErrorMarksProviderUnavailable() throws Exception { + assertPollError("Subscription: broker bound to consumer group [group] does not exist", true); + } + + @Test + public void testMissingTabletBrokerPollErrorFromChineseServerMarksProviderUnavailable() + throws Exception { + assertPollError("订阅:绑定到消费者组 [group] 的 Broker 不存在", true); + } + + @Test + public void testOtherPollErrorRemainsNonCritical() throws Exception { + assertPollError("other poll error", false /* broker missing */); + } + + @Test + public void testNullPollErrorRemainsNonCritical() throws Exception { + assertPollError(null, false /* broker missing */); + } + + private void assertPollError(final String message, final boolean brokerMissing) throws Exception { + final AtomicInteger subscriptionCloseCount = new AtomicInteger(); + final AtomicInteger sessionCloseCount = new AtomicInteger(); + final ServerSocket socket = new ServerSocket(0, 50, InetAddress.getByName("127.0.0.1")); + final TServerSocket serverSocket = new TServerSocket(socket); + final IClientRPCService.Iface service = + (IClientRPCService.Iface) + Proxy.newProxyInstance( + IClientRPCService.Iface.class.getClassLoader(), + new Class[] {IClientRPCService.Iface.class}, + (proxy, method, args) -> { + switch (method.getName()) { + case "openSession": + return new TSOpenSessionResp( + successStatus(), TSProtocolVersion.IOTDB_SERVICE_PROTOCOL_V3) + .setSessionId(1L); + case "requestStatementId": + return 1L; + case "getProperties": + return new ServerProperties( + "test", Arrays.asList("max_time", "min_time"), "ms", 1); + case "pipeSubscribe": + final TPipeSubscribeReq request = (TPipeSubscribeReq) args[0]; + if (request.getType() == PipeSubscribeRequestType.HANDSHAKE.getType()) { + return PipeSubscribeHandshakeResp.toTPipeSubscribeResp( + successStatus(), 1, "consumer", "group"); + } + if (request.getType() != PipeSubscribeRequestType.POLL.getType()) { + if (request.getType() == PipeSubscribeRequestType.CLOSE.getType()) { + subscriptionCloseCount.incrementAndGet(); + } + return new TPipeSubscribeResp(successStatus(), (byte) 1, (short) 6); + } + return new TPipeSubscribeResp( + new TSStatus(TSStatusCode.SUBSCRIPTION_POLL_ERROR.getStatusCode()) + .setMessage(message), + (byte) 1, + (short) 6); + case "closeSession": + sessionCloseCount.incrementAndGet(); + return successStatus(); + default: + return defaultValue(method.getReturnType()); + } + }); + final TSimpleServer server = + new TSimpleServer( + new TSimpleServer.Args(serverSocket) + .processor(new IClientRPCService.Processor<>(service)) + .transportFactory(new org.apache.iotdb.rpc.TElasticFramedTransport.Factory())); + final Thread serverThread = + new Thread(server::serve, "subscription-provider-poll-error-server"); + serverThread.setDaemon(true); + serverThread.start(); + + final TestSubscriptionProvider provider = new TestSubscriptionProvider(socket.getLocalPort()); + try { + provider.handshake(); + final TTransport transport = getTransport(provider.getSessionConnection()); + final SubscriptionCommitContext context = + new SubscriptionCommitContext(1, 0, "topic", "group", 1L); + + if (brokerMissing) { + Assert.assertThrows( + SubscriptionConnectionException.class, + () -> provider.poll(Collections.singleton("topic"), 1_000L)); + Assert.assertFalse(provider.isAvailable()); + Assert.assertThrows( + SubscriptionConnectionException.class, () -> provider.pollFile(context, 0L, 1_000L)); + Assert.assertThrows( + SubscriptionConnectionException.class, () -> provider.pollTablets(context, 0, 1_000L)); + provider.closeSession(); + Assert.assertFalse(transport.isOpen()); + Assert.assertEquals(0, subscriptionCloseCount.get()); + Assert.assertEquals(0, sessionCloseCount.get()); + } else { + Assert.assertThrows( + SubscriptionRuntimeNonCriticalException.class, + () -> provider.poll(Collections.singleton("topic"), 1_000L)); + Assert.assertTrue(provider.isAvailable()); + Assert.assertThrows( + SubscriptionRuntimeNonCriticalException.class, + () -> provider.pollFile(context, 0L, 1_000L)); + Assert.assertThrows( + SubscriptionRuntimeNonCriticalException.class, + () -> provider.pollTablets(context, 0, 1_000L)); + } + } finally { + provider.close(); + server.stop(); + serverThread.join(2_000); + } + } + private static TTransport getTransport(final SubscriptionSessionConnection connection) throws Exception { final Field field = SessionConnection.class.getDeclaredField("transport"); diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensus.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensus.java index 63ddcc6eba861..28b0458c8b6f4 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensus.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensus.java @@ -82,6 +82,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; @@ -104,6 +105,7 @@ public class IoTConsensus implements IConsensus { private final IStateMachine.Registry registry; private final Map stateMachineMap = new ConcurrentHashMap<>(); + private final Set pendingPeerCreatedCallbacks = ConcurrentHashMap.newKeySet(); private final IoTConsensusRPCService service; private final RegisterManager registerManager = new RegisterManager(); private final UserDataTransferAuditHandler userDataTransferAuditHandler; @@ -223,6 +225,7 @@ private void initAndRecover() throws IOException { config, userDataTransferAuditHandler, userDataTransferAuditClassifier); + pendingPeerCreatedCallbacks.add(consensusGroupId); stateMachineMap.put(consensusGroupId, consensus); } } catch (DiskSpaceInsufficientException e) { @@ -253,15 +256,19 @@ private void initAndRecover() throws IOException { } stateMachineMap.forEach( (consensusGroupId, consensus) -> { - final BiConsumer callback = onPeerRecovering; - if (callback != null) { - callback.accept(consensusGroupId, consensus); + try { + final BiConsumer callback = onPeerRecovering; + if (callback != null) { + callback.accept(consensusGroupId, consensus); + } + consensus.start(); + // Subscription metadata may have been loaded before any local consensus peers existed. + // Recovered peers must trigger the same binding as newly created peers, after startup + // has restored their indices and installed the pre-start WAL-retention guards. + notifyPeerCreated(consensusGroupId, consensus); + } finally { + pendingPeerCreatedCallbacks.remove(consensusGroupId); } - consensus.start(); - // Subscription metadata may have been loaded before any local consensus peers existed. - // Recovered peers must trigger the same binding as newly created peers, after startup - // has restored their indices and installed the pre-start WAL-retention guards. - notifyPeerCreated(consensusGroupId, consensus); }); } @@ -355,6 +362,7 @@ public void createLocalPeer(ConsensusGroupId groupId, List peers) throw new RuntimeException(e); } impl.start(); + pendingPeerCreatedCallbacks.add(groupId); return impl; })) .orElseThrow( @@ -367,7 +375,16 @@ public void createLocalPeer(ConsensusGroupId groupId, List peers) throw new ConsensusGroupAlreadyExistException(groupId); } - notifyPeerCreated(groupId, stateMachineMap.get(groupId)); + try { + notifyPeerCreated(groupId, stateMachineMap.get(groupId)); + } finally { + pendingPeerCreatedCallbacks.remove(groupId); + } + } + + /** Returns whether a new or recovered local peer is still waiting for its creation callback. */ + public boolean isPeerCreationCallbackPending(final ConsensusGroupId groupId) { + return pendingPeerCreatedCallbacks.contains(groupId); } private void notifyPeerCreated( @@ -412,6 +429,7 @@ public void deleteLocalPeer(ConsensusGroupId groupId) throws ConsensusException if (!exist.get()) { throw new ConsensusGroupNotExistException(groupId); } + pendingPeerCreatedCallbacks.remove(groupId); FileUtils.deleteFileOrDirectoryWithRateLimiter( new File(buildPeerDir(storageDir, groupId)), RegionMigrationFileRemoveRateLimiter.getInstance()::acquire); diff --git a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/IoTConsensusRecoveryTest.java b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/IoTConsensusRecoveryTest.java index fcbb2772a62d5..4e7131f1a5c03 100644 --- a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/IoTConsensusRecoveryTest.java +++ b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/IoTConsensusRecoveryTest.java @@ -101,12 +101,14 @@ public void testRecoveredPeersNotifyAfterStarting() throws Exception { IoTConsensus.onPeerRecovering = (groupId, server) -> { assertSame(server, consensus.getImpl(groupId)); + assertTrue(consensus.isPeerCreationCallbackPending(groupId)); assertFalse(stateMachines.get(groupId).started); notifications.computeIfAbsent(groupId, ignored -> new ArrayList<>()).add("recovering"); }; IoTConsensus.onNewPeerCreated = (groupId, server) -> { assertSame(server, consensus.getImpl(groupId)); + assertTrue(consensus.isPeerCreationCallbackPending(groupId)); assertTrue(stateMachines.get(groupId).started); notifications.computeIfAbsent(groupId, ignored -> new ArrayList<>()).add("available"); }; @@ -116,6 +118,8 @@ public void testRecoveredPeersNotifyAfterStarting() throws Exception { assertEquals(2, notifications.size()); assertEquals(Arrays.asList("recovering", "available"), notifications.get(firstRegion)); assertEquals(Arrays.asList("recovering", "available"), notifications.get(secondRegion)); + assertFalse(consensus.isPeerCreationCallbackPending(firstRegion)); + assertFalse(consensus.isPeerCreationCallbackPending(secondRegion)); } @Test @@ -141,6 +145,24 @@ public void testRecoveryContinuesAfterPeerNotificationFailure() throws Exception assertTrue(notifiedRegions.containsAll(Arrays.asList(firstRegion, secondRegion))); assertTrue(stateMachines.get(firstRegion).started); assertTrue(stateMachines.get(secondRegion).started); + assertFalse(consensus.isPeerCreationCallbackPending(firstRegion)); + assertFalse(consensus.isPeerCreationCallbackPending(secondRegion)); + } + + @Test + public void testNewPeerRemainsPendingUntilCallbackReturns() throws Exception { + final DataRegionId regionId = new DataRegionId(3); + IoTConsensus.onNewPeerCreated = + (groupId, server) -> { + assertSame(server, consensus.getImpl(groupId)); + assertTrue(consensus.getAllConsensusGroupIds().contains(groupId)); + assertTrue(consensus.isPeerCreationCallbackPending(groupId)); + throw new IllegalStateException("subscription binding failed"); + }; + + createPeer(regionId); + + assertFalse(consensus.isPeerCreationCallbackPending(regionId)); } private IoTConsensus newConsensus() { diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java index f3e3f5ec09d9a..56f39342242c0 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java @@ -2378,6 +2378,10 @@ private DataNodePipeMessages() {} public static final String PIPE_EXCEPTION_FORCERESIZE_FAILED_TO_ALLOCATE_MEMORY_AFTER_D_RETRIES_TOTAL_8C6948BC = "forceResize: failed to allocate memory after %d retries, total memory size %d bytes, used " + "memory size %d bytes, requested memory size %d bytes"; + public static final String EXCEPTION_FORCERESIZE_FAILED_AFTER_ARG_RETRIES_TOTAL_ARG_BYTES_USED_ARG_BYTES_USEDTABLETS_ARG_BYTES_USEDTSFILES_ARG_BYTES_TABLETLIMIT_ARG_BYTES_TSFILELIMIT_ARG_BYTES_TABLETANDTSFILELIMIT_ARG_BYTES_PROTECTEDPARSERLIMIT_ARG_BYTES_REQUESTED_ARG_BYTES_311C8C7E = + "forceResize failed after %d retries: total=%d bytes, used=%d bytes, " + + "usedTablets=%d bytes, usedTsFiles=%d bytes, tabletLimit=%d bytes, tsFileLimit=%d bytes, " + + "tabletAndTsFileLimit=%d bytes, protectedParserLimit=%d bytes, requested=%d bytes"; public static final String EXCEPTION_UNSUPPORTED_BATCH_TYPE_ARG_WHEN_TRANSFERRING_TABLET_INSERTION_EVENT_66153E12 = "Unsupported batch type %s when transferring tablet insertion event."; diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java index 2b963bb1c0e0b..f2759b8725879 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java @@ -2210,6 +2210,10 @@ private DataNodePipeMessages() {} "forceAllocate:重试 %d 次后仍无法分配内存,总内存大小 %d bytes,已用内存大小 %d bytes,请求内存大小 %d bytes"; public static final String PIPE_EXCEPTION_FORCERESIZE_FAILED_TO_ALLOCATE_MEMORY_AFTER_D_RETRIES_TOTAL_8C6948BC = "forceResize:重试 %d 次后仍无法分配内存,总内存大小 %d bytes,已用内存大小 %d bytes,请求内存大小 %d bytes"; + public static final String EXCEPTION_FORCERESIZE_FAILED_AFTER_ARG_RETRIES_TOTAL_ARG_BYTES_USED_ARG_BYTES_USEDTABLETS_ARG_BYTES_USEDTSFILES_ARG_BYTES_TABLETLIMIT_ARG_BYTES_TSFILELIMIT_ARG_BYTES_TABLETANDTSFILELIMIT_ARG_BYTES_PROTECTEDPARSERLIMIT_ARG_BYTES_REQUESTED_ARG_BYTES_311C8C7E = + "forceResize 重试 %d 次后仍无法分配内存:total=%d bytes,used=%d bytes," + + "usedTablets=%d bytes,usedTsFiles=%d bytes,tabletLimit=%d bytes,tsFileLimit=%d bytes," + + "tabletAndTsFileLimit=%d bytes,protectedParserLimit=%d bytes,requested=%d bytes"; public static final String EXCEPTION_UNSUPPORTED_BATCH_TYPE_ARG_WHEN_TRANSFERRING_TABLET_INSERTION_EVENT_66153E12 = "传输 tablet insertion event 时不支持 batch 类型 %s。"; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/TsFileInsertionEventParserMemoryManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/TsFileInsertionEventParserMemoryManager.java index 11a31ff77aef0..7bb9a153477b8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/TsFileInsertionEventParserMemoryManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/TsFileInsertionEventParserMemoryManager.java @@ -20,9 +20,9 @@ package org.apache.iotdb.db.pipe.event.common.tsfile.parser; import org.apache.iotdb.commons.exception.pipe.PipeRuntimeOutOfMemoryCriticalException; -import org.apache.iotdb.db.i18n.DataNodePipeMessages; import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager; import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryBlock; +import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryManager; /** Allocates parser working memory from the pool owned by the caller. */ public interface TsFileInsertionEventParserMemoryManager { @@ -42,12 +42,15 @@ final class PipeHolder { public TsFileInsertionEventParserMemoryBlock forceAllocateForTabletWithRetry( final long sizeInBytes) { return new PipeBlock( - PipeDataNodeResourceManager.memory().forceAllocateForTabletWithRetry(sizeInBytes)); + PipeDataNodeResourceManager.memory() + .forceAllocateForTsFileParserWithRetry(sizeInBytes)); } @Override public TsFileInsertionEventParserMemoryBlock forceAllocate(final long sizeInBytes) { - return new PipeBlock(PipeDataNodeResourceManager.memory().forceAllocate(sizeInBytes)); + return new PipeBlock( + PipeDataNodeResourceManager.memory() + .forceAllocateForTsFileParserWithRetry(sizeInBytes)); } }; } @@ -67,15 +70,10 @@ public long getMemoryUsageInBytes() { @Override public void forceResize(final long newSizeInBytes) { final long oldSize = delegate.getMemoryUsageInBytes(); - if (!PipeDataNodeResourceManager.memory().tryResize(delegate, newSizeInBytes)) { + final PipeMemoryManager memoryManager = PipeDataNodeResourceManager.memory(); + if (!memoryManager.tryResize(delegate, newSizeInBytes)) { throw new PipeRuntimeOutOfMemoryCriticalException( - String.format( - DataNodePipeMessages - .PIPE_EXCEPTION_FORCERESIZE_FAILED_TO_ALLOCATE_MEMORY_AFTER_D_RETRIES_TOTAL_8C6948BC, - 0, - PipeDataNodeResourceManager.memory().getTotalNonFloatingMemorySizeInBytes(), - PipeDataNodeResourceManager.memory().getUsedMemorySizeInBytes(), - newSizeInBytes - oldSize)); + memoryManager.getResizeFailureMessage(0, newSizeInBytes - oldSize)); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryBlockCategory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryBlockCategory.java new file mode 100644 index 0000000000000..fb1e8df2c591d --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryBlockCategory.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.resource.memory; + +/** The diagnostic category of a Pipe memory block. */ +public enum PipeMemoryBlockCategory { + GLOBAL, + EVENT, + EVENT_CHILD, + PARSER, + TABLET, + TS_FILE, + BATCH, + WAL, + CACHE, + RECEIVER, + SINK, + SUBSCRIPTION, + FLOATING, + OTHER; + + public static PipeMemoryBlockCategory fromType(final PipeMemoryBlockType type) { + if (type == null) { + return OTHER; + } + return switch (type) { + case NORMAL -> OTHER; + case TABLET -> TABLET; + case TS_FILE_PARSER -> PARSER; + case TS_FILE -> TS_FILE; + case BATCH -> BATCH; + case WAL -> WAL; + }; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryBlockType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryBlockType.java index 846fc7dd1ce17..78fd9aff6704a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryBlockType.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryBlockType.java @@ -22,6 +22,7 @@ public enum PipeMemoryBlockType { NORMAL, TABLET, + TS_FILE_PARSER, TS_FILE, BATCH, WAL diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java index 6d89ac0d9d221..11c5d14aaae5a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java @@ -138,6 +138,34 @@ private double allowedMaxMemorySizeInBytesOfTsTiles( * totalNonFloatingMemorySizeInBytes; } + private boolean isEnoughForTsFileParserMemory( + final long extraMemoryInBytes, final long totalNonFloatingMemorySizeInBytes) { + return (double) usedMemorySizeInBytesOfTablets + + (double) usedMemorySizeInBytesOfTsFiles + + (double) extraMemoryInBytes + < EXCEED_PROTECT_THRESHOLD + * allowedMaxMemorySizeInBytesOfTabletsAndTsFiles(totalNonFloatingMemorySizeInBytes); + } + + public synchronized String getResizeFailureMessage(final int retries, final long requestedSize) { + final long totalNonFloatingMemorySizeInBytes = getTotalNonFloatingMemorySizeInBytes(); + final double combinedLimit = + allowedMaxMemorySizeInBytesOfTabletsAndTsFiles(totalNonFloatingMemorySizeInBytes); + return String.format( + DataNodePipeMessages + .EXCEPTION_FORCERESIZE_FAILED_AFTER_ARG_RETRIES_TOTAL_ARG_BYTES_USED_ARG_BYTES_USEDTABLETS_ARG_BYTES_USEDTSFILES_ARG_BYTES_TABLETLIMIT_ARG_BYTES_TSFILELIMIT_ARG_BYTES_TABLETANDTSFILELIMIT_ARG_BYTES_PROTECTEDPARSERLIMIT_ARG_BYTES_REQUESTED_ARG_BYTES_311C8C7E, + retries, + totalNonFloatingMemorySizeInBytes, + memoryBlock.getUsedMemoryInBytes(), + usedMemorySizeInBytesOfTablets, + usedMemorySizeInBytesOfTsFiles, + Math.round(allowedMaxMemorySizeInBytesOfTablets(totalNonFloatingMemorySizeInBytes)), + Math.round(allowedMaxMemorySizeInBytesOfTsTiles(totalNonFloatingMemorySizeInBytes)), + Math.round(combinedLimit), + Math.round(EXCEED_PROTECT_THRESHOLD * combinedLimit), + requestedSize); + } + private static long getTsFileParserMemorySizeInBytes() { return Math.max( PIPE_CONFIG.getTsFileParserMemory(), PIPE_CONFIG.getPipeMemoryAllocateMinSizeInBytes()); @@ -542,6 +570,10 @@ < allowedMaxMemorySizeInBytesOfTabletsAndTsFiles(totalNonFloatingMemorySizeInByt private boolean isHardEnoughForResizing( final PipeMemoryBlock block, final long extraMemoryInBytes) { + if (block instanceof PipeTsFileParserMemoryBlock) { + return isEnoughForTsFileParserMemory( + extraMemoryInBytes, getTotalNonFloatingMemorySizeInBytes()); + } if (!(block instanceof PipeTabletMemoryBlock) && !(block instanceof PipeTsFileMemoryBlock)) { return true; } @@ -625,6 +657,27 @@ public PipeTabletMemoryBlock forceAllocateForTabletWithRetry(long tabletSizeInBy } } + public synchronized PipeTsFileParserMemoryBlock forceAllocateForTsFileParserWithRetry( + final long parserMemorySizeInBytes) { + if (!PIPE_MEMORY_MANAGEMENT_ENABLED) { + return new PipeTsFileParserMemoryBlock(0); + } + + final PipeTsFileParserMemoryBlock block = + (PipeTsFileParserMemoryBlock) registerMemoryBlock(0, PipeMemoryBlockType.TS_FILE_PARSER); + if (parserMemorySizeInBytes == 0) { + return block; + } + + try { + forceResize(block, parserMemorySizeInBytes); + return block; + } catch (final RuntimeException e) { + release(block); + throw e; + } + } + public PipeTsFileMemoryBlock forceAllocateForTsFileWithRetry(long tsFileSizeInBytes) throws PipeRuntimeOutOfMemoryCriticalException { if (!PIPE_MEMORY_MANAGEMENT_ENABLED) { @@ -712,6 +765,8 @@ private PipeMemoryBlock forceAllocateWithRetry(long sizeInBytes, PipeMemoryBlock switch (type) { case TABLET: return new PipeTabletMemoryBlock(sizeInBytes); + case TS_FILE_PARSER: + return new PipeTsFileParserMemoryBlock(sizeInBytes); case TS_FILE: return new PipeTsFileMemoryBlock(sizeInBytes); case BATCH: @@ -800,17 +855,14 @@ public synchronized void resize( if (force) { throw new PipeRuntimeOutOfMemoryCriticalException( - String.format( - DataNodePipeMessages - .PIPE_EXCEPTION_FORCERESIZE_FAILED_TO_ALLOCATE_MEMORY_AFTER_D_RETRIES_TOTAL_8C6948BC, - memoryAllocateMaxRetries, - getTotalNonFloatingMemorySizeInBytes(), - memoryBlock.getUsedMemoryInBytes(), - sizeInBytes)); + getResizeFailureMessage(memoryAllocateMaxRetries, sizeInBytes)); } } private boolean tryResizeInternal(final PipeMemoryBlock block, final long targetSize) { + if (block.isReleased()) { + return false; + } if (!PIPE_MEMORY_MANAGEMENT_ENABLED) { block.setMemoryUsageInBytes(targetSize); return true; @@ -983,6 +1035,9 @@ private PipeMemoryBlock registerMemoryBlock(long sizeInBytes, PipeMemoryBlockTyp case TABLET: returnedMemoryBlock = new PipeTabletMemoryBlock(sizeInBytes); break; + case TS_FILE_PARSER: + returnedMemoryBlock = new PipeTsFileParserMemoryBlock(sizeInBytes); + break; case TS_FILE: returnedMemoryBlock = new PipeTsFileMemoryBlock(sizeInBytes); break; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeTsFileParserMemoryBlock.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeTsFileParserMemoryBlock.java new file mode 100644 index 0000000000000..5258d5216b5cd --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeTsFileParserMemoryBlock.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.db.pipe.resource.memory; + +/** Parser working memory charged to TsFiles and admitted against the protected combined quota. */ +public class PipeTsFileParserMemoryBlock extends PipeTsFileMemoryBlock { + + public PipeTsFileParserMemoryBlock(final long memoryUsageInBytes) { + super(memoryUsageInBytes); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java index 3601f433c25af..1c0dd008dfe65 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java @@ -1475,7 +1475,8 @@ private void updateMemoryInfo( break; } // The asynchronous submitter needs the region write lock held by this insertion. - // Submit other already-selected MemTables here so they can release memory while we wait. + // Submit other already-selected MemTables here so they can release memory while we + // wait. submitOtherSelectedMemTablesToFlush(); try { TimeUnit.MILLISECONDS.sleep(config.getCheckPeriodWhenInsertBlocked()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionBrokerAgent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionBrokerAgent.java index 0528bff3125c5..981069d9b73d6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionBrokerAgent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionBrokerAgent.java @@ -799,6 +799,33 @@ public void unbindConsensusPrefetchingQueue( prefetchingQueueCount.invalidate(); } + public Map> getConsensusQueueRegionIds( + final String consumerGroupId, final Set topicNames) { + final Map> regionIdsByTopic = new LinkedHashMap<>(); + final ConsensusSubscriptionBroker broker = getConsensusBroker(consumerGroupId); + if (Objects.nonNull(broker)) { + for (final String topicName : topicNames) { + regionIdsByTopic.put(topicName, broker.getQueueRegionIds(topicName)); + } + } + return regionIdsByTopic; + } + + public boolean hasConsensusQueueForRegion( + final String consumerGroupId, final String topicName, final String regionId) { + final ConsensusSubscriptionBroker broker = getConsensusBroker(consumerGroupId); + return Objects.nonNull(broker) && broker.hasQueueForRegion(topicName, regionId); + } + + public void rollbackConsensusPrefetchingQueues( + final String consumerGroupId, final String topicName, final Set existingRegionIds) { + final ConsensusSubscriptionBroker broker = getConsensusBroker(consumerGroupId); + if (Objects.nonNull(broker)) { + broker.rollbackConsensusPrefetchingQueues(topicName, existingRegionIds); + prefetchingQueueCount.invalidate(); + } + } + public void unbindByRegion(final ConsensusGroupId regionId) { int totalClosed = 0; for (final ConsensusSubscriptionBroker broker : getBrokers(ConsensusSubscriptionBroker.class)) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionConsumerAgent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionConsumerAgent.java index 79f0c59ec7e01..b3c00f287a41a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionConsumerAgent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionConsumerAgent.java @@ -30,6 +30,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Collections; import java.util.LinkedHashSet; import java.util.List; import java.util.Objects; @@ -106,11 +107,7 @@ private void handleSingleConsumerGroupMetaChangesInternal( // if consumer group meta does not exist on local agent if (Objects.isNull(metaInAgent)) { - SubscriptionAgent.broker().createPipeBrokerIfNotExist(consumerGroupId); - ConsensusSubscriptionSetupHandler.setupConsensusSubscriptions( - consumerGroupId, - metaFromCoordinator.getSubscribedTopicNames(), - metaFromCoordinator.visibleUnder(true)); + ensureConsumerGroupBrokerInternal(metaFromCoordinator); replaceConsumerGroupMeta(consumerGroupId, metaFromCoordinator); return; } @@ -134,12 +131,8 @@ private void handleSingleConsumerGroupMetaChangesInternal( } } - ConsensusSubscriptionSetupHandler.setupConsensusSubscriptions( - consumerGroupId, - metaFromCoordinator.getSubscribedTopicNames(), - metaFromCoordinator.visibleUnder(true)); + ensureConsumerGroupBrokerInternal(metaFromCoordinator); replaceConsumerGroupMeta(consumerGroupId, metaFromCoordinator); - // no need to create broker manually return; } @@ -168,12 +161,7 @@ private void handleSingleConsumerGroupMetaChangesInternal( newlySubscribedTopics); applyTopicDiff( - () -> { - if (!newlySubscribedTopics.isEmpty()) { - ConsensusSubscriptionSetupHandler.handleNewSubscriptions( - consumerGroupId, newlySubscribedTopics); - } - }, + () -> ensureConsumerGroupBrokerInternal(metaFromCoordinator), () -> { for (final String topicName : pipeTopicsUnsubByGroup) { SubscriptionAgent.broker().removePrefetchingQueue(consumerGroupId, topicName); @@ -259,6 +247,48 @@ private void handleDropConsumerGroupInternal(final String consumerGroupId) { removeConsumerGroupMeta(consumerGroupId); } + /** + * Ensures that a non-empty consumer group has the local broker and its consensus subscriptions. + * + *

The coordinator metadata is authoritative here. In particular, an empty metadata must not + * recreate a broker or restore topics that were explicitly removed from the group. + */ + public void ensureConsumerGroupBroker(final String consumerGroupId) { + executeWithConsumerGroupLock( + consumerGroupId, + () -> { + final ConsumerGroupMeta consumerGroupMeta = getConsumerGroupMeta(consumerGroupId); + if (Objects.nonNull(consumerGroupMeta)) { + ensureConsumerGroupBrokerInternal(consumerGroupMeta); + } + }); + } + + private void ensureConsumerGroupBrokerInternal(final ConsumerGroupMeta consumerGroupMeta) { + if (consumerGroupMeta.isEmpty()) { + return; + } + + final String consumerGroupId = consumerGroupMeta.getConsumerGroupId(); + final boolean brokerExisted = SubscriptionAgent.broker().isBrokerExist(consumerGroupId); + SubscriptionAgent.broker().createPipeBrokerIfNotExist(consumerGroupId); + try { + // setupConsensusSubscriptions is idempotent and also repairs a missing consensus broker or + // queue when a pipe broker still exists. + ConsensusSubscriptionSetupHandler.setupConsensusSubscriptions( + consumerGroupId, + consumerGroupMeta.getSubscribedTopicNames(), + consumerGroupMeta.visibleUnder(true)); + } catch (final RuntimeException e) { + // Allow a later metadata push or handshake to retry after a partial setup failure. Preserve + // an existing broker because it may still contain usable queues for the current metadata. + if (!brokerExisted) { + SubscriptionAgent.broker().dropBroker(consumerGroupId); + } + throw e; + } + } + private ConsumerGroupMeta getConsumerGroupMeta(final String consumerGroupId) { acquireReadLock(); try { @@ -287,7 +317,7 @@ private void removeConsumerGroupMeta(final String consumerGroupId) { } } - void executeWithConsumerGroupLock(final String consumerGroupId, final Runnable operation) { + public void executeWithConsumerGroupLock(final String consumerGroupId, final Runnable operation) { final ConsumerGroupOperationLock operationLock = consumerGroupOperationLocks.compute( consumerGroupId, @@ -352,6 +382,29 @@ public boolean isTableModel(final String consumerGroupId) { } } + public boolean isTopicSubscribedByConsumerGroup( + final String consumerGroupId, final String topicName) { + acquireReadLock(); + try { + return consumerGroupMetaKeeper.isTopicSubscribedByConsumerGroup(topicName, consumerGroupId); + } finally { + releaseReadLock(); + } + } + + public Set getTopicNamesSubscribedByConsumerGroup(final String consumerGroupId) { + acquireReadLock(); + try { + final ConsumerGroupMeta consumerGroupMeta = + consumerGroupMetaKeeper.getConsumerGroupMeta(consumerGroupId); + return Objects.isNull(consumerGroupMeta) + ? Collections.emptySet() + : new LinkedHashSet<>(consumerGroupMeta.getSubscribedTopicNames()); + } finally { + releaseReadLock(); + } + } + /** * Get all active subscriptions: consumerGroupId → set of subscribed topic names. Used by * consensus subscription auto-binding when a new DataRegion is created. diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/ConsensusSubscriptionBroker.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/ConsensusSubscriptionBroker.java index d5c140d0fe7f5..2d9d7a58c02ef 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/ConsensusSubscriptionBroker.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/ConsensusSubscriptionBroker.java @@ -749,6 +749,52 @@ public void unbindConsensusPrefetchingQueue(final String topicName) { closeAndRemoveConsensusPrefetchingQueues(topicName, true, false); } + public Set getQueueRegionIds(final String topicName) { + synchronized (queueLifecycleLock) { + return topicNameToConsensusPrefetchingQueues + .getOrDefault(topicName, Collections.emptyList()) + .stream() + .filter(queue -> !queue.isClosed()) + .map(queue -> queue.getConsensusGroupId().toString()) + .collect(Collectors.toSet()); + } + } + + public boolean hasQueueForRegion(final String topicName, final String regionId) { + synchronized (queueLifecycleLock) { + return topicNameToConsensusPrefetchingQueues + .getOrDefault(topicName, Collections.emptyList()) + .stream() + .anyMatch( + queue -> + !queue.isClosed() && regionId.equals(queue.getConsensusGroupId().toString())); + } + } + + public void rollbackConsensusPrefetchingQueues( + final String topicName, final Set existingRegionIds) { + final List queuesToClose = new ArrayList<>(); + synchronized (queueLifecycleLock) { + final List queues = + topicNameToConsensusPrefetchingQueues.get(topicName); + if (Objects.isNull(queues)) { + return; + } + for (final ConsensusPrefetchingQueue queue : queues) { + if (!existingRegionIds.contains(queue.getConsensusGroupId().toString())) { + queuesToClose.add(queue); + } + } + queues.removeAll(queuesToClose); + if (queues.isEmpty()) { + topicNameToConsensusPrefetchingQueues.remove(topicName, queues); + topicConsumerLastPollMs.remove(topicName); + topicOwnershipSnapshots.remove(topicName); + } + } + queuesToClose.forEach(queue -> queue.closeAsync(false)); + } + public void wakeUpPrefetchingQueues(final String topicName, final boolean isTableModel) { final List queues = topicNameToConsensusPrefetchingQueues.get(topicName); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionCommitManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionCommitManager.java index 14e50ae44d0a4..53280e8d636da 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionCommitManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionCommitManager.java @@ -273,17 +273,36 @@ SetupSnapshot captureSetupSnapshot(final String consumerGroupId, final Set attemptedTopicNames) { + restoreSetupSnapshot(setupSnapshot, attemptedTopicNames, Collections.emptyMap()); + } + + void restoreSetupSnapshot( + final SetupSnapshot setupSnapshot, + final Set attemptedTopicNames, + final Map> existingQueueRegionIds) { for (final String topicName : attemptedTopicNames) { final TopicSetupSnapshot topicSnapshot = setupSnapshot.topicSnapshots.get(topicName); if (Objects.isNull(topicSnapshot)) { continue; } synchronized (getTopicPersistLock(topicSnapshot.topicFileKey)) { - removeTopicStateFromMemory(setupSnapshot.consumerGroupId, topicName); + final Set preservedRegionIds = + existingQueueRegionIds.getOrDefault(topicName, Collections.emptySet()); + // Existing queues can commit while setup is running. Keep their state objects, progress, + // and initialization markers instead of rewinding them to the setup snapshot. + removeTopicStateFromMemory(setupSnapshot.consumerGroupId, topicName, preservedRegionIds); CommitStateKey stateKeyForRewrite = null; + if (!preservedRegionIds.isEmpty()) { + stateKeyForRewrite = + getCommitStateKey( + setupSnapshot.consumerGroupId, topicName, preservedRegionIds.iterator().next()); + } for (final Map.Entry entry : topicSnapshot.stateSnapshots.entrySet()) { + if (preservedRegionIds.contains(entry.getKey())) { + continue; + } final CommitStateKey stateKey = getCommitStateKey(setupSnapshot.consumerGroupId, topicName, entry.getKey()); final StateSetupSnapshot stateSnapshot = entry.getValue(); @@ -333,12 +352,14 @@ void restoreSetupSnapshot( } } - private void removeTopicStateFromMemory(final String consumerGroupId, final String topicName) { + private void removeTopicStateFromMemory( + final String consumerGroupId, final String topicName, final Set preservedRegionIds) { final Iterator> keyIterator = commitStateKeys.entrySet().iterator(); while (keyIterator.hasNext()) { final Map.Entry entry = keyIterator.next(); - if (!entry.getValue().sameTopic(consumerGroupId, topicName)) { + if (!entry.getValue().sameTopic(consumerGroupId, topicName) + || preservedRegionIds.contains(entry.getValue().regionIdStr)) { continue; } commitStates.remove(entry.getKey()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionSetupHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionSetupHandler.java index 87b3c66f698f1..c41260fb997df 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionSetupHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionSetupHandler.java @@ -138,7 +138,7 @@ public static void ensureNewRegionListenerRegistered() { * existing subscription metadata to find all active consensus subscriptions and binds prefetching * queues to the new region. */ - private static void onNewRegionCreated( + static void onNewRegionCreated( final ConsensusGroupId groupId, final IoTConsensusServerImpl serverImpl) { if (!(groupId instanceof DataRegionId)) { return; @@ -161,110 +161,128 @@ private static void onNewRegionCreated( allSubscriptions.size(), serverImpl.getSearchIndex()); - for (final Map.Entry> groupEntry : allSubscriptions.entrySet()) { - final String consumerGroupId = groupEntry.getKey(); - final boolean isTableModel = SubscriptionAgent.consumer().isTableModel(consumerGroupId); - for (final String topicName : groupEntry.getValue()) { - if (!isConsensusBasedTopic(topicName, isTableModel)) { + for (final String consumerGroupId : allSubscriptions.keySet()) { + // Serialize binding with metadata repair and its rollback. Re-read the group's topics after + // acquiring the lock so an unsubscribe while this callback was waiting cannot be undone. + SubscriptionAgent.consumer() + .executeWithConsumerGroupLock( + consumerGroupId, + () -> + bindNewRegionForConsumerGroup( + consumerGroupId, groupId, serverImpl, commitManager)); + } + } + + private static void bindNewRegionForConsumerGroup( + final String consumerGroupId, + final ConsensusGroupId groupId, + final IoTConsensusServerImpl serverImpl, + final ConsensusSubscriptionCommitManager commitManager) { + final boolean isTableModel = SubscriptionAgent.consumer().isTableModel(consumerGroupId); + for (final String topicName : + SubscriptionAgent.consumer().getTopicNamesSubscribedByConsumerGroup(consumerGroupId)) { + if (!isConsensusBasedTopic(topicName, isTableModel)) { + continue; + } + if (SubscriptionAgent.broker() + .hasConsensusQueueForRegion(consumerGroupId, topicName, groupId.toString())) { + continue; + } + try { + final Map topicConfigs = + SubscriptionAgent.topic() + .getTopicConfigs(java.util.Collections.singleton(topicName), isTableModel); + final TopicConfig topicConfig = topicConfigs.get(topicName); + if (topicConfig == null) { continue; } - try { - final Map topicConfigs = - SubscriptionAgent.topic() - .getTopicConfigs(java.util.Collections.singleton(topicName), isTableModel); - final TopicConfig topicConfig = topicConfigs.get(topicName); - if (topicConfig == null) { - continue; - } - - // Resolve the new DataRegion's actual database name - final DataRegion dataRegion = - StorageEngine.getInstance().getDataRegion((DataRegionId) groupId); - if (dataRegion == null) { - continue; - } - final String dbRaw = dataRegion.getDatabaseName(); - final String dbTableModel = dbRaw.startsWith("root.") ? dbRaw.substring(5) : dbRaw; - - if (!matchesTopicDataRegion(dbRaw, topicConfig, isTableModel)) { - continue; - } - - final String actualDbName = topicConfig.isTableTopic() ? dbTableModel : null; - final ConsensusLogToTabletConverter converter = - buildConverter(topicName, topicConfig, actualDbName); - final SubscriptionWalRetentionPolicy retentionPolicy = - buildSubscriptionWalRetentionPolicy(topicName, topicConfig, serverImpl); - - // Recover from persisted per-writer region progress when available. The queue will - // resolve a replay start from that progress on first poll via the region-level locator. - // A newly-created region starts from its first consensus index and never proposes its - // current local tail as an initial committed frontier. - commitManager.initializeStateWithoutTailProposal(consumerGroupId, topicName, groupId); - final RegionProgress committedRegionProgress = - resolveFallbackCommittedRegionProgress( - commitManager, consumerGroupId, topicName, groupId); - final boolean hasLocalPersistedState = - commitManager.hasPersistedState(consumerGroupId, topicName, groupId); - // This region is created after the subscription already exists. Its first WAL entry may - // already be visible by the time this callback runs, so start from the beginning of this - // new region instead of tail+1 to avoid skipping the first write. - final long tailStartSearchIndex = FIRST_CONSENSUS_SEARCH_INDEX; - final long initialRuntimeVersion = - regionRuntimeVersion.getOrDefault(groupId.convertToTConsensusGroupId(), 0L); - final boolean initialActive = - lastKnownPreferredWriter.getOrDefault(groupId.convertToTConsensusGroupId(), -1) - == IOTDB_CONFIG.getDataNodeId(); - final Set initialActiveWriterNodeIds = - regionActiveWriterNodeIds.getOrDefault( - groupId.convertToTConsensusGroupId(), - initialActive - ? Collections.singleton(IOTDB_CONFIG.getDataNodeId()) - : Collections.emptySet()); - final ConsensusRegionRuntimeState initialRuntimeState = - new ConsensusRegionRuntimeState( - initialRuntimeVersion, - lastKnownPreferredWriter.getOrDefault(groupId.convertToTConsensusGroupId(), -1), - initialActive, - initialActiveWriterNodeIds); - LOGGER.info( - DataNodePipeMessages - .PIPE_LOG_AUTO_BINDING_CONSENSUS_QUEUE_FOR_TOPIC_IN_GROUP_TO_NEW_REGION_86F21649, - topicName, - consumerGroupId, - groupId, - dbTableModel, - tailStartSearchIndex, - hasLocalPersistedState, - committedRegionProgress, - initialRuntimeVersion, - initialActive); + // Resolve the new DataRegion's actual database name + final DataRegion dataRegion = + StorageEngine.getInstance().getDataRegion((DataRegionId) groupId); + if (dataRegion == null) { + continue; + } + final String dbRaw = dataRegion.getDatabaseName(); + final String dbTableModel = dbRaw.startsWith("root.") ? dbRaw.substring(5) : dbRaw; - SubscriptionAgent.broker() - .bindConsensusPrefetchingQueue( - consumerGroupId, - topicName, - topicConfig.getOrderMode(), - groupId, - serverImpl, - retentionPolicy, - converter, - commitManager, - committedRegionProgress, - tailStartSearchIndex, - initialRuntimeVersion, - initialActive); - SubscriptionAgent.broker().applyRuntimeStateForRegion(groupId, initialRuntimeState); - } catch (final Exception e) { - LOGGER.error( - DataNodePipeMessages - .PIPE_LOG_FAILED_TO_AUTO_BIND_TOPIC_IN_GROUP_TO_NEW_REGION_5BFD0E7D, - topicName, - consumerGroupId, - groupId, - e); + if (!matchesTopicDataRegion(dbRaw, topicConfig, isTableModel)) { + continue; } + + final String actualDbName = topicConfig.isTableTopic() ? dbTableModel : null; + final ConsensusLogToTabletConverter converter = + buildConverter(topicName, topicConfig, actualDbName); + final SubscriptionWalRetentionPolicy retentionPolicy = + buildSubscriptionWalRetentionPolicy(topicName, topicConfig, serverImpl); + + // Recover from persisted per-writer region progress when available. The queue will + // resolve a replay start from that progress on first poll via the region-level locator. + // A newly-created region starts from its first consensus index and never proposes its + // current local tail as an initial committed frontier. + commitManager.initializeStateWithoutTailProposal(consumerGroupId, topicName, groupId); + final RegionProgress committedRegionProgress = + resolveFallbackCommittedRegionProgress( + commitManager, consumerGroupId, topicName, groupId); + final boolean hasLocalPersistedState = + commitManager.hasPersistedState(consumerGroupId, topicName, groupId); + // This region is created after the subscription already exists. Its first WAL entry may + // already be visible by the time this callback runs, so start from the beginning of this + // new region instead of tail+1 to avoid skipping the first write. + final long tailStartSearchIndex = FIRST_CONSENSUS_SEARCH_INDEX; + final long initialRuntimeVersion = + regionRuntimeVersion.getOrDefault(groupId.convertToTConsensusGroupId(), 0L); + final boolean initialActive = + lastKnownPreferredWriter.getOrDefault(groupId.convertToTConsensusGroupId(), -1) + == IOTDB_CONFIG.getDataNodeId(); + final Set initialActiveWriterNodeIds = + regionActiveWriterNodeIds.getOrDefault( + groupId.convertToTConsensusGroupId(), + initialActive + ? Collections.singleton(IOTDB_CONFIG.getDataNodeId()) + : Collections.emptySet()); + final ConsensusRegionRuntimeState initialRuntimeState = + new ConsensusRegionRuntimeState( + initialRuntimeVersion, + lastKnownPreferredWriter.getOrDefault(groupId.convertToTConsensusGroupId(), -1), + initialActive, + initialActiveWriterNodeIds); + + LOGGER.info( + DataNodePipeMessages + .PIPE_LOG_AUTO_BINDING_CONSENSUS_QUEUE_FOR_TOPIC_IN_GROUP_TO_NEW_REGION_86F21649, + topicName, + consumerGroupId, + groupId, + dbTableModel, + tailStartSearchIndex, + hasLocalPersistedState, + committedRegionProgress, + initialRuntimeVersion, + initialActive); + + SubscriptionAgent.broker() + .bindConsensusPrefetchingQueue( + consumerGroupId, + topicName, + topicConfig.getOrderMode(), + groupId, + serverImpl, + retentionPolicy, + converter, + commitManager, + committedRegionProgress, + tailStartSearchIndex, + initialRuntimeVersion, + initialActive); + SubscriptionAgent.broker().applyRuntimeStateForRegion(groupId, initialRuntimeState); + } catch (final Exception e) { + LOGGER.error( + DataNodePipeMessages.PIPE_LOG_FAILED_TO_AUTO_BIND_TOPIC_IN_GROUP_TO_NEW_REGION_5BFD0E7D, + topicName, + consumerGroupId, + groupId, + e); } } } @@ -341,6 +359,14 @@ public static void setupConsensusSubscriptions( public static void setupConsensusSubscriptions( final String consumerGroupId, final Set topicNames, final boolean isTableModel) { + SubscriptionAgent.consumer() + .executeWithConsumerGroupLock( + consumerGroupId, + () -> setupConsensusSubscriptionsInternal(consumerGroupId, topicNames, isTableModel)); + } + + private static void setupConsensusSubscriptionsInternal( + final String consumerGroupId, final Set topicNames, final boolean isTableModel) { final IConsensus dataRegionConsensus = DataRegionConsensusImpl.getInstance(); if (!(dataRegionConsensus instanceof IoTConsensus)) { final String configuredProtocol = IOTDB_CONFIG.getDataRegionConsensusProtocolClass(); @@ -370,6 +396,10 @@ public static void setupConsensusSubscriptions( topicNames, ioTConsensus.getAllConsensusGroupIds().size()); + // Recovery also visits topics with live queues. Roll back only newly bound regions so a + // failure on another topic cannot discard their in-flight events or committed progress. + final Map> existingQueueRegionIds = + SubscriptionAgent.broker().getConsensusQueueRegionIds(consumerGroupId, topicNames); final ConsensusSubscriptionCommitManager.SetupSnapshot setupSnapshot = commitManager.captureSetupSnapshot(consumerGroupId, topicNames); setupConsensusTopics( @@ -381,18 +411,27 @@ public static void setupConsensusSubscriptions( consumerGroupId, topicName, isTableModel, ioTConsensus, commitManager), attemptedTopicNames -> rollbackConsensusSubscriptionSetup( - consumerGroupId, attemptedTopicNames, commitManager, setupSnapshot)); + consumerGroupId, + attemptedTopicNames, + commitManager, + setupSnapshot, + existingQueueRegionIds)); } private static void rollbackConsensusSubscriptionSetup( final String consumerGroupId, final Set attemptedTopicNames, final ConsensusSubscriptionCommitManager commitManager, - final ConsensusSubscriptionCommitManager.SetupSnapshot setupSnapshot) { + final ConsensusSubscriptionCommitManager.SetupSnapshot setupSnapshot, + final Map> existingQueueRegionIds) { RuntimeException rollbackFailure = null; for (final String topicName : attemptedTopicNames) { try { - SubscriptionAgent.broker().unbindConsensusPrefetchingQueue(consumerGroupId, topicName); + SubscriptionAgent.broker() + .rollbackConsensusPrefetchingQueues( + consumerGroupId, + topicName, + existingQueueRegionIds.getOrDefault(topicName, Collections.emptySet())); } catch (final RuntimeException e) { if (Objects.isNull(rollbackFailure)) { rollbackFailure = e; @@ -402,7 +441,8 @@ private static void rollbackConsensusSubscriptionSetup( } } try { - commitManager.restoreSetupSnapshot(setupSnapshot, attemptedTopicNames); + commitManager.restoreSetupSnapshot( + setupSnapshot, attemptedTopicNames, existingQueueRegionIds); } catch (final RuntimeException e) { if (Objects.isNull(rollbackFailure)) { rollbackFailure = e; @@ -471,7 +511,7 @@ interface ConsensusTopicSetup { * DATABASE_KEY} filter. Additionally, the {@link #onNewRegionCreated} callback ensures that * regions created after this method runs are also automatically bound. */ - private static void setupConsensusQueueForTopic( + static void setupConsensusQueueForTopic( final String consumerGroupId, final String topicName, final boolean isTableModel, @@ -515,6 +555,23 @@ private static void setupConsensusQueueForTopic( if (!(groupId instanceof DataRegionId)) { continue; } + if (ioTConsensus.isPeerCreationCallbackPending(groupId) + && SubscriptionAgent.consumer() + .isTopicSubscribedByConsumerGroup(consumerGroupId, topicName)) { + // The new-region callback binds this peer from its first WAL index. Binding it here from + // the current tail could skip writes replicated before the callback finishes. + bound = true; + continue; + } + if (SubscriptionAgent.broker() + .hasConsensusQueueForRegion(consumerGroupId, topicName, groupId.toString())) { + // In particular, a queue created by onNewRegionCreated must keep its initialization from + // the beginning of the region instead of being reinitialized from a local tail proposal. + // Check the live broker as well because a region callback can finish after the setup + // snapshot was captured. + bound = true; + continue; + } final IoTConsensusServerImpl serverImpl = ioTConsensus.getImpl(groupId); if (serverImpl == null) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1.java index cd53d9376ab77..93d309cc5db80 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1.java @@ -454,6 +454,12 @@ private TPipeSubscribeResp handlePipeSubscribeHandshakeInternal( consumerConfig); } + // A previous inactivity cleanup or metadata race may have removed the local broker while the + // coordinator still has a non-empty consumer-group metadata record. Reconcile that state before + // exposing the consumer to subsequent requests. The operation is idempotent and serialized with + // metadata pushes by SubscriptionConsumerAgent. + SubscriptionAgent.consumer().ensureConsumerGroupBroker(consumerGroupId); + activateConsumer(consumerConfig); final int dataNodeId = IoTDBDescriptor.getInstance().getConfig().getDataNodeId(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManagerResizeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManagerResizeTest.java index ddaf043b969d9..6de1776e8be95 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManagerResizeTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManagerResizeTest.java @@ -30,6 +30,13 @@ import org.junit.Before; import org.junit.Test; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; public class PipeMemoryManagerResizeTest { @@ -226,4 +233,168 @@ public void testFloatingAndNonFloatingMemoryShareTheSamePool() { manager.release(nonFloatingMemory); } } + + @Test + public void testTsFileParserAllocationAndResizeShareProtectedCombinedLimit() { + final AtomicLong floatingMemoryUsageInBytes = new AtomicLong(0); + final PipeMemoryManager manager = + new PipeMemoryManager( + new AtomicLongMemoryBlock( + "PipeMemoryManagerResizeTest", + null, + TOTAL_MEMORY_SIZE_IN_BYTES, + MemoryBlockType.DYNAMIC), + floatingMemoryUsageInBytes::get); + final PipeTsFileParserMemoryBlock parser = manager.forceAllocateForTsFileParserWithRetry(1000); + + try { + Assert.assertTrue(manager.tryResize(parser, 1100)); + Assert.assertFalse(manager.tryResize(parser, 1140)); + Assert.assertEquals(1100, parser.getMemoryUsageInBytes()); + Assert.assertEquals(0, manager.getUsedMemorySizeInBytesOfTablets()); + Assert.assertEquals(1100, manager.getUsedMemorySizeInBytesOfTsFiles()); + Assert.assertEquals(1100, manager.getUsedMemorySizeInBytes()); + + final PipeRuntimeOutOfMemoryCriticalException allocationException = + Assert.assertThrows( + PipeRuntimeOutOfMemoryCriticalException.class, + () -> manager.forceAllocateForTsFileParserWithRetry(40)); + Assert.assertTrue( + allocationException.getMessage(), + allocationException.getMessage().contains("usedTsFiles=1100 bytes")); + Assert.assertTrue(allocationException.getMessage().contains("tabletLimit=900 bytes")); + Assert.assertTrue( + allocationException.getMessage().contains("tabletAndTsFileLimit=1200 bytes")); + Assert.assertTrue( + allocationException.getMessage().contains("protectedParserLimit=1140 bytes")); + + // Parser working memory must not consume the tablet-only quota: the generated tablet + // still needs to be retained and sent while the parser owns its decoded pages/chunks. + final PipeTabletMemoryBlock output = manager.forceAllocateForTabletWithRetry(50); + try { + Assert.assertTrue(manager.tryResize(output, 90)); + Assert.assertEquals(90, manager.getUsedMemorySizeInBytesOfTablets()); + Assert.assertEquals(1100, manager.getUsedMemorySizeInBytesOfTsFiles()); + } finally { + manager.release(output); + } + } finally { + manager.release(parser); + } + + Assert.assertEquals(0, manager.getUsedMemorySizeInBytes()); + } + + @Test + public void testZeroSizedTsFileParserCanGrowBeyondTabletLimitAndRelease() { + final PipeMemoryManager manager = + new PipeMemoryManager( + new AtomicLongMemoryBlock( + "PipeMemoryManagerResizeTest", + null, + TOTAL_MEMORY_SIZE_IN_BYTES, + MemoryBlockType.DYNAMIC), + () -> 0); + final PipeTsFileParserMemoryBlock parser = manager.forceAllocateForTsFileParserWithRetry(0); + try { + Assert.assertTrue(manager.tryResize(parser, 1000)); + Assert.assertEquals(1000, manager.getUsedMemorySizeInBytesOfTsFiles()); + Assert.assertEquals(1000, manager.getUsedMemorySizeInBytes()); + + Assert.assertTrue(manager.tryResize(parser, 100)); + Assert.assertEquals(100, manager.getUsedMemorySizeInBytesOfTsFiles()); + manager.release(parser); + Assert.assertTrue(parser.isReleased()); + Assert.assertEquals(0, manager.getUsedMemorySizeInBytesOfTsFiles()); + Assert.assertEquals(0, manager.getUsedMemorySizeInBytesOfTablets()); + Assert.assertEquals(0, manager.getUsedMemorySizeInBytes()); + } finally { + manager.release(parser); + } + } + + @Test + public void testTsFileParserSharesQuotaWithTabletsAndTsFilesAndHonorsGlobalLimit() { + final PipeMemoryManager manager = + new PipeMemoryManager( + new AtomicLongMemoryBlock( + "PipeMemoryManagerResizeTest", + null, + TOTAL_MEMORY_SIZE_IN_BYTES, + MemoryBlockType.DYNAMIC), + () -> 0); + final PipeTabletMemoryBlock tablet = manager.forceAllocateForTabletWithRetry(200); + final PipeTsFileMemoryBlock tsFile = manager.forceAllocateForTsFileWithRetry(500); + final PipeTsFileParserMemoryBlock parser = manager.forceAllocateForTsFileParserWithRetry(400); + final PipeMemoryBlock sink = manager.forceAllocate(900); + try { + Assert.assertFalse(manager.tryResize(parser, 440)); + Assert.assertFalse(manager.tryResize(parser, 401)); + Assert.assertEquals(200, manager.getUsedMemorySizeInBytesOfTablets()); + Assert.assertEquals(900, manager.getUsedMemorySizeInBytesOfTsFiles()); + Assert.assertEquals(2000, manager.getUsedMemorySizeInBytes()); + + manager.release(sink); + Assert.assertTrue(manager.tryResize(parser, 439)); + Assert.assertFalse(manager.tryResize(parser, 440)); + Assert.assertEquals(200, manager.getUsedMemorySizeInBytesOfTablets()); + Assert.assertEquals(939, manager.getUsedMemorySizeInBytesOfTsFiles()); + Assert.assertEquals(1139, manager.getUsedMemorySizeInBytes()); + } finally { + manager.release(tablet); + manager.release(tsFile); + manager.release(parser); + manager.release(sink); + } + Assert.assertEquals(0, manager.getUsedMemorySizeInBytesOfTablets()); + Assert.assertEquals(0, manager.getUsedMemorySizeInBytesOfTsFiles()); + Assert.assertEquals(0, manager.getUsedMemorySizeInBytes()); + } + + @Test + public void testConcurrentTsFileParserAllocationsCannotOvershootQuota() throws Exception { + final PipeMemoryManager manager = + new PipeMemoryManager( + new AtomicLongMemoryBlock( + "PipeMemoryManagerResizeTest", + null, + TOTAL_MEMORY_SIZE_IN_BYTES, + MemoryBlockType.DYNAMIC), + () -> 0); + final ExecutorService executor = Executors.newFixedThreadPool(4); + final CountDownLatch start = new CountDownLatch(1); + final List> futures = new ArrayList<>(); + final List allocated = new ArrayList<>(); + try { + for (int i = 0; i < 20; i++) { + futures.add( + executor.submit( + () -> { + start.await(); + try { + return manager.forceAllocateForTsFileParserWithRetry(300); + } catch (final PipeRuntimeOutOfMemoryCriticalException e) { + return null; + } + })); + } + start.countDown(); + for (final Future future : futures) { + final PipeTsFileParserMemoryBlock block = future.get(10, TimeUnit.SECONDS); + if (block != null) { + allocated.add(block); + } + } + Assert.assertEquals(3, allocated.size()); + Assert.assertEquals(900, manager.getUsedMemorySizeInBytes()); + Assert.assertEquals(0, manager.getUsedMemorySizeInBytesOfTablets()); + Assert.assertEquals(900, manager.getUsedMemorySizeInBytesOfTsFiles()); + } finally { + executor.shutdownNow(); + allocated.forEach(manager::release); + } + Assert.assertEquals(0, manager.getUsedMemorySizeInBytes()); + Assert.assertEquals(0, manager.getUsedMemorySizeInBytesOfTsFiles()); + Assert.assertEquals(0, manager.getUsedMemorySizeInBytesOfTablets()); + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java index 70810de95faf3..d652b831b1dd1 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java @@ -67,6 +67,7 @@ import org.apache.iotdb.db.storageengine.dataregion.compaction.utils.CompactionConfigRestorer; import org.apache.iotdb.db.storageengine.dataregion.flush.FlushManager; import org.apache.iotdb.db.storageengine.dataregion.flush.TsFileFlushPolicy; +import org.apache.iotdb.db.storageengine.dataregion.memtable.IMemTable; import org.apache.iotdb.db.storageengine.dataregion.memtable.ReadOnlyMemChunk; import org.apache.iotdb.db.storageengine.dataregion.memtable.TsFileProcessor; import org.apache.iotdb.db.storageengine.dataregion.modification.DeletionPredicate; @@ -79,7 +80,6 @@ import org.apache.iotdb.db.storageengine.dataregion.tsfile.generator.TsFileNameGenerator; import org.apache.iotdb.db.storageengine.rescon.memory.MemTableManager; import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo; -import org.apache.iotdb.db.storageengine.rescon.memory.WriteMemoryBlock; import org.apache.iotdb.db.utils.EnvironmentUtils; import org.apache.iotdb.db.utils.constant.TestConstant; import org.apache.iotdb.rpc.RpcUtils; @@ -241,6 +241,8 @@ private void assertRejectedInsertionFlushesOnlySelectedOtherMemtable(boolean tab final boolean originalSeparateData = config.isEnableSeparateData(); final long partitionInterval = TimePartitionUtils.getTimePartitionInterval(); try { + systemInfo.close(); + systemInfo.loadWriteMemory(); config.setEnableSeparateData(true); for (int i = 1; i <= 384; i++) { insertMemoryAdmissionTestRow(i, i); @@ -275,8 +277,7 @@ private void assertRejectedInsertionFlushesOnlySelectedOtherMemtable(boolean tab final DataRegionInfo regionInfo = (DataRegionInfo) regionInfoField.get(dataRegion); final long memoryBefore = regionInfo.getMemCost(); setMemoryAdmissionTestThreshold("memorySizeForMemtable", 2 * memoryBefore); - setMemoryAdmissionTestThreshold( - "FLUSH_THRESHOLD", (double) memoryBefore - selected.getWorkMemTableRamCost() / 2); + setMemoryAdmissionTestThreshold("FLUSH_THRESHOLD", (double) memoryBefore - 1); setMemoryAdmissionTestThreshold("REJECT_THRESHOLD", (double) memoryBefore - 1); config.setMaxWaitingTimeWhenInsertBlocked(5_000); config.setCheckPeriodWhenInsertBlocked(10); @@ -309,9 +310,6 @@ private void assertRejectedInsertionFlushesOnlySelectedOtherMemtable(boolean tab Assert.assertFalse(currentMemTable.shouldFlush()); Assert.assertFalse(unselectedMemTable.shouldFlush()); Assert.assertEquals(129, currentMemTable.getTotalPointsNum()); - for (WriteMemoryBlock block : regionInfo.getWriteMemoryManager().getMemoryBlocks()) { - Assert.assertEquals(0, block.getReservedMemoryInBytes()); - } Assert.assertFalse(systemInfo.isRejected()); } finally { dataRegion.writeUnlock(); @@ -321,7 +319,7 @@ private void assertRejectedInsertionFlushesOnlySelectedOtherMemtable(boolean tab insertMemoryAdmissionTestRow(partitionInterval + 130, 130); Assert.assertSame(currentMemTable, current.getWorkMemTable()); Assert.assertEquals(3, dataRegion.getTsFileManager().getTsFileList(true).size()); - dataRegion.syncCloseAllWorkingAndClosingTsFileProcessors(); + dataRegion.syncCloseAllWorkingTsFileProcessors(); int points = 0; for (TsFileResource resource : dataRegion.getTsFileManager().getTsFileList(true)) { try (TsFileSequenceReader reader = new TsFileSequenceReader(resource.getTsFilePath()); @@ -340,6 +338,7 @@ private void assertRejectedInsertionFlushesOnlySelectedOtherMemtable(boolean tab } Assert.assertEquals(515, points); } finally { + systemInfo.close(); systemInfo.loadWriteMemory(); config.setMaxWaitingTimeWhenInsertBlocked(originalWait); config.setCheckPeriodWhenInsertBlocked(originalPeriod); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/memory/SystemInfoTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/memory/SystemInfoTest.java index d330af1777e3a..8c5e3e362d56c 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/memory/SystemInfoTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/memory/SystemInfoTest.java @@ -86,7 +86,7 @@ public void tearDown() throws Exception { @Test public void testRejectedWriterCannotFlushSmallerCurrentProcessor() throws Exception { - dataRegionInfo.getWriteMemoryManager().addMemory(900); + dataRegionInfo.addStorageGroupMemCost(900); Assert.assertFalse(systemInfo.reportStorageGroupStatus(dataRegionInfo, currentProcessor)); Assert.assertTrue(systemInfo.isRejected()); @@ -97,7 +97,7 @@ public void testRejectedWriterCannotFlushSmallerCurrentProcessor() throws Except @Test public void testRejectedWriterFirstTabletCannotBypassSelection() throws Exception { when(currentProcessor.getWorkMemTableRamCost()).thenReturn(0L); - dataRegionInfo.getWriteMemoryManager().addMemory(900); + dataRegionInfo.addStorageGroupMemCost(900); Assert.assertFalse(systemInfo.reportStorageGroupStatus(dataRegionInfo, currentProcessor)); verify(currentProcessor, never()).setWorkMemTableShouldFlush(); @@ -106,7 +106,7 @@ public void testRejectedWriterFirstTabletCannotBypassSelection() throws Exceptio @Test public void testRepeatedReportsSubmitEachMemtableOnce() throws Exception { - dataRegionInfo.getWriteMemoryManager().addMemory(900); + dataRegionInfo.addStorageGroupMemCost(900); for (int i = 0; i < 10; i++) { Assert.assertFalse(systemInfo.reportStorageGroupStatus(dataRegionInfo, currentProcessor)); @@ -120,7 +120,7 @@ public void testRepeatedReportsSubmitEachMemtableOnce() throws Exception { @Test public void testRepeatedReportsForFirstTabletSubmitOnce() throws Exception { when(currentProcessor.getWorkMemTableRamCost()).thenReturn(0L); - dataRegionInfo.getWriteMemoryManager().addMemory(900); + dataRegionInfo.addStorageGroupMemCost(900); for (int i = 0; i < 10; i++) { Assert.assertFalse(systemInfo.reportStorageGroupStatus(dataRegionInfo, currentProcessor)); @@ -134,7 +134,7 @@ public void testRepeatedReportsForFirstTabletSubmitOnce() throws Exception { public void testNewMemtableCanBeScheduledAfterPreviousFlush() throws Exception { when(currentProcessor.getWorkMemTableRamCost()).thenReturn(700L); when(largerProcessor.getWorkMemTableRamCost()).thenReturn(100L); - dataRegionInfo.getWriteMemoryManager().addMemory(900); + dataRegionInfo.addStorageGroupMemCost(900); Assert.assertTrue(systemInfo.reportStorageGroupStatus(dataRegionInfo, currentProcessor)); mockWorkMemTable(currentProcessor); @@ -148,9 +148,9 @@ public void testNewMemtableCanBeScheduledAfterPreviousFlush() throws Exception { public void testMarkedCurrentProcessorCannotReachHardMemoryLimit() throws Exception { when(currentProcessor.getWorkMemTableRamCost()).thenReturn(700L); when(largerProcessor.getWorkMemTableRamCost()).thenReturn(100L); - dataRegionInfo.getWriteMemoryManager().addMemory(900); + dataRegionInfo.addStorageGroupMemCost(900); Assert.assertTrue(systemInfo.reportStorageGroupStatus(dataRegionInfo, currentProcessor)); - dataRegionInfo.getWriteMemoryManager().addMemory(100); + dataRegionInfo.addStorageGroupMemCost(100); try { systemInfo.reportStorageGroupStatus(dataRegionInfo, currentProcessor); @@ -166,7 +166,7 @@ public void testMarkedCurrentProcessorCannotReachHardMemoryLimit() throws Except public void testNoCurrentMemtableCannotBypassRejection() throws Exception { when(currentProcessor.getWorkMemTable()).thenReturn(null); when(currentProcessor.getWorkMemTableRamCost()).thenReturn(0L); - dataRegionInfo.getWriteMemoryManager().addMemory(900); + dataRegionInfo.addStorageGroupMemCost(900); Assert.assertFalse(systemInfo.reportStorageGroupStatus(dataRegionInfo, currentProcessor)); Assert.assertTrue(systemInfo.isRejected()); @@ -176,10 +176,10 @@ public void testNoCurrentMemtableCannotBypassRejection() throws Exception { @Test public void testRejectionClearsAfterMemoryIsReleased() throws Exception { - dataRegionInfo.getWriteMemoryManager().addMemory(900); + dataRegionInfo.addStorageGroupMemCost(900); Assert.assertFalse(systemInfo.reportStorageGroupStatus(dataRegionInfo, currentProcessor)); - dataRegionInfo.getWriteMemoryManager().releaseMemory(500); + dataRegionInfo.releaseStorageGroupMemCost(500); systemInfo.resetStorageGroupStatus(dataRegionInfo); Assert.assertFalse(systemInfo.isRejected()); Assert.assertTrue(systemInfo.reportStorageGroupStatus(dataRegionInfo, currentProcessor)); @@ -190,7 +190,7 @@ public void testRejectionClearsAfterMemoryIsReleased() throws Exception { public void testCurrentProcessorCannotExceedHardMemoryLimit() throws Exception { when(currentProcessor.getWorkMemTableRamCost()).thenReturn(700L); when(largerProcessor.getWorkMemTableRamCost()).thenReturn(100L); - dataRegionInfo.getWriteMemoryManager().addMemory(1100); + dataRegionInfo.addStorageGroupMemCost(1100); try { systemInfo.reportStorageGroupStatus(dataRegionInfo, currentProcessor); @@ -202,7 +202,7 @@ public void testCurrentProcessorCannotExceedHardMemoryLimit() throws Exception { @Test public void testNormalFlushStillSelectsLargestMemtable() throws Exception { - dataRegionInfo.getWriteMemoryManager().addMemory(700); + dataRegionInfo.addStorageGroupMemCost(700); Assert.assertTrue(systemInfo.reportStorageGroupStatus(dataRegionInfo, currentProcessor)); Assert.assertFalse(systemInfo.isRejected()); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/agent/SubscriptionConsumerAgentBrokerRecoveryTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/agent/SubscriptionConsumerAgentBrokerRecoveryTest.java new file mode 100644 index 0000000000000..d281c85d5ccc8 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/agent/SubscriptionConsumerAgentBrokerRecoveryTest.java @@ -0,0 +1,279 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.subscription.agent; + +import org.apache.iotdb.commons.pipe.agent.task.connection.UnboundedBlockingPendingQueue; +import org.apache.iotdb.commons.subscription.meta.consumer.ConsumerGroupMeta; +import org.apache.iotdb.commons.subscription.meta.consumer.ConsumerMeta; +import org.apache.iotdb.commons.subscription.meta.topic.TopicMeta; +import org.apache.iotdb.consensus.IConsensus; +import org.apache.iotdb.consensus.iot.IoTConsensus; +import org.apache.iotdb.db.consensus.DataRegionConsensusImpl; +import org.apache.iotdb.db.pipe.metric.source.PipeDataRegionEventCounter; +import org.apache.iotdb.db.subscription.broker.ISubscriptionBroker; +import org.apache.iotdb.db.subscription.broker.SubscriptionBroker; +import org.apache.iotdb.db.subscription.task.subtask.SubscriptionSinkSubtask; +import org.apache.iotdb.pipe.api.event.Event; +import org.apache.iotdb.rpc.subscription.config.ConsumerConfig; +import org.apache.iotdb.rpc.subscription.config.ConsumerConstant; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; + +import org.junit.Test; + +import java.lang.reflect.Field; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.clearInvocations; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class SubscriptionConsumerAgentBrokerRecoveryTest { + + private static final long TEST_TIMEOUT_SECONDS = 5L; + + @Test + public void testNonEmptyMetadataRecreatesMissingBroker() { + final String consumerGroupId = unique("group"); + final String consumerId = unique("consumer"); + final SubscriptionConsumerAgent agent = SubscriptionAgent.consumer(); + try { + final ConsumerGroupMeta meta = createMeta(consumerGroupId, consumerId); + assertNull(agent.handleSingleConsumerGroupMetaChanges(meta)); + assertTrue(SubscriptionAgent.broker().isBrokerExist(consumerGroupId)); + + assertTrue(SubscriptionAgent.broker().dropBroker(consumerGroupId)); + assertFalse(SubscriptionAgent.broker().isBrokerExist(consumerGroupId)); + + agent.ensureConsumerGroupBroker(consumerGroupId); + + assertTrue(SubscriptionAgent.broker().isBrokerExist(consumerGroupId)); + assertTrue(agent.isConsumerExisted(consumerGroupId, consumerId)); + } finally { + agent.handleDropConsumerGroup(consumerGroupId); + } + } + + @Test + public void testMetadataPushSetsUpNewConsensusTopicOnce() { + final String consumerGroupId = unique("group"); + final String consumerId = unique("consumer"); + final String topicName = unique("topic"); + final SubscriptionConsumerAgent agent = SubscriptionAgent.consumer(); + final IConsensus originalConsensus = DataRegionConsensusImpl.getInstance(); + final IoTConsensus consensus = mock(IoTConsensus.class); + when(consensus.getAllConsensusGroupIds()).thenReturn(Collections.emptyList()); + DataRegionConsensusImpl.setInstance(consensus); + try { + assertNull( + SubscriptionAgent.topic() + .handleSingleTopicMetaChanges( + new TopicMeta( + topicName, + 1L, + Collections.singletonMap( + TopicConstant.MODE_KEY, TopicConstant.MODE_INCREMENTAL_VALUE)))); + final ConsumerGroupMeta meta = createMeta(consumerGroupId, consumerId); + assertNull(agent.handleSingleConsumerGroupMetaChanges(meta)); + clearInvocations(consensus); + + final ConsumerGroupMeta subscribedMeta = meta.deepCopy(); + subscribedMeta.addSubscription(consumerId, Collections.singleton(topicName)); + assertNull(agent.handleSingleConsumerGroupMetaChanges(subscribedMeta)); + + // One discovery for setup logging and one for binding the topic's regions. + verify(consensus, times(2)).getAllConsensusGroupIds(); + assertEquals( + Collections.singleton(topicName), + agent.getTopicNamesSubscribedByConsumer(consumerGroupId, consumerId)); + } finally { + agent.handleDropConsumerGroup(consumerGroupId); + SubscriptionAgent.topic().handleDropTopic(topicName, false); + DataRegionConsensusImpl.setInstance(originalConsensus); + } + } + + @Test + public void testEmptyMetadataDoesNotRecreateBroker() { + final String consumerGroupId = unique("group"); + final String consumerId = unique("consumer"); + final SubscriptionConsumerAgent agent = SubscriptionAgent.consumer(); + try { + final ConsumerGroupMeta meta = createMeta(consumerGroupId, consumerId); + assertNull(agent.handleSingleConsumerGroupMetaChanges(meta)); + assertTrue(SubscriptionAgent.broker().isBrokerExist(consumerGroupId)); + + final ConsumerGroupMeta emptyMeta = meta.deepCopy(); + emptyMeta.removeConsumer(consumerId); + assertTrue(emptyMeta.isEmpty()); + assertNull(agent.handleSingleConsumerGroupMetaChanges(emptyMeta)); + + assertFalse(SubscriptionAgent.broker().isBrokerExist(consumerGroupId)); + agent.ensureConsumerGroupBroker(consumerGroupId); + assertFalse(SubscriptionAgent.broker().isBrokerExist(consumerGroupId)); + } finally { + agent.handleDropConsumerGroup(consumerGroupId); + } + } + + @Test + public void testConcurrentMetadataPushAndBrokerEnsureCreateOneBroker() throws Exception { + final String consumerGroupId = unique("group"); + final String consumerId = unique("consumer"); + final SubscriptionConsumerAgent agent = SubscriptionAgent.consumer(); + final ExecutorService executor = Executors.newFixedThreadPool(2); + try { + final ConsumerGroupMeta meta = createMeta(consumerGroupId, consumerId); + assertNull(agent.handleSingleConsumerGroupMetaChanges(meta)); + assertTrue(SubscriptionAgent.broker().dropBroker(consumerGroupId)); + + final Future metadataPush = + executor.submit(() -> agent.handleSingleConsumerGroupMetaChanges(meta.deepCopy())); + final Future handshakeRepair = + executor.submit(() -> agent.ensureConsumerGroupBroker(consumerGroupId)); + metadataPush.get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS); + handshakeRepair.get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS); + + assertEquals(1, getBrokers(consumerGroupId).size()); + } finally { + executor.shutdownNow(); + executor.awaitTermination(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS); + agent.handleDropConsumerGroup(consumerGroupId); + } + } + + @Test + public void testBrokerAndExistingQueueSurviveProviderReplacementRepair() throws Exception { + final String consumerGroupId = unique("group"); + final String consumerId = unique("consumer"); + final String topicName = unique("topic"); + final SubscriptionConsumerAgent agent = SubscriptionAgent.consumer(); + try { + assertNull( + SubscriptionAgent.topic() + .handleSingleTopicMetaChanges(new TopicMeta(topicName, 1L, Collections.emptyMap()))); + final ConsumerGroupMeta meta = createMeta(consumerGroupId, consumerId); + meta.addSubscription(consumerId, Collections.singleton(topicName)); + assertNull(agent.handleSingleConsumerGroupMetaChanges(meta)); + + bindPipeQueue(consumerGroupId, topicName); + final SubscriptionBroker brokerBeforeRepair = getPipeBroker(consumerGroupId); + assertTrue(brokerBeforeRepair.hasQueue(topicName)); + + // A provider replacement closes only its session. The server-side repair must preserve the + // existing broker and queue when it reconciles the same non-empty metadata. + agent.ensureConsumerGroupBroker(consumerGroupId); + + assertSame(brokerBeforeRepair, getPipeBroker(consumerGroupId)); + assertTrue(getPipeBroker(consumerGroupId).hasQueue(topicName)); + } finally { + agent.handleDropConsumerGroup(consumerGroupId); + SubscriptionAgent.topic().handleDropTopic(topicName, false); + } + } + + @Test + public void testCanceledTopicIsNotRestoredWhenBrokerIsRecreated() throws Exception { + final String consumerGroupId = unique("group"); + final String consumerId = unique("consumer"); + final String topicName = unique("topic"); + final SubscriptionConsumerAgent agent = SubscriptionAgent.consumer(); + try { + assertNull( + SubscriptionAgent.topic() + .handleSingleTopicMetaChanges(new TopicMeta(topicName, 1L, Collections.emptyMap()))); + final ConsumerGroupMeta meta = createMeta(consumerGroupId, consumerId); + meta.addSubscription(consumerId, Collections.singleton(topicName)); + assertNull(agent.handleSingleConsumerGroupMetaChanges(meta)); + + bindPipeQueue(consumerGroupId, topicName); + assertTrue(getPipeBroker(consumerGroupId).hasQueue(topicName)); + + final ConsumerGroupMeta unsubscribedMeta = meta.deepCopy(); + unsubscribedMeta.removeSubscription(consumerId, Collections.singleton(topicName)); + assertNull(agent.handleSingleConsumerGroupMetaChanges(unsubscribedMeta)); + assertFalse(getPipeBroker(consumerGroupId).hasQueue(topicName)); + + assertTrue(SubscriptionAgent.broker().dropBroker(consumerGroupId)); + agent.ensureConsumerGroupBroker(consumerGroupId); + + assertFalse(getPipeBroker(consumerGroupId).hasQueue(topicName)); + assertFalse(agent.getAllSubscriptions().containsKey(consumerGroupId)); + } finally { + agent.handleDropConsumerGroup(consumerGroupId); + SubscriptionAgent.topic().handleDropTopic(topicName, false); + } + } + + private static ConsumerGroupMeta createMeta( + final String consumerGroupId, final String consumerId) { + final Map attributes = new HashMap<>(); + attributes.put(ConsumerConstant.CONSUMER_ID_KEY, consumerId); + attributes.put(ConsumerConstant.CONSUMER_GROUP_ID_KEY, consumerGroupId); + final ConsumerConfig consumerConfig = new ConsumerConfig(attributes); + return new ConsumerGroupMeta( + consumerGroupId, 1L, new ConsumerMeta(consumerId, 1L, consumerConfig.getAttribute())); + } + + private static void bindPipeQueue(final String consumerGroupId, final String topicName) { + final UnboundedBlockingPendingQueue inputQueue = + new UnboundedBlockingPendingQueue<>(new PipeDataRegionEventCounter()); + final SubscriptionSinkSubtask subtask = + new SubscriptionSinkSubtask( + unique("task"), 1L, "", 0, inputQueue, null, topicName, consumerGroupId); + SubscriptionAgent.broker().bindPrefetchingQueue(subtask); + } + + @SuppressWarnings("unchecked") + private static List getBrokers(final String consumerGroupId) + throws Exception { + final Field field = SubscriptionBrokerAgent.class.getDeclaredField("consumerGroupIdToBrokers"); + field.setAccessible(true); + final Map> brokersByConsumerGroup = + (Map>) field.get(SubscriptionAgent.broker()); + return brokersByConsumerGroup.getOrDefault(consumerGroupId, Collections.emptyList()); + } + + private static SubscriptionBroker getPipeBroker(final String consumerGroupId) throws Exception { + return getBrokers(consumerGroupId).stream() + .filter(SubscriptionBroker.class::isInstance) + .map(SubscriptionBroker.class::cast) + .findFirst() + .orElseThrow(AssertionError::new); + } + + private static String unique(final String prefix) { + return prefix + '-' + UUID.randomUUID(); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionSetupHandlerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionSetupHandlerTest.java deleted file mode 100644 index 30cbd487668a7..0000000000000 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionSetupHandlerTest.java +++ /dev/null @@ -1,207 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.subscription.broker.consensus; - -import org.apache.iotdb.commons.consensus.DataRegionId; -import org.apache.iotdb.commons.pipe.config.constant.SystemConstant; -import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.rpc.subscription.config.TopicConfig; -import org.apache.iotdb.rpc.subscription.config.TopicConstant; -import org.apache.iotdb.rpc.subscription.exception.SubscriptionException; - -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -import java.io.File; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedHashSet; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; - -public class ConsensusSubscriptionSetupHandlerTest { - - @Rule public final TemporaryFolder temporaryFolder = new TemporaryFolder(); - - @Test - public void testRuntimeUpdatesAreIgnoredWhenSubscriptionIsDisabled() { - ConsensusSubscriptionSetupHandler.applyRuntimeState(null, null); - ConsensusSubscriptionSetupHandler.onRegionRouteChanged(null, 0); - } - - @Test - public void testSingleTopicSetupFailurePropagates() { - SubscriptionException failure = null; - try { - ConsensusSubscriptionSetupHandler.setupConsensusTopics( - "consumerGroup", - Collections.singleton("topic"), - topicName -> true, - topicName -> { - throw new IllegalStateException("setup failed"); - }, - ignored -> {}); - } catch (final SubscriptionException e) { - failure = e; - } - - assertNotNull(failure); - assertTrue(failure.getMessage().contains("topic")); - assertTrue(failure.getMessage().contains("consumerGroup")); - assertTrue(failure.getCause() instanceof IllegalStateException); - } - - @Test - public void testTopicModeLookupFailurePropagates() { - SubscriptionException failure = null; - try { - ConsensusSubscriptionSetupHandler.setupConsensusTopics( - "consumerGroup", - Collections.singleton("topic"), - topicName -> { - throw new IllegalStateException("topic metadata missing"); - }, - topicName -> {}, - ignored -> {}); - } catch (final SubscriptionException e) { - failure = e; - } - - assertNotNull(failure); - assertTrue(failure.getCause() instanceof IllegalStateException); - } - - @Test - public void testMultiTopicSetupFailureRollsBackAllAttemptedTopics() { - final Set attemptedTopicNames = new LinkedHashSet<>(); - final Set rolledBackTopicNames = new LinkedHashSet<>(); - final Set topicNames = new LinkedHashSet<>(Arrays.asList("first", "second", "third")); - SubscriptionException failure = null; - - try { - ConsensusSubscriptionSetupHandler.setupConsensusTopics( - "consumerGroup", - topicNames, - topicName -> true, - topicName -> failOnSecondTopic(topicName, attemptedTopicNames), - attemptedTopics -> rolledBackTopicNames.addAll(attemptedTopics)); - } catch (final SubscriptionException e) { - failure = e; - } - - assertNotNull(failure); - assertEquals(new LinkedHashSet<>(Arrays.asList("first", "second")), attemptedTopicNames); - assertEquals(attemptedTopicNames, rolledBackTopicNames); - } - - @Test - public void testFallbackLookupDoesNotQueryConfigNodeWithoutLocalPersistence() throws Exception { - final String originalSystemDir = IoTDBDescriptor.getInstance().getConfig().getSystemDir(); - final File systemDir = temporaryFolder.newFolder("noLocalPersistence"); - try { - IoTDBDescriptor.getInstance().getConfig().setSystemDir(systemDir.getAbsolutePath()); - final AtomicInteger queryCount = new AtomicInteger(); - final ConsensusSubscriptionCommitManager commitManager = - new ConsensusSubscriptionCommitManager( - (consumerGroupId, topicName, regionId) -> { - queryCount.incrementAndGet(); - return ConsensusSubscriptionCommitManager.ConfigNodeProgressQueryResult.absent(); - }); - - assertNull( - ConsensusSubscriptionSetupHandler.resolveFallbackCommittedRegionProgress( - commitManager, "consumerGroup", "topic", new DataRegionId(1))); - assertEquals(0, queryCount.get()); - } finally { - IoTDBDescriptor.getInstance().getConfig().setSystemDir(originalSystemDir); - } - } - - @Test - public void testAuditDatabaseNeverMatchesTopic() { - final Map tableTopicAttributes = new HashMap<>(); - tableTopicAttributes.put( - SystemConstant.SQL_DIALECT_KEY, SystemConstant.SQL_DIALECT_TABLE_VALUE); - tableTopicAttributes.put(TopicConstant.DATABASE_KEY, ".*"); - final TopicConfig tableTopicConfig = new TopicConfig(tableTopicAttributes); - - assertFalse( - ConsensusSubscriptionSetupHandler.matchesTopicDatabase(tableTopicConfig, "__audit")); - assertFalse( - ConsensusSubscriptionSetupHandler.matchesTopicDatabase(tableTopicConfig, "root.__audit")); - assertTrue(ConsensusSubscriptionSetupHandler.matchesTopicDatabase(tableTopicConfig, "user_db")); - - final TopicConfig treeTopicConfig = new TopicConfig(Collections.emptyMap()); - assertFalse(ConsensusSubscriptionSetupHandler.matchesTopicDatabase(treeTopicConfig, "__audit")); - assertTrue(ConsensusSubscriptionSetupHandler.matchesTopicDatabase(treeTopicConfig, "user_db")); - } - - @Test - public void testTopicDataRegionModelIsolation() { - final TopicConfig treeTopicConfig = new TopicConfig(Collections.emptyMap()); - final Map tableTopicAttributes = new HashMap<>(); - tableTopicAttributes.put( - SystemConstant.SQL_DIALECT_KEY, SystemConstant.SQL_DIALECT_TABLE_VALUE); - tableTopicAttributes.put(TopicConstant.DATABASE_KEY, "table_db"); - final TopicConfig tableTopicConfig = new TopicConfig(tableTopicAttributes); - - assertTrue( - ConsensusSubscriptionSetupHandler.matchesTopicDataRegion( - "root.tree_db", treeTopicConfig, false)); - assertFalse( - ConsensusSubscriptionSetupHandler.matchesTopicDataRegion( - "table_db", treeTopicConfig, false)); - assertFalse( - ConsensusSubscriptionSetupHandler.matchesTopicDataRegion( - "table_db", treeTopicConfig, true)); - assertFalse( - ConsensusSubscriptionSetupHandler.matchesTopicDataRegion( - "root.table_db", tableTopicConfig, true)); - assertTrue( - ConsensusSubscriptionSetupHandler.matchesTopicDataRegion( - "table_db", tableTopicConfig, true)); - assertFalse( - ConsensusSubscriptionSetupHandler.matchesTopicDataRegion( - "other_table_db", tableTopicConfig, true)); - assertFalse( - ConsensusSubscriptionSetupHandler.matchesTopicDataRegion( - "table_db", tableTopicConfig, false)); - assertFalse( - ConsensusSubscriptionSetupHandler.matchesTopicDataRegion( - "root.table_db", tableTopicConfig, false)); - } - - private static void failOnSecondTopic( - final String topicName, final Set attemptedTopicNames) { - attemptedTopicNames.add(topicName); - if ("second".equals(topicName)) { - throw new IllegalStateException("setup failed"); - } - } -} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1Test.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1Test.java index 40482fe1b20a2..05eccb978f90a 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1Test.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1Test.java @@ -221,6 +221,39 @@ public void testFencedConsumerCannotHandshakeAgain() throws Exception { Assert.assertTrue((boolean) getField(receiver, "consumerFenced")); } + @Test + public void testHandshakeRepairsMissingBrokerWithoutChangingConsumerMetadata() throws Exception { + final String consumerGroupId = "group-" + UUID.randomUUID(); + final String consumerId = "consumer-" + UUID.randomUUID(); + final ConsumerConfig consumerConfig = + createConsumerConfig(1_000L, null, null, consumerId, consumerGroupId); + final ConsumerGroupMeta consumerGroupMeta = + new ConsumerGroupMeta( + consumerGroupId, + System.currentTimeMillis(), + new ConsumerMeta( + consumerId, System.currentTimeMillis(), consumerConfig.getAttribute())); + final SubscriptionReceiverV1 receiver = new SubscriptionReceiverV1(); + Assert.assertNull( + SubscriptionAgent.consumer().handleSingleConsumerGroupMetaChanges(consumerGroupMeta)); + try { + Assert.assertTrue(SubscriptionAgent.broker().dropBroker(consumerGroupId)); + Assert.assertFalse(SubscriptionAgent.broker().isBrokerExist(consumerGroupId)); + + final TSStatus status = + receiver + .handle(PipeSubscribeHandshakeReq.toTPipeSubscribeReq(consumerConfig)) + .getStatus(); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertTrue(SubscriptionAgent.broker().isBrokerExist(consumerGroupId)); + Assert.assertTrue( + SubscriptionAgent.consumer().isConsumerExisted(consumerGroupId, consumerId)); + } finally { + SubscriptionAgent.consumer().handleDropConsumerGroup(consumerGroupId); + } + } + @Test public void testNeverHandshakenConsumerStillReturnsMissingConsumerStatus() { final SubscriptionReceiverV1 receiver = new SubscriptionReceiverV1();