Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,9 @@ abstract class AbstractSubscriptionConsumer implements AutoCloseable {
private static final long SLEEP_MS = 100L;
private static final long SLEEP_DELTA_MS = 50L;
private static final long TIMER_DELTA_MS = 250L;
private static final int MAX_COMMIT_RETRIES = 3;
private static final long COMMIT_RETRY_INTERVAL_MS =
ConsumerConstant.ENDPOINTS_SYNC_INTERVAL_MS_MIN_VALUE;
private static final AtomicLong LAST_CONSUMER_INSTANCE_EPOCH = new AtomicLong();

private final String username;
Expand Down Expand Up @@ -1060,9 +1063,7 @@ private Optional<SubscriptionMessage> pollFile(
inFlightFilesCommitContextSet.remove(commitContext);
}
// construct temporary message to nack
nack(
Collections.singletonList(
new SubscriptionMessage(commitContext, file.getAbsolutePath(), null)));
nackAfterPollFailure(new SubscriptionMessage(commitContext, file.getAbsolutePath(), null), e);
throw new SubscriptionRuntimeNonCriticalException(e.getMessage(), e);
}
}
Expand Down Expand Up @@ -1271,13 +1272,25 @@ private Optional<SubscriptionMessage> pollTablets(
throw (SubscriptionConsumerFencedException) e;
}
// construct temporary message to nack
nack(
Collections.singletonList(
new SubscriptionMessage(response.getCommitContext(), Collections.emptyMap())));
nackAfterPollFailure(
new SubscriptionMessage(response.getCommitContext(), Collections.emptyMap()), e);
throw new SubscriptionRuntimeNonCriticalException(e.getMessage(), e);
}
}

private void nackAfterPollFailure(final SubscriptionMessage message, final Exception pollFailure)
throws SubscriptionException {
try {
nack(Collections.singletonList(message));
} catch (final SubscriptionConnectionException nackFailure) {
// Recovery was scheduled by commitInternal. A NACK replay can increment the server's poison
// message counter again. Preserve the poll failure and let the server recycle the event.
if (nackFailure != pollFailure) {
pollFailure.addSuppressed(nackFailure);
}
}
}

private Optional<SubscriptionMessage> pollTabletsInternal(
final SubscriptionPollResponse initialResponse, final PollTimer timer) {
final Map<String, List<Tablet>> tablets =
Expand Down Expand Up @@ -1448,6 +1461,7 @@ private List<SubscriptionPollResponse> pollFileInternal(
if (isClosed()) {
return Collections.emptyList();
}
providers.syncIfUnavailable(this);
throw new SubscriptionConnectionException(
String.format(
SubscriptionMessages
Expand All @@ -1459,6 +1473,7 @@ private List<SubscriptionPollResponse> pollFileInternal(
try {
return provider.pollFile(commitContext, writingOffset, timeoutMs);
} catch (final SubscriptionConnectionException ignored) {
providers.syncIfUnavailable(this);
return Collections.emptyList();
}
} finally {
Expand All @@ -1477,6 +1492,7 @@ private List<SubscriptionPollResponse> pollTabletsInternal(
if (isClosed()) {
return Collections.emptyList();
}
providers.syncIfUnavailable(this);
throw new SubscriptionConnectionException(
String.format(
SubscriptionMessages
Expand All @@ -1488,6 +1504,7 @@ private List<SubscriptionPollResponse> pollTabletsInternal(
try {
return provider.pollTablets(commitContext, offset, timeoutMs);
} catch (final SubscriptionConnectionException ignored) {
providers.syncIfUnavailable(this);
return Collections.emptyList();
}
} finally {
Expand Down Expand Up @@ -1532,15 +1549,9 @@ private void commit(final Iterable<SubscriptionCommitContext> commitContexts, fi
dataNodeIdToSubscriptionCommitContexts.entrySet()) {
final List<SubscriptionCommitContext> groupedCommitContexts = entry.getValue();
requestedCount += groupedCommitContexts.size();
final AbstractSubscriptionProvider.CommitResult commitResp =
commitInternal(entry.getKey(), groupedCommitContexts, nack);
final List<SubscriptionCommitContext> acceptedCommitContexts =
commitResp.getAcceptedCommitContexts();
commitWithRetry(entry.getKey(), groupedCommitContexts, nack);
acceptedCount += acceptedCommitContexts.size();
if (!nack) {
overlayCommittedPositions(commitResp.getCommittedProgressByTopic());
onAckedCommitContexts(acceptedCommitContexts);
}
if (acceptedCommitContexts.size() != groupedCommitContexts.size()) {
final List<SubscriptionCommitContext> failedInGroup =
new ArrayList<>(groupedCommitContexts);
Expand All @@ -1561,6 +1572,88 @@ private void commit(final Iterable<SubscriptionCommitContext> commitContexts, fi
}
}

private List<SubscriptionCommitContext> commitWithRetry(
final int dataNodeId,
final List<SubscriptionCommitContext> commitContexts,
final boolean nack)
throws SubscriptionException {
final List<SubscriptionCommitContext> remainingCommitContexts = new ArrayList<>(commitContexts);
final List<SubscriptionCommitContext> acceptedCommitContexts = new ArrayList<>();
for (int retries = 0; ; retries++) {
checkIfFenced();
AbstractSubscriptionProvider.CommitResult result =
AbstractSubscriptionProvider.CommitResult.empty();
SubscriptionConnectionException connectionFailure = null;
SubscriptionException failure = null;
try {
result = commitInternal(dataNodeId, remainingCommitContexts, nack);
} catch (final SubscriptionException e) {
failure = getCommitFailure(e);
result = getCompletedCommitResult(e);
if (failure instanceof SubscriptionConnectionException) {
connectionFailure = (SubscriptionConnectionException) failure;
}
}

final List<SubscriptionCommitContext> acceptedInAttempt = result.getAcceptedCommitContexts();
acceptedCommitContexts.addAll(acceptedInAttempt);
if (!nack) {
// Apply each response once, before waiting. Reapplying older progress at retry completion
// could overwrite a concurrent commit; accepted contexts must also leave the buffer now.
overlayCommittedPositions(result.getCommittedProgressByTopic());
onAckedCommitContexts(acceptedInAttempt);
}
if (failure != null && connectionFailure == null) {
throw failure;
}
remainingCommitContexts.removeAll(new HashSet<>(acceptedInAttempt));
if (nack
|| remainingCommitContexts.isEmpty()
|| retries >= MAX_COMMIT_RETRIES
|| isClosed()
|| Thread.currentThread().isInterrupted()) {
if (connectionFailure != null) {
throw connectionFailure;
}
return acceptedCommitContexts;
}
checkIfFenced();
// A partial response has no permanent-rejection reason. Retry only its rejected contexts
// and any batches whose response was lost, using the same budget as connection recovery.
// commitInternal has released the provider read lock before we wait for recovery.
try {
Thread.sleep(COMMIT_RETRY_INTERVAL_MS);
} catch (final InterruptedException interruptedException) {
Thread.currentThread().interrupt();
if (connectionFailure != null) {
connectionFailure.addSuppressed(interruptedException);
throw connectionFailure;
}
return acceptedCommitContexts;
}
checkIfFenced();
if (isClosed()) {
if (connectionFailure != null) {
throw connectionFailure;
}
return acceptedCommitContexts;
}
}
}

private SubscriptionException getCommitFailure(final SubscriptionException e) {
return e instanceof AbstractSubscriptionProvider.CommitException
? ((AbstractSubscriptionProvider.CommitException) e).getFailure()
: e;
}

private AbstractSubscriptionProvider.CommitResult getCompletedCommitResult(
final SubscriptionException e) {
return e instanceof AbstractSubscriptionProvider.CommitException
? ((AbstractSubscriptionProvider.CommitException) e).getCompletedResult()
: AbstractSubscriptionProvider.CommitResult.empty();
}

/** Invoked after the server accepts commit contexts from an acknowledgement request. */
protected void onAckedCommitContexts(
final Collection<SubscriptionCommitContext> acceptedCommitContexts) {
Expand Down Expand Up @@ -1606,10 +1699,26 @@ protected Set<SubscriptionCommitContext> ackCommitContextsWithPartialProgress(
commitInternal(entry.getKey(), groupedCommitContexts, false);
overlayCommittedPositions(commitResp.getCommittedProgressByTopic());
removableCommitContexts.addAll(commitResp.getAcceptedCommitContexts());
} catch (final SubscriptionConnectionException e) {
} catch (final SubscriptionException e) {
final SubscriptionException failure = getCommitFailure(e);
final AbstractSubscriptionProvider.CommitResult completedResult =
getCompletedCommitResult(e);
overlayCommittedPositions(completedResult.getCommittedProgressByTopic());
final Set<SubscriptionCommitContext> acceptedBeforeFailure =
new HashSet<>(completedResult.getAcceptedCommitContexts());
removableCommitContexts.addAll(acceptedBeforeFailure);
if (!(failure instanceof SubscriptionConnectionException)) {
// The caller cannot receive the removable set when an error propagates. Remove confirmed
// contexts from every completed group before leaving the automatic/close-time ACK path.
onAckedCommitContexts(removableCommitContexts);
throw failure;
}
int stagedCount = 0;
int retainedCount = 0;
for (final SubscriptionCommitContext commitContext : groupedCommitContexts) {
if (acceptedBeforeFailure.contains(commitContext)) {
continue;
}
if (isConsensusCommitContext(commitContext)) {
stagePendingRedirectAck(commitContext);
removableCommitContexts.add(commitContext);
Expand All @@ -1633,7 +1742,7 @@ protected Set<SubscriptionCommitContext> ackCommitContextsWithPartialProgress(
this,
retainedCount,
entry.getKey(),
e);
failure);
}
}
}
Expand Down Expand Up @@ -1672,13 +1781,15 @@ private AbstractSubscriptionProvider.CommitResult commitInternal(
final List<SubscriptionCommitContext> subscriptionCommitContexts,
final boolean nack)
throws SubscriptionException {
checkIfFenced();
providers.acquireReadLock();
try {
final AbstractSubscriptionProvider provider = providers.getProvider(dataNodeId);
if (Objects.isNull(provider) || !provider.isAvailable()) {
if (isClosed()) {
return AbstractSubscriptionProvider.CommitResult.empty();
}
providers.syncIfUnavailable(this);
throw new SubscriptionConnectionException(
String.format(
SubscriptionMessages
Expand All @@ -1689,8 +1800,13 @@ private AbstractSubscriptionProvider.CommitResult commitInternal(
}
try {
return provider.commit(subscriptionCommitContexts, nack);
} catch (final SubscriptionConsumerFencedException e) {
fence(e);
} catch (final SubscriptionException e) {
final SubscriptionException failure = getCommitFailure(e);
if (failure instanceof SubscriptionConsumerFencedException) {
fence((SubscriptionConsumerFencedException) failure);
} else if (failure instanceof SubscriptionConnectionException) {
providers.syncIfUnavailable(this);
}
throw e;
}
} finally {
Expand Down Expand Up @@ -2082,15 +2198,11 @@ private void flushPendingRedirectAcks(final List<SubscriptionMessage> currentMes
}

final List<SubscriptionCommitContext> contextsToRedirect = new ArrayList<>(pendingContexts);
AbstractSubscriptionProvider.CommitResult commitResp;
try {
final AbstractSubscriptionProvider.CommitResult commitResp =
commitInternal(entry.getValue(), contextsToRedirect, false);
overlayCommittedPositions(commitResp.getCommittedProgressByTopic());
commitResp.getAcceptedCommitContexts().forEach(pendingContexts::remove);
if (pendingContexts.isEmpty()) {
pendingRedirectAcksByTopicRegion.remove(entry.getKey(), pendingContexts);
}
commitResp = commitInternal(entry.getValue(), contextsToRedirect, false);
} catch (final SubscriptionException e) {
commitResp = getCompletedCommitResult(e);
LOGGER.warn(
SubscriptionMessages
.LOG_ARG_FAILED_REDIRECT_ARG_PENDING_CONSENSUS_ACK_S_ARG_VIA_E6A10AC5,
Expand All @@ -2100,6 +2212,11 @@ private void flushPendingRedirectAcks(final List<SubscriptionMessage> currentMes
entry.getValue(),
e);
}
overlayCommittedPositions(commitResp.getCommittedProgressByTopic());
commitResp.getAcceptedCommitContexts().forEach(pendingContexts::remove);
if (pendingContexts.isEmpty()) {
pendingRedirectAcksByTopicRegion.remove(entry.getKey(), pendingContexts);
}
}
}

Expand Down Expand Up @@ -2154,7 +2271,24 @@ private void overlayCommittedPositions(final Map<String, TopicProgress> progress
if (Objects.isNull(progressByTopic) || progressByTopic.isEmpty()) {
return;
}
progressByTopic.forEach(this::overlayCommittedPositions);
// ACK responses from different providers can arrive out of order, including completed batches
// carried by a failure. Preserve the newest position for each writer. Explicit seek operations
// use the topic-scoped overlay above and can still move a checkpoint backward.
for (final Entry<String, TopicProgress> entry : progressByTopic.entrySet()) {
if (Objects.isNull(entry.getValue())) {
continue;
}
for (final RegionProgress regionProgress : entry.getValue().getRegionProgress().values()) {
if (Objects.nonNull(regionProgress)) {
regionProgress
.getWriterPositions()
.forEach(
(writerId, writerProgress) ->
mergeTopicProgress(
committedPositionsByTopic, entry.getKey(), writerId, writerProgress));
}
}
}
}

private void overlayTopicProgress(
Expand Down
Loading
Loading