[client] Throw Exception when retry several times when downloading log.

fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogFetchBuffer.java

@@ -18,8 +18,10 @@
 package org.apache.fluss.client.table.scanner.log;
 
 import org.apache.fluss.annotation.Internal;
+import org.apache.fluss.exception.FetchException;
 import org.apache.fluss.exception.WakeupException;
 import org.apache.fluss.metadata.TableBucket;
+import org.apache.fluss.utils.ExceptionUtils;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -73,17 +75,19 @@ public class LogFetchBuffer implements AutoCloseable {
     @GuardedBy("lock")
     private @Nullable CompletedFetch nextInLineFetch;
 
+    @GuardedBy("lock")
+    private @Nullable Throwable throwable;
+
     public LogFetchBuffer() {
         this.completedFetches = new LinkedList<>();
     }
 
     /**
-     * Returns {@code true} if there are no completed fetches pending to return to the user.
-     *
-     * @return {@code true} if the buffer is empty, {@code false} otherwise
+     * @return {@code true} if there are no completed fetches pending to return to the user and no
+     *     error has been recorded, {@code false} otherwise
      */
     boolean isEmpty() {
-        return inLock(lock, completedFetches::isEmpty);
+        return inLock(lock, () -> completedFetches.isEmpty() && throwable == null);
     }
 
     void pend(PendingFetch pendingFetch) {
@@ -100,10 +104,16 @@ void pend(PendingFetch pendingFetch) {
      * Tries to complete the pending fetches in order, convert them into completed fetches in the
      * buffer.
      */
-    void tryComplete(TableBucket tableBucket) {
+    void tryComplete(TableBucket tableBucket, Throwable t) {
         inLock(
                 lock,
                 () -> {
+                    if (t != null) {
+                        this.throwable = t;
+                        notEmptyCondition.signalAll();
+                        return;
+                    }
+
                     boolean hasCompleted = false;
                     LinkedList<PendingFetch> pendings = this.pendingFetches.get(tableBucket);
                     while (pendings != null && !pendings.isEmpty()) {
@@ -157,12 +167,22 @@ void setNextInLineFetch(@Nullable CompletedFetch nextInLineFetch) {
         inLock(lock, () -> this.nextInLineFetch = nextInLineFetch);
     }
 
-    CompletedFetch peek() {
-        return inLock(lock, completedFetches::peek);
+    CompletedFetch peek() throws FetchException {
+        return inLock(
+                lock,
+                () -> {
+                    checkException();
+                    return completedFetches.peek();
+                });
     }
 
-    CompletedFetch poll() {
-        return inLock(lock, completedFetches::poll);
+    CompletedFetch poll() throws FetchException {
+        return inLock(
+                lock,
+                () -> {
+                    checkException();
+                    return completedFetches.poll();
+                });
     }
 
     /**
@@ -282,6 +302,12 @@ Set<TableBucket> pendedBuckets() {
         return inLock(lock, pendingFetches::keySet);
     }
 
+    void checkException() throws FetchException {
+        if (throwable != null) {
+            throw new FetchException(ExceptionUtils.stripCompletionException(throwable));
+        }
+    }
+
     @Override
     public void close() throws Exception {
         inLock(lock, () -> retainAll(Collections.emptySet()));

fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogFetcher.java

@@ -560,7 +560,8 @@ private void pendRemoteFetches(
                             logScannerStatus,
                             isCheckCrcs);
             logFetchBuffer.pend(pendingFetch);
-            downloadFuture.onComplete(() -> logFetchBuffer.tryComplete(segment.tableBucket()));
+            downloadFuture.whenComplete(
+                    (throwable) -> logFetchBuffer.tryComplete(segment.tableBucket(), throwable));
         }
     }
 

fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/RemoteLogDownloadFuture.java

@@ -24,6 +24,7 @@
 import java.io.IOException;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Consumer;
 
 /** Represents the future of a remote log download request. */
 public class RemoteLogDownloadFuture {
@@ -77,4 +78,13 @@ public void discard() {
     public void onComplete(Runnable callback) {
         logFileFuture.thenRun(callback);
     }
+
+    public void whenComplete(Consumer<Throwable> callback) {
+        logFileFuture.whenComplete(
+                (file, throwable) -> {
+                    if (!logFileFuture.isCancelled()) {
+                        callback.accept(throwable);
+                    }
+                });
+    }
 }

fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/RemoteLogDownloader.java

@@ -28,6 +28,7 @@
 import org.apache.fluss.metadata.TableBucket;
 import org.apache.fluss.remote.RemoteLogSegment;
 import org.apache.fluss.utils.ExceptionUtils;
+import org.apache.fluss.utils.ExponentialBackoff;
 import org.apache.fluss.utils.FlussPaths;
 import org.apache.fluss.utils.concurrent.ShutdownableThread;
 
@@ -62,6 +63,17 @@ public class RemoteLogDownloader implements Closeable {
     private static final Logger LOG = LoggerFactory.getLogger(RemoteLogDownloader.class);
 
     private static final long POLL_TIMEOUT = 5000L;
+    private static final long RETRY_BACKOFF_INITIAL_MS = 100L;
+    private static final int RETRY_BACKOFF_MULTIPLIER = 2;
+    private static final long RETRY_BACKOFF_MAX_MS = 5000L;
+    private static final double RETRY_BACKOFF_JITTER = 0.25D;
+    private static final ExponentialBackoff RETRY_BACKOFF =
+            new ExponentialBackoff(
+                    RETRY_BACKOFF_INITIAL_MS,
+                    RETRY_BACKOFF_MULTIPLIER,
+                    RETRY_BACKOFF_MAX_MS,
+                    RETRY_BACKOFF_JITTER);
+    private final int maxRetryCount;
 
     private final Path localLogDir;
 
@@ -107,6 +119,7 @@ public RemoteLogDownloader(
         this.remoteFileDownloader = remoteFileDownloader;
         this.scannerMetricGroup = scannerMetricGroup;
         this.pollTimeout = pollTimeout;
+        this.maxRetryCount = conf.getInt(ConfigOptions.CLIENT_SCANNER_REMOTE_LOG_FETCH_MAX_RETRIES);
         this.prefetchSemaphore =
                 new Semaphore(conf.getInt(ConfigOptions.CLIENT_SCANNER_REMOTE_LOG_PREFETCH_NUM));
         // The local tmp dir to store the fetched log segment files,
@@ -171,7 +184,21 @@ void fetchOnce() throws Exception {
             return;
         }
 
-        TableBucket tableBucket = request.getTableBucket();
+        downloadRemoteLog(request, maxRetryCount, System.currentTimeMillis());
+    }
+
+    private void downloadRemoteLog(
+            RemoteLogDownloadRequest request, int retryCount, long startTime) {
+        if (closed || request.future.isCancelled()) {
+            if (!request.future.isDone()) {
+                request.future.cancel(false);
+            }
+            prefetchSemaphore.release();
+            if (closed) {
+                deleteDirectoryQuietly(localLogDir.toFile());
+            }
+            return;
+        }
         try {
             // 1. cleanup the finished logs first to free up disk space
             cleanupRemoteLogs();
@@ -180,7 +207,6 @@ void fetchOnce() throws Exception {
             FsPathAndFileName fsPathAndFileName = request.getFsPathAndFileName();
             scannerMetricGroup.remoteFetchRequestCount().inc();
 
-            long startTime = System.currentTimeMillis();
             // download the remote file to local
             CompletableFuture<Long> completableFuture =
                     remoteFileDownloader.downloadFileAsync(fsPathAndFileName, localLogDir);
@@ -190,7 +216,7 @@ void fetchOnce() throws Exception {
                             LOG.warn(
                                     "RemoteLogDownloader closed when remote log segment file {} for table bucket {}.",
                                     fsPathAndFileName.getFileName(),
-                                    tableBucket);
+                                    request.getTableBucket());
                             // In-flight download completed after close. Cancel the
                             // external future so consumers blocked on .get() are
                             // unblocked, release the semaphore (no consumer will
@@ -211,20 +237,13 @@ void fetchOnce() throws Exception {
                             return;
                         }
                         if (throwable != null) {
-                            LOG.error(
-                                    "Failed to download remote log segment file {} for table bucket {}.",
-                                    fsPathAndFileName.getFileName(),
-                                    tableBucket,
-                                    ExceptionUtils.stripExecutionException(throwable));
-                            prefetchSemaphore.release();
-                            segmentsToFetch.add(request);
-                            scannerMetricGroup.remoteFetchErrorCount().inc();
+                            handleFetchException(request, throwable, retryCount, startTime);
                         } else {
                             LOG.info(
                                     "Successfully downloaded remote log segment file {} to local for "
                                             + "table bucket {} cost {} ms.",
                                     fsPathAndFileName.getFileName(),
-                                    tableBucket,
+                                    request.getTableBucket(),
                                     System.currentTimeMillis() - startTime);
                             File localFile =
                                     new File(localLogDir.toFile(), fsPathAndFileName.getFileName());
@@ -236,16 +255,70 @@ void fetchOnce() throws Exception {
                         }
                     });
         } catch (Throwable t) {
-            prefetchSemaphore.release();
-            // only re-queue the request if the downloader is still active
-            if (!closed && !request.future.isCancelled()) {
-                segmentsToFetch.add(request);
-            } else if (!request.future.isDone()) {
+            if (closed || request.future.isCancelled()) {
+                if (!request.future.isDone()) {
+                    request.future.cancel(false);
+                }
+                prefetchSemaphore.release();
+                if (closed) {
+                    deleteDirectoryQuietly(localLogDir.toFile());
+                }
+            } else {
+                handleFetchException(request, t, retryCount, startTime);
+            }
+        }
+    }
+
+    private void handleFetchException(
+            RemoteLogDownloadRequest request, Throwable throwable, int retryCount, long startTime) {
+        if (closed || request.future.isCancelled()) {
+            if (!request.future.isDone()) {
                 request.future.cancel(false);
             }
-            scannerMetricGroup.remoteFetchErrorCount().inc();
-            // log the error and continue instead of shutdown the download thread
-            LOG.error("Failed to download remote log segment for table bucket {}.", tableBucket, t);
+            prefetchSemaphore.release();
+            if (closed) {
+                deleteDirectoryQuietly(localLogDir.toFile());
+            }
+            return;
+        }
+
+        LOG.error(
+                "Failed to download remote log segment file {} for table bucket {}.",
+                request.getFsPathAndFileName().getFileName(),
+                request.getTableBucket(),
+                ExceptionUtils.stripExecutionException(throwable));
+        scannerMetricGroup.remoteFetchErrorCount().inc();
+        if (retryCount >= 1) {
+            long backoffMs = RETRY_BACKOFF.backoff(maxRetryCount - retryCount);
+            LOG.warn(
+                    "Retrying download of remote log segment file {} for table bucket {} "
+                            + "in {} ms (retry {}/{}).",
+                    request.getFsPathAndFileName().getFileName(),
+                    request.getTableBucket(),
+                    backoffMs,
+                    maxRetryCount - retryCount + 1,
+                    maxRetryCount);
+            try {
+                Thread.sleep(backoffMs);
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+                prefetchSemaphore.release();
+                request.future.completeExceptionally(
+                        new IOException(
+                                "Interrupted while retrying download of remote log segment file "
+                                        + request.getFsPathAndFileName().getFileName(),
+                                e));
+                return;
+            }
+            downloadRemoteLog(request, retryCount - 1, startTime);
+        } else {
+            prefetchSemaphore.release();
+            request.future.completeExceptionally(
+                    new IOException(
+                            String.format(
+                                    "Failed to download remote log segment file %s, retry count %d",
+                                    request.getFsPathAndFileName().getFileName(), maxRetryCount),
+                            throwable));
         }
     }
 

fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchBufferTest.java

@@ -17,6 +17,7 @@
 
 package org.apache.fluss.client.table.scanner.log;
 
+import org.apache.fluss.exception.FetchException;
 import org.apache.fluss.exception.WakeupException;
 import org.apache.fluss.metadata.TableBucket;
 import org.apache.fluss.record.LogRecordReadContext;
@@ -26,6 +27,7 @@
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 
+import java.io.IOException;
 import java.time.Duration;
 import java.util.Arrays;
 import java.util.Collections;
@@ -46,6 +48,7 @@
 import static org.apache.fluss.record.TestData.TEST_SCHEMA_GETTER;
 import static org.apache.fluss.testutils.DataTestUtils.genMemoryLogRecordsByObject;
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
 
 /** Test for {@link LogFetchBuffer}. */
 public class LogFetchBufferTest {
@@ -259,15 +262,15 @@ void testPendFetches() throws Exception {
 
             Future<Boolean> signal =
                     service.submit(() -> await(logFetchBuffer, Duration.ofSeconds(1)));
-            logFetchBuffer.tryComplete(pending1.tableBucket());
+            logFetchBuffer.tryComplete(pending1.tableBucket(), null);
             // nothing happen, as pending1 is not completed
             assertThat(logFetchBuffer.isEmpty()).isTrue();
             // no condition signal
             assertThat(signal.get()).isFalse();
 
             signal = service.submit(() -> await(logFetchBuffer, Duration.ofMinutes(1)));
             completed1.set(true);
-            logFetchBuffer.tryComplete(pending1.tableBucket());
+            logFetchBuffer.tryComplete(pending1.tableBucket(), null);
             assertThat(signal.get()).isTrue();
             assertThat(logFetchBuffer.isEmpty()).isFalse();
             assertThat(logFetchBuffer.poll().tableBucket).isEqualTo(tableBucket1);
@@ -277,18 +280,48 @@ void testPendFetches() throws Exception {
 
             signal = service.submit(() -> await(logFetchBuffer, Duration.ofMinutes(1)));
             completed2.set(true);
-            logFetchBuffer.tryComplete(pending2.tableBucket());
+            logFetchBuffer.tryComplete(pending2.tableBucket(), null);
             assertThat(signal.get()).isTrue();
             assertThat(logFetchBuffer.isEmpty()).isFalse();
-            logFetchBuffer.tryComplete(pending3.tableBucket());
-            logFetchBuffer.tryComplete(pending4.tableBucket());
+            logFetchBuffer.tryComplete(pending3.tableBucket(), null);
+            logFetchBuffer.tryComplete(pending4.tableBucket(), null);
             assertThat(logFetchBuffer.poll().tableBucket).isEqualTo(tableBucket2);
             assertThat(logFetchBuffer.poll().tableBucket).isEqualTo(tableBucket3);
             assertThat(logFetchBuffer.poll().tableBucket).isEqualTo(tableBucket3);
             assertThat(logFetchBuffer.isEmpty()).isTrue();
         }
     }
 
+    @Test
+    void testFetchException() throws Exception {
+        ExecutorService service = Executors.newSingleThreadExecutor();
+        try (LogFetchBuffer logFetchBuffer = new LogFetchBuffer()) {
+            AtomicBoolean completed = new AtomicBoolean(false);
+            PendingFetch pendingFetch = makePendingFetch(tableBucket1, completed);
+
+            logFetchBuffer.tryComplete(pendingFetch.tableBucket(), null);
+            assertThat(logFetchBuffer.isEmpty()).isTrue();
+            logFetchBuffer.pend(pendingFetch);
+            assertThat(logFetchBuffer.isEmpty()).isTrue();
+
+            Future<Boolean> signal =
+                    service.submit(() -> await(logFetchBuffer, Duration.ofMinutes(1)));
+            completed.set(true);
+            logFetchBuffer.tryComplete(
+                    pendingFetch.tableBucket(), new IOException("Test fetch exception"));
+            assertThat(signal.get()).isTrue();
+            assertThat(logFetchBuffer.isEmpty()).isFalse();
+            assertThatThrownBy(logFetchBuffer::poll)
+                    .isExactlyInstanceOf(FetchException.class)
+                    .hasMessageContaining("Test fetch exception");
+            assertThatThrownBy(logFetchBuffer::peek)
+                    .isExactlyInstanceOf(FetchException.class)
+                    .hasMessageContaining("Test fetch exception");
+        } finally {
+            service.shutdownNow();
+        }
+    }
+
     private boolean await(LogFetchBuffer buffer, Duration waitTime) throws InterruptedException {
         return buffer.awaitNotEmpty(System.nanoTime() + waitTime.toNanos());
     }