From 7a5273c1179608aa5fd2f18f9234f4f6c224a817 Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Wed, 2 Sep 2026 13:01:28 -0400 Subject: [PATCH 1/6] Work around DB failure, add logging for safemode failure --- .../org/apache/hadoop/hdds/scm/pipeline/Pipeline.java | 9 ++++++++- .../hadoop/hdds/scm/pipeline/PipelineReportHandler.java | 2 ++ .../apache/hadoop/hdds/upgrade/HddsUpgradeTestUtils.java | 5 ++++- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java index 8c084d7c8ebe..d8b26d11aa6c 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java @@ -350,10 +350,17 @@ public boolean isHealthy() { } for (Long reportedTime : nodeStatus.values()) { if (reportedTime < 0) { + LOG.info("---Pipeline {} reported time was negative {}", this, reportedTime); return false; } } - return leaderId != null; + if (leaderId == null) { + LOG.info("---Pipeline {} leader was null", this); + return false; + } else { + LOG.info("---Pipeline {} is healthy", this); + return true; + } } public boolean isEmpty() { diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineReportHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineReportHandler.java index 9b9d08751574..fbe8ead31c9b 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineReportHandler.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineReportHandler.java @@ -132,6 +132,8 @@ protected void processPipelineReport(PipelineReport report, LOGGER.info("Opened pipeline {}", pipelineID); } } + LOGGER.info("---Received pipeline {} report on {}", pipeline, + this.scmContext.getScm().getScmNodeDetails().getNodeId()); if (pipeline.isHealthy()) { if (scmSafeModeManager.getInSafeMode()) { publisher.fireEvent(SCMEvents.OPEN_PIPELINE, pipeline); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/HddsUpgradeTestUtils.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/HddsUpgradeTestUtils.java index 7fcc4cd53a58..6bdef90dc6f1 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/HddsUpgradeTestUtils.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/HddsUpgradeTestUtils.java @@ -133,7 +133,10 @@ private static boolean isScmFinalized(StorageContainerManager scm, boolean waitF try { dbKeyFlushed = scm.getScmMetadataStore().getMetaTable().get(OzoneConsts.APPARENT_VERSION_KEY) != null; } catch (RocksDatabaseException | CodecException e) { - throw new RuntimeException(e); + // The metadata RocksDB is briefly closed while a Ratis snapshot install reloads the checkpoint. + // Treat that as "not finalized yet" and let waitFor retry rather than failing the test. + LOG.info("SCM {} metadata DB not readable yet (snapshot install in progress?), will retry.", + scm.getSCMNodeId(), e); } LOG.info("Waiting for SCM {} (leader? {}) to finalize.\n" + From c485f283f288e576c43e4044a4df4939f3ba5891 Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Wed, 2 Sep 2026 14:14:47 -0400 Subject: [PATCH 2/6] Revert "Work around DB failure, add logging for safemode failure" This reverts commit 7a5273c1179608aa5fd2f18f9234f4f6c224a817. --- .../org/apache/hadoop/hdds/scm/pipeline/Pipeline.java | 9 +-------- .../hadoop/hdds/scm/pipeline/PipelineReportHandler.java | 2 -- .../apache/hadoop/hdds/upgrade/HddsUpgradeTestUtils.java | 5 +---- 3 files changed, 2 insertions(+), 14 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java index d8b26d11aa6c..8c084d7c8ebe 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java @@ -350,17 +350,10 @@ public boolean isHealthy() { } for (Long reportedTime : nodeStatus.values()) { if (reportedTime < 0) { - LOG.info("---Pipeline {} reported time was negative {}", this, reportedTime); return false; } } - if (leaderId == null) { - LOG.info("---Pipeline {} leader was null", this); - return false; - } else { - LOG.info("---Pipeline {} is healthy", this); - return true; - } + return leaderId != null; } public boolean isEmpty() { diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineReportHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineReportHandler.java index fbe8ead31c9b..9b9d08751574 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineReportHandler.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineReportHandler.java @@ -132,8 +132,6 @@ protected void processPipelineReport(PipelineReport report, LOGGER.info("Opened pipeline {}", pipelineID); } } - LOGGER.info("---Received pipeline {} report on {}", pipeline, - this.scmContext.getScm().getScmNodeDetails().getNodeId()); if (pipeline.isHealthy()) { if (scmSafeModeManager.getInSafeMode()) { publisher.fireEvent(SCMEvents.OPEN_PIPELINE, pipeline); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/HddsUpgradeTestUtils.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/HddsUpgradeTestUtils.java index 6bdef90dc6f1..7fcc4cd53a58 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/HddsUpgradeTestUtils.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/HddsUpgradeTestUtils.java @@ -133,10 +133,7 @@ private static boolean isScmFinalized(StorageContainerManager scm, boolean waitF try { dbKeyFlushed = scm.getScmMetadataStore().getMetaTable().get(OzoneConsts.APPARENT_VERSION_KEY) != null; } catch (RocksDatabaseException | CodecException e) { - // The metadata RocksDB is briefly closed while a Ratis snapshot install reloads the checkpoint. - // Treat that as "not finalized yet" and let waitFor retry rather than failing the test. - LOG.info("SCM {} metadata DB not readable yet (snapshot install in progress?), will retry.", - scm.getSCMNodeId(), e); + throw new RuntimeException(e); } LOG.info("Waiting for SCM {} (leader? {}) to finalize.\n" + From 3b476c5ec8e0abe76f608c44985f793e41167413 Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Wed, 2 Sep 2026 14:15:04 -0400 Subject: [PATCH 3/6] Gracefully handle snapshot install in-progress while checking DB for finalized state --- .../org/apache/hadoop/hdds/upgrade/HddsUpgradeTestUtils.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/HddsUpgradeTestUtils.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/HddsUpgradeTestUtils.java index 7fcc4cd53a58..6bdef90dc6f1 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/HddsUpgradeTestUtils.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/HddsUpgradeTestUtils.java @@ -133,7 +133,10 @@ private static boolean isScmFinalized(StorageContainerManager scm, boolean waitF try { dbKeyFlushed = scm.getScmMetadataStore().getMetaTable().get(OzoneConsts.APPARENT_VERSION_KEY) != null; } catch (RocksDatabaseException | CodecException e) { - throw new RuntimeException(e); + // The metadata RocksDB is briefly closed while a Ratis snapshot install reloads the checkpoint. + // Treat that as "not finalized yet" and let waitFor retry rather than failing the test. + LOG.info("SCM {} metadata DB not readable yet (snapshot install in progress?), will retry.", + scm.getSCMNodeId(), e); } LOG.info("Waiting for SCM {} (leader? {}) to finalize.\n" + From b4ca75477d7d1acae93057be3fc71d4013e71fc1 Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Wed, 2 Sep 2026 14:23:11 -0400 Subject: [PATCH 4/6] Remove safemode exit as criteria --- .../org/apache/hadoop/hdds/upgrade/HddsUpgradeTestUtils.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/HddsUpgradeTestUtils.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/HddsUpgradeTestUtils.java index 6bdef90dc6f1..31752abee562 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/HddsUpgradeTestUtils.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/HddsUpgradeTestUtils.java @@ -140,12 +140,13 @@ private static boolean isScmFinalized(StorageContainerManager scm, boolean waitF } LOG.info("Waiting for SCM {} (leader? {}) to finalize.\n" + - "Exited safemode? {}\n" + + "Exited safemode? (not required) {}\n" + "version manager finalized? {}\n" + "DB key flushed? {}\n" + "Requiring DB key to flush? {}", scm.getSCMNodeId(), scm.checkLeader(), exitedSafemode, isFinalized, dbKeyFlushed, waitForDBKeyFlush); - return exitedSafemode && isFinalized && (!waitForDBKeyFlush || dbKeyFlushed); + // Safemode exit status is included for logging purposes, but SCMs can still finalize while in safemode. + return isFinalized && (!waitForDBKeyFlush || dbKeyFlushed); } } From 4779634ab4f87cdf12928e360ab3022c4619f384 Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Thu, 3 Sep 2026 14:01:05 -0400 Subject: [PATCH 5/6] Add more specific logs --- .../hadoop/hdds/scm/pipeline/PipelineReportHandler.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineReportHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineReportHandler.java index 9b9d08751574..5b9d88319a5f 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineReportHandler.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineReportHandler.java @@ -85,6 +85,9 @@ public void onMessage(PipelineReportFromDatanode pipelineReportFromDatanode, if (!isNotLeaderException(e)) { LOGGER.error("Could not process pipeline report={} from dn={}.", report, dn, e); + } else { + LOGGER.info("NLE prevented pipeline from being processed on {}", + scmContext.getScm().getScmNodeDetails().getNodeId(), e); } } } @@ -128,8 +131,10 @@ protected void processPipelineReport(PipelineReport report, dn); } if (pipeline.isHealthy()) { + String nodeID = scmContext.getScm().getScmNodeDetails().getNodeId(); + LOGGER.info("Attempting to open pipeline {} on {}", pipeline, nodeID); pipelineManager.openPipeline(pipelineID); - LOGGER.info("Opened pipeline {}", pipelineID); + LOGGER.info("Opened pipeline {} on {}", pipeline, nodeID); } } if (pipeline.isHealthy()) { From 85b2b98fe95234408544cc708ee0080db3a1aeb7 Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Thu, 3 Sep 2026 14:41:50 -0400 Subject: [PATCH 6/6] Revert "Add more specific logs" This reverts commit 4779634ab4f87cdf12928e360ab3022c4619f384. --- .../hadoop/hdds/scm/pipeline/PipelineReportHandler.java | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineReportHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineReportHandler.java index 5b9d88319a5f..9b9d08751574 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineReportHandler.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineReportHandler.java @@ -85,9 +85,6 @@ public void onMessage(PipelineReportFromDatanode pipelineReportFromDatanode, if (!isNotLeaderException(e)) { LOGGER.error("Could not process pipeline report={} from dn={}.", report, dn, e); - } else { - LOGGER.info("NLE prevented pipeline from being processed on {}", - scmContext.getScm().getScmNodeDetails().getNodeId(), e); } } } @@ -131,10 +128,8 @@ protected void processPipelineReport(PipelineReport report, dn); } if (pipeline.isHealthy()) { - String nodeID = scmContext.getScm().getScmNodeDetails().getNodeId(); - LOGGER.info("Attempting to open pipeline {} on {}", pipeline, nodeID); pipelineManager.openPipeline(pipelineID); - LOGGER.info("Opened pipeline {} on {}", pipeline, nodeID); + LOGGER.info("Opened pipeline {}", pipelineID); } } if (pipeline.isHealthy()) {