From 04b06e08aeebee6ecbaa52e21617fa8090cdc0fc Mon Sep 17 00:00:00 2001 From: Umeshkumar9414 <9414umeshkumar@gmail.com> Date: Wed, 29 Jul 2026 15:44:57 +0530 Subject: [PATCH 1/3] HBASE-23996 add metric for the split wal procedure --- .../apache/hadoop/hbase/master/MetricsMasterSource.java | 4 ++++ .../hadoop/hbase/master/MetricsMasterSourceImpl.java | 7 +++++++ .../org/apache/hadoop/hbase/master/MetricsMaster.java | 9 ++++++++- .../hadoop/hbase/master/procedure/SplitWALProcedure.java | 6 ++++++ 4 files changed, 25 insertions(+), 1 deletion(-) diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSource.java index d606ed630881..a9361231ea63 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSource.java @@ -92,6 +92,7 @@ public interface MetricsMasterSource extends BaseSource { String OFFLINE_REGION_COUNT_DESC = "Number of Offline Regions"; String SERVER_CRASH_METRIC_PREFIX = "serverCrash"; + String SPLIT_WAL_METRIC_PREFIX = "splitWAL"; String OLD_WAL_DIR_SIZE_DESC = "size of old WALs directory in bytes"; /** @@ -114,4 +115,7 @@ public interface MetricsMasterSource extends BaseSource { /** Returns {@link OperationMetrics} containing common metrics for server crash operation */ OperationMetrics getServerCrashMetrics(); + + /** Returns {@link OperationMetrics} containing common metrics for split WAL operation */ + OperationMetrics getSplitWALMetrics(); } diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSourceImpl.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSourceImpl.java index 011e66312aa3..bf918ac46931 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSourceImpl.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSourceImpl.java @@ -39,6 +39,7 @@ public class MetricsMasterSourceImpl extends BaseSourceImpl implements MetricsMa private MutableFastCounter clusterWriteRequestsCounter; private OperationMetrics serverCrashMetrics; + private OperationMetrics splitWALMetrics; public MetricsMasterSourceImpl(MetricsMasterWrapper masterWrapper) { this(METRICS_NAME, METRICS_DESCRIPTION, METRICS_CONTEXT, METRICS_JMX_CONTEXT, masterWrapper); @@ -64,6 +65,7 @@ public void init() { * BaseSourceImpl#registry} to register the new metrics. */ serverCrashMetrics = new OperationMetrics(registry, SERVER_CRASH_METRIC_PREFIX); + splitWALMetrics = new OperationMetrics(registry, SPLIT_WAL_METRIC_PREFIX); } @Override @@ -144,4 +146,9 @@ public void getMetrics(MetricsCollector metricsCollector, boolean all) { public OperationMetrics getServerCrashMetrics() { return serverCrashMetrics; } + + @Override + public OperationMetrics getSplitWALMetrics() { + return splitWALMetrics; + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMaster.java index bb5fc7233f12..03bce2929547 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMaster.java @@ -43,6 +43,7 @@ public class MetricsMaster { private MetricsMasterQuotaSource masterQuotaSource; private ProcedureMetrics serverCrashProcMetrics; + private ProcedureMetrics splitWALProcMetrics; public MetricsMaster(MetricsMasterWrapper masterWrapper) { masterSource = CompatibilitySingletonFactory.getInstance(MetricsMasterSourceFactory.class) @@ -53,6 +54,7 @@ public MetricsMaster(MetricsMasterWrapper masterWrapper) { .getInstance(MetricsMasterQuotaSourceFactory.class).create(masterWrapper); serverCrashProcMetrics = convertToProcedureMetrics(masterSource.getServerCrashMetrics()); + splitWALProcMetrics = convertToProcedureMetrics(masterSource.getSplitWALMetrics()); } // for unit-test usage @@ -130,11 +132,16 @@ public void incrementQuotaObserverTime(final long executionTime) { masterQuotaSource.incrementSpaceQuotaObserverChoreTime(executionTime); } - /** Returns Set of metrics for assign procedure */ + /** Returns Set of metrics for server crash procedure */ public ProcedureMetrics getServerCrashProcMetrics() { return serverCrashProcMetrics; } + /** Returns Set of metrics for split WAL procedure */ + public ProcedureMetrics getSplitWALProcMetrics() { + return splitWALProcMetrics; + } + /** * This is utility function that converts {@link OperationMetrics} to {@link ProcedureMetrics}. * NOTE: Procedure framework in hbase-procedure module accesses metrics common to most procedures diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/SplitWALProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/SplitWALProcedure.java index 98c2c0ec6930..4171137d958e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/SplitWALProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/SplitWALProcedure.java @@ -21,6 +21,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.master.SplitWALManager; +import org.apache.hadoop.hbase.procedure2.ProcedureMetrics; import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer; import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException; import org.apache.hadoop.hbase.procedure2.ProcedureUtil; @@ -193,6 +194,11 @@ protected void afterReplay(MasterProcedureEnv env) { } } + @Override + protected ProcedureMetrics getProcedureMetrics(MasterProcedureEnv env) { + return env.getMasterServices().getMasterMetrics().getSplitWALProcMetrics(); + } + @Override protected void toStringClassDetails(StringBuilder builder) { builder.append(getProcName()); From 259d07065a2e26ccbf08a42ea047d0c2a13e0e83 Mon Sep 17 00:00:00 2001 From: Umeshkumar9414 <9414umeshkumar@gmail.com> Date: Thu, 30 Jul 2026 14:18:26 +0530 Subject: [PATCH 2/3] HBASE-23996 add tests for the split wal procedure metric Assert the new splitWAL procedure metric in two places: TestMasterMetrics#testDefaultMasterMetrics (splitWALSubmittedCount is registered and reads 0 at rest) and TestSplitWALManager#testSplitLogs (increments by one per SplitWALProcedure, verified against a captured baseline on the procedure-based non-ZK split path). Co-authored-by: Claude Opus 4.8 --- .../hadoop/hbase/master/TestMasterMetrics.java | 2 ++ .../hadoop/hbase/master/TestSplitWALManager.java | 13 +++++++++++++ 2 files changed, 15 insertions(+) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java index 46cc88825bc3..6a970e4cabf6 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java @@ -198,6 +198,8 @@ public void testDefaultMasterMetrics() throws Exception { metricsHelper.assertCounter(MetricsMasterSource.SERVER_CRASH_METRIC_PREFIX + "SubmittedCount", 0, masterSource); + metricsHelper.assertCounter(MetricsMasterSource.SPLIT_WAL_METRIC_PREFIX + "SubmittedCount", 0, + masterSource); metricsHelper.assertGauge("oldWALsDirSize", master.getMasterWalManager().getOldWALsDirSize(), masterSource); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitWALManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitWALManager.java index 76ae6345c046..16c36fdbc3fd 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitWALManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitWALManager.java @@ -31,6 +31,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.CompatibilityFactory; import org.apache.hadoop.hbase.HBaseTestingUtil; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.ServerName; @@ -45,6 +46,7 @@ import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; import org.apache.hadoop.hbase.procedure2.ProcedureYieldException; import org.apache.hadoop.hbase.procedure2.StateMachineProcedure; +import org.apache.hadoop.hbase.test.MetricsAssertHelper; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.util.Bytes; @@ -69,6 +71,8 @@ public class TestSplitWALManager { private static final Logger LOG = LoggerFactory.getLogger(TestSplitWALManager.class); + private static final MetricsAssertHelper METRICS_HELPER = + CompatibilityFactory.getInstance(MetricsAssertHelper.class); private static HBaseTestingUtil TEST_UTIL; private HMaster master; private SplitWALManager splitWALManager; @@ -231,6 +235,9 @@ private void splitLogsTestHelper(HBaseTestingUtil testUtil) throws Exception { // load table testUtil.loadTable(testUtil.getConnection().getTable(TABLE_NAME), FAMILY); ProcedureExecutor masterPE = hmaster.getMasterProcedureExecutor(); + MetricsMasterSource masterSource = hmaster.getMasterMetrics().getMetricsSource(); + long splitWALSubmittedBase = METRICS_HELPER + .getCounter(MetricsMasterSource.SPLIT_WAL_METRIC_PREFIX + "SubmittedCount", masterSource); ServerName metaServer = testUtil.getHBaseCluster().getServerHoldingMeta(); ServerName testServer = testUtil.getHBaseCluster().getRegionServerThreads().stream() .map(rs -> rs.getRegionServer().getServerName()).filter(rs -> rs != metaServer).findAny() @@ -239,6 +246,9 @@ private void splitLogsTestHelper(HBaseTestingUtil testUtil) throws Exception { assertEquals(1, procedures.size()); ProcedureTestingUtility.submitAndWait(masterPE, procedures.get(0)); assertEquals(0, splitWALManager.getWALsToSplit(testServer, false).size()); + // The SplitWALProcedure above should have been reported to the split WAL metric. + METRICS_HELPER.assertCounter(MetricsMasterSource.SPLIT_WAL_METRIC_PREFIX + "SubmittedCount", + splitWALSubmittedBase + 1, masterSource); // Validate the old WAL file archive dir Path walRootDir = hmaster.getMasterFileSystem().getWALRootDir(); @@ -251,6 +261,9 @@ private void splitLogsTestHelper(HBaseTestingUtil testUtil) throws Exception { ProcedureTestingUtility.submitAndWait(masterPE, procedures.get(0)); assertEquals(0, splitWALManager.getWALsToSplit(metaServer, true).size()); assertEquals(1, splitWALManager.getWALsToSplit(metaServer, false).size()); + // The meta SplitWALProcedure should also have been counted by the split WAL metric. + METRICS_HELPER.assertCounter(MetricsMasterSource.SPLIT_WAL_METRIC_PREFIX + "SubmittedCount", + splitWALSubmittedBase + 2, masterSource); // There should be archiveFileCount + 1 WALs after SplitWALProcedure finish assertEquals(archiveFileCount + 1, walFS.listStatus(walArchivePath).length, "Splitted WAL files should be archived"); From 4bff3da32b263ebde2ab0d8bf9ec6a7a6910be4c Mon Sep 17 00:00:00 2001 From: Umeshkumar9414 <9414umeshkumar@gmail.com> Date: Thu, 30 Jul 2026 14:18:26 +0530 Subject: [PATCH 3/3] HBASE-23996 deprecate ZK-coordinated WAL split classes and metrics Mark the legacy ZooKeeper-coordinated WAL split coordination classes (ZKSplitLogManagerCoordination, ZkSplitLogWorkerCoordination) and the associated MetricsMasterFileSystem source as deprecated, since the procedure-based distributed WAL splitter (SplitWALManager) is the default and these are only reachable under the legacy hbase.split.writer.coordinated.by.zk path. Co-authored-by: Claude Opus 4.8 --- .../hadoop/hbase/master/MetricsMasterFileSystemSource.java | 7 +++++++ .../hbase/master/MetricsMasterFilesystemSourceImpl.java | 7 +++++++ .../hbase/coordination/ZKSplitLogManagerCoordination.java | 5 ++++- .../hbase/coordination/ZkSplitLogWorkerCoordination.java | 6 ++++-- .../hadoop/hbase/master/MetricsMasterFileSystem.java | 7 +++++++ 5 files changed, 29 insertions(+), 3 deletions(-) diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterFileSystemSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterFileSystemSource.java index 579171e1c3d7..1a107af60af2 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterFileSystemSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterFileSystemSource.java @@ -21,6 +21,13 @@ import org.apache.hadoop.hbase.metrics.BaseSource; import org.apache.yetus.audience.InterfaceAudience; +/** + * @deprecated since 2.4.0 and in 3.0.0, to be removed in 4.0.0, replaced by procedure-based + * distributed WAL splitter; see SplitWALManager. These metrics ({@code hlogSplitTime}, + * {@code hlogSplitSize}, {@code metaHlogSplitTime}, {@code metaHlogSplitSize}) are only + * emitted by the deprecated ZK-coordinated WAL split path. + */ +@Deprecated @InterfaceAudience.Private public interface MetricsMasterFileSystemSource extends BaseSource { diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterFilesystemSourceImpl.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterFilesystemSourceImpl.java index dc2a2824269b..b44c3b1decd1 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterFilesystemSourceImpl.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterFilesystemSourceImpl.java @@ -21,6 +21,13 @@ import org.apache.hadoop.metrics2.MetricHistogram; import org.apache.yetus.audience.InterfaceAudience; +/** + * @deprecated since 2.4.0 and in 3.0.0, to be removed in 4.0.0, replaced by procedure-based + * distributed WAL splitter; see SplitWALManager. Implementation of the deprecated + * {@link MetricsMasterFileSystemSource}, whose metrics are only emitted by the + * ZK-coordinated WAL split path. + */ +@Deprecated @InterfaceAudience.Private public class MetricsMasterFilesystemSourceImpl extends BaseSourceImpl implements MetricsMasterFileSystemSource { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/ZKSplitLogManagerCoordination.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/ZKSplitLogManagerCoordination.java index c243251e595e..6d6697781bbf 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/ZKSplitLogManagerCoordination.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/ZKSplitLogManagerCoordination.java @@ -58,8 +58,11 @@ import org.slf4j.LoggerFactory; /** - * ZooKeeper based implementation of {@link SplitLogManagerCoordination} + * ZooKeeper based implementation of {@link SplitLogManagerCoordination}. + * @deprecated since 2.4.0 and in 3.0.0, to be removed in 4.0.0, replaced by procedure-based + * distributed WAL splitter; see SplitWALManager. */ +@Deprecated @InterfaceAudience.Private public class ZKSplitLogManagerCoordination extends ZKListener implements SplitLogManagerCoordination { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/ZkSplitLogWorkerCoordination.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/ZkSplitLogWorkerCoordination.java index 6def70f97143..f670c3fd41fd 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/ZkSplitLogWorkerCoordination.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/ZkSplitLogWorkerCoordination.java @@ -57,9 +57,11 @@ import org.slf4j.LoggerFactory; /** - * ZooKeeper based implementation of {@link SplitLogWorkerCoordination} It listen for changes in - * ZooKeeper and + * ZooKeeper based implementation of {@link SplitLogWorkerCoordination}. + * @deprecated since 2.4.0 and in 3.0.0, to be removed in 4.0.0, replaced by procedure-based + * distributed WAL splitter; see SplitWALManager. */ +@Deprecated @InterfaceAudience.Private public class ZkSplitLogWorkerCoordination extends ZKListener implements SplitLogWorkerCoordination { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterFileSystem.java index 7b7348519ddd..bf6d6b8bb196 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterFileSystem.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterFileSystem.java @@ -20,6 +20,13 @@ import org.apache.hadoop.hbase.CompatibilitySingletonFactory; import org.apache.yetus.audience.InterfaceAudience; +/** + * @deprecated since 2.4.0 and in 3.0.0, to be removed in 4.0.0, replaced by procedure-based + * distributed WAL splitter; see SplitWALManager. These metrics ({@code hlogSplitTime}, + * {@code hlogSplitSize}, {@code metaHlogSplitTime}, {@code metaHlogSplitSize}) are only + * emitted by the deprecated ZK-coordinated WAL split path. + */ +@Deprecated @InterfaceAudience.Private public class MetricsMasterFileSystem {