diff --git a/hbase/Dockerfile b/hbase/Dockerfile index 2f7113b14..0d60e847b 100644 --- a/hbase/Dockerfile +++ b/hbase/Dockerfile @@ -1,6 +1,10 @@ # syntax=docker/dockerfile:1.16.0@sha256:e2dd261f92e4b763d789984f6eab84be66ab4f5f08052316d8eb8f173593acf7 # check=error=true +# Selects the final-phoenix-* stage below. Set to "false" for HBase versions that Phoenix does not +# support yet (HBase 3.x), in which case hbase/phoenix must not be listed in boil-config.toml. +ARG WITH_PHOENIX=true + FROM local-image/hadoop/hadoop AS hadoop-builder FROM local-image/hbase/hbase AS hbase-builder @@ -50,21 +54,19 @@ rm /stackable/bin/export-snapshot-to-s3.env chmod --recursive g=u /stackable EOF -# Final Image -FROM local-image/java-base AS final +# Final Image, without Phoenix (added by final-phoenix-true) +FROM local-image/java-base AS final-base ARG PRODUCT_VERSION ARG RELEASE_VERSION ARG HADOOP_HADOOP_VERSION # Reassign the arg to `HADOOP_VERSION` for better readability. ENV HADOOP_VERSION=${HADOOP_HADOOP_VERSION} -ARG HBASE_PROFILE ARG HBASE_HBASE_VERSION # Reassign the arg to `HBASE_VERSION` for better readability. ENV HBASE_VERSION=${HBASE_HBASE_VERSION} ARG HBASE_HBASE_OPERATOR_TOOLS_VERSION ARG HBASE_HBASE_OPA_AUTHORIZER_VERSION -ARG HBASE_PHOENIX_VERSION ARG STACKABLE_USER_UID ARG NAME="Apache HBase" @@ -98,9 +100,6 @@ COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-operator-tools /stackable/hbas COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-operator-tools /stackable/bin/hbck2 /stackable/bin/hbck2 COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-operator-tools /stackable/bin/hbase-entrypoint.sh /stackable/hbase-${HBASE_VERSION}-stackable${RELEASE_VERSION}/bin/hbase-entrypoint.sh -COPY --chown=${STACKABLE_USER_UID}:0 --from=phoenix /stackable/phoenix /stackable/phoenix/ -COPY --chown=${STACKABLE_USER_UID}:0 --from=phoenix /stackable/phoenix-${HBASE_PHOENIX_VERSION}-stackable${RELEASE_VERSION}-src.tar.gz /stackable - COPY --chown=${STACKABLE_USER_UID}:0 --from=hadoop-s3-builder /stackable/bin/export-snapshot-to-s3 /stackable/bin/export-snapshot-to-s3 COPY --chown=${STACKABLE_USER_UID}:0 --from=hadoop-s3-builder /stackable/hadoop/share/hadoop/tools/lib/ /stackable/hadoop/share/hadoop/tools/lib/ @@ -130,23 +129,52 @@ ln --symbolic --logical --verbose "/stackable/hbase-${HBASE_VERSION}-stackable${ chown --no-dereference ${STACKABLE_USER_UID}:0 /stackable/hbase chmod g=u /stackable/hbase +# bin/hbase defaults HBASE_LOG_DIR to $HBASE_HOME/logs, which the web UI serves as /logs. The HBase +# tarball does not contain that directory. Jetty 9 (HBase 2.x) accepts a missing directory, but +# Jetty 12 (HBase 3.x) refuses to start the HTTP server ("Base Resource is not valid"). +mkdir --parents "/stackable/hbase-${HBASE_VERSION}-stackable${RELEASE_VERSION}/logs" +chown ${STACKABLE_USER_UID}:0 "/stackable/hbase-${HBASE_VERSION}-stackable${RELEASE_VERSION}/logs" +chmod g=u "/stackable/hbase-${HBASE_VERSION}-stackable${RELEASE_VERSION}/logs" + ln --symbolic --logical --verbose "/stackable/hbase-operator-tools-${HBASE_HBASE_OPERATOR_TOOLS_VERSION}-stackable${RELEASE_VERSION}" /stackable/hbase-operator-tools chown --no-dereference ${STACKABLE_USER_UID}:0 /stackable/hbase-operator-tools chmod g=u /stackable/hbase-operator-tools -ln --symbolic --logical --verbose "/stackable/phoenix/phoenix-server-hbase-${HBASE_PROFILE}.jar" "/stackable/hbase/lib/phoenix-server-hbase-${HBASE_PROFILE}.jar" -chown --no-dereference ${STACKABLE_USER_UID}:0 "/stackable/hbase/lib/phoenix-server-hbase-${HBASE_PROFILE}.jar" -chmod g=u "/stackable/hbase/lib/phoenix-server-hbase-${HBASE_PROFILE}.jar" - # fix missing permissions chmod g=u /stackable/async-profiler chmod g=u /stackable/bin -chmod g=u /stackable/phoenix chmod g=u /stackable/*-src.tar.gz # the whole directory tree /stackable/hadoop/share/hadoop/tools/lib/ must be adapted find /stackable/hadoop -type d -exec chmod g=u {} + EOF +FROM final-base AS final-phoenix-true + +ARG RELEASE_VERSION +ARG HBASE_PROFILE +ARG HBASE_PHOENIX_VERSION +ARG STACKABLE_USER_UID + +COPY --chown=${STACKABLE_USER_UID}:0 --from=phoenix /stackable/phoenix /stackable/phoenix/ +COPY --chown=${STACKABLE_USER_UID}:0 --from=phoenix /stackable/phoenix-${HBASE_PHOENIX_VERSION}-stackable${RELEASE_VERSION}-src.tar.gz /stackable + +RUN < -Date: Tue, 6 Feb 2024 16:10:54 +0100 -Subject: HBASE-28242: Updates async-profiler support - ---- - .../hadoop/hbase/http/ProfileServlet.java | 205 +++++++++++------- - 1 file changed, 121 insertions(+), 84 deletions(-) - -diff --git a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/ProfileServlet.java b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/ProfileServlet.java -index 122d04ff17..5c2df0076d 100644 ---- a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/ProfileServlet.java -+++ b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/ProfileServlet.java -@@ -28,9 +28,11 @@ import java.util.concurrent.TimeUnit; - import java.util.concurrent.atomic.AtomicInteger; - import java.util.concurrent.locks.Lock; - import java.util.concurrent.locks.ReentrantLock; -+ - import javax.servlet.http.HttpServlet; - import javax.servlet.http.HttpServletRequest; - import javax.servlet.http.HttpServletResponse; -+ - import org.apache.hadoop.hbase.util.ProcessUtils; - import org.apache.yetus.audience.InterfaceAudience; - import org.slf4j.Logger; -@@ -40,23 +42,60 @@ import org.apache.hbase.thirdparty.com.google.common.base.Joiner; - - /** - * Servlet that runs async-profiler as web-endpoint. Following options from async-profiler can be -- * specified as query paramater. // -e event profiling event: cpu|alloc|lock|cache-misses etc. // -d -- * duration run profiling for 'duration' seconds (integer) // -i interval sampling interval in -- * nanoseconds (long) // -j jstackdepth maximum Java stack depth (integer) // -b bufsize frame -- * buffer size (long) // -t profile different threads separately // -s simple class names instead of -- * FQN // -o fmt[,fmt...] output format: summary|traces|flat|collapsed|svg|tree|jfr|html // --width -- * px SVG width pixels (integer) // --height px SVG frame height pixels (integer) // --minwidth px -- * skip frames smaller than px (double) // --reverse generate stack-reversed FlameGraph / Call tree -- * Example: - To collect 30 second CPU profile of current process (returns FlameGraph svg) curl -- * "http://localhost:10002/prof" - To collect 1 minute CPU profile of current process and output in -- * tree format (html) curl "http://localhost:10002/prof?output=tree&duration=60" - To collect 30 -- * second heap allocation profile of current process (returns FlameGraph svg) curl -- * "http://localhost:10002/prof?event=alloc" - To collect lock contention profile of current process -- * (returns FlameGraph svg) curl "http://localhost:10002/prof?event=lock" Following event types are -- * supported (default is 'cpu') (NOTE: not all OS'es support all events) // Perf events: // cpu // -- * page-faults // context-switches // cycles // instructions // cache-references // cache-misses // -- * branches // branch-misses // bus-cycles // L1-dcache-load-misses // LLC-load-misses // -- * dTLB-load-misses // mem:breakpoint // trace:tracepoint // Java events: // alloc // lock -+ * specified as query parameter. -+ * -+ * Example: -+ * -+ * Following event types are supported (default is 'cpu') (NOTE: not all OS'es support all -+ * events).
-+ * Basic events: -+ * -+ * Perf events: -+ * - */ - @InterfaceAudience.Private - public class ProfileServlet extends HttpServlet { -@@ -81,19 +120,20 @@ public class ProfileServlet extends HttpServlet { - WALL("wall"), - ALLOC("alloc"), - LOCK("lock"), -- PAGE_FAULTS("page-faults"), -+ ITIMER("itimer"), -+ BRANCH_INSTRUCTIONS("branch-instructions"), -+ BRANCH_MISSES("branch-misses"), -+ BUS_CYCLES("bus-cycles"), -+ CACHE_MISSES("cache-misses"), -+ CACHE_REFERENCES("cache-references"), - CONTEXT_SWITCHES("context-switches"), - CYCLES("cycles"), -+ DTLB_LOAD_MISSES("dTLB-load-misses"), - INSTRUCTIONS("instructions"), -- CACHE_REFERENCES("cache-references"), -- CACHE_MISSES("cache-misses"), -- BRANCHES("branches"), -- BRANCH_MISSES("branch-misses"), -- BUS_CYCLES("bus-cycles"), - L1_DCACHE_LOAD_MISSES("L1-dcache-load-misses"), - LLC_LOAD_MISSES("LLC-load-misses"), -- DTLB_LOAD_MISSES("dTLB-load-misses"), - MEM_BREAKPOINT("mem:breakpoint"), -+ PAGE_FAULTS("page-faults"), - TRACE_TRACEPOINT("trace:tracepoint"),; - - private final String internalName; -@@ -102,11 +142,11 @@ public class ProfileServlet extends HttpServlet { - this.internalName = internalName; - } - -- public String getInternalName() { -+ String getInternalName() { - return internalName; - } - -- public static Event fromInternalName(final String name) { -+ static Event fromInternalName(final String name) { - for (Event event : values()) { - if (event.getInternalName().equalsIgnoreCase(name)) { - return event; -@@ -117,30 +157,26 @@ public class ProfileServlet extends HttpServlet { - } - } - -- enum Output { -- SUMMARY, -- TRACES, -- FLAT, -+ private enum Output { - COLLAPSED, -- // No SVG in 2.x asyncprofiler. -- SVG, -- TREE, -+ FLAMEGRAPH, -+ FLAT, - JFR, -- // In 2.x asyncprofiler, this is how you get flamegraphs. -- HTML -+ TRACES, -+ TREE - } - - @edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "SE_TRANSIENT_FIELD_NOT_RESTORED", - justification = "This class is never serialized nor restored.") -- private transient Lock profilerLock = new ReentrantLock(); -+ private final transient Lock profilerLock = new ReentrantLock(); - private transient volatile Process process; -- private String asyncProfilerHome; -+ private final String asyncProfilerHome; - private Integer pid; - - public ProfileServlet() { - this.asyncProfilerHome = getAsyncProfilerHome(); - this.pid = ProcessUtils.getPid(); -- LOG.info("Servlet process PID: " + pid + " asyncProfilerHome: " + asyncProfilerHome); -+ LOG.info("Servlet process PID: {} asyncProfilerHome: {}", pid, asyncProfilerHome); - } - - @Override -@@ -159,9 +195,9 @@ public class ProfileServlet extends HttpServlet { - setResponseHeader(resp); - resp.getWriter() - .write("ASYNC_PROFILER_HOME env is not set.\n\n" -- + "Please ensure the prerequsites for the Profiler Servlet have been installed and the\n" -+ + "Please ensure the prerequisites for the Profiler Servlet have been installed and the\n" - + "environment is properly configured. For more information please see\n" -- + "http://hbase.apache.org/book.html#profiler\n"); -+ + "https://hbase.apache.org/book.html#profiler\n"); - return; - } - -@@ -177,18 +213,18 @@ public class ProfileServlet extends HttpServlet { - return; - } - -- final int duration = getInteger(req, "duration", DEFAULT_DURATION_SECONDS); -- final Output output = getOutput(req); -- final Event event = getEvent(req); -- final Long interval = getLong(req, "interval"); -- final Integer jstackDepth = getInteger(req, "jstackdepth", null); -- final Long bufsize = getLong(req, "bufsize"); -- final boolean thread = req.getParameterMap().containsKey("thread"); -- final boolean simple = req.getParameterMap().containsKey("simple"); -- final Integer width = getInteger(req, "width", null); -- final Integer height = getInteger(req, "height", null); -- final Double minwidth = getMinWidth(req); -- final boolean reverse = req.getParameterMap().containsKey("reverse"); -+ Event event = getEvent(req); -+ int duration = getInteger(req, "duration", DEFAULT_DURATION_SECONDS); -+ Long interval = getLong(req, "interval"); -+ Integer jstackDepth = getInteger(req, "jstackdepth", null); -+ boolean thread = req.getParameterMap().containsKey("thread"); -+ boolean simple = req.getParameterMap().containsKey("simple"); -+ boolean signature = req.getParameterMap().containsKey("signature"); -+ boolean annotate = req.getParameterMap().containsKey("annotate"); -+ boolean prependLib = req.getParameterMap().containsKey("prependlib"); -+ Output output = getOutput(req); -+ Double minwidth = getMinWidth(req); -+ boolean reverse = req.getParameterMap().containsKey("reverse"); - - if (process == null || !process.isAlive()) { - try { -@@ -209,11 +245,7 @@ public class ProfileServlet extends HttpServlet { - cmd.add("-e"); - cmd.add(event.getInternalName()); - cmd.add("-d"); -- cmd.add("" + duration); -- cmd.add("-o"); -- cmd.add(output.name().toLowerCase()); -- cmd.add("-f"); -- cmd.add(outputFile.getAbsolutePath()); -+ cmd.add(String.valueOf(duration)); - if (interval != null) { - cmd.add("-i"); - cmd.add(interval.toString()); -@@ -222,24 +254,25 @@ public class ProfileServlet extends HttpServlet { - cmd.add("-j"); - cmd.add(jstackDepth.toString()); - } -- if (bufsize != null) { -- cmd.add("-b"); -- cmd.add(bufsize.toString()); -- } - if (thread) { - cmd.add("-t"); - } - if (simple) { - cmd.add("-s"); - } -- if (width != null) { -- cmd.add("--width"); -- cmd.add(width.toString()); -+ if (signature) { -+ cmd.add("-g"); - } -- if (height != null) { -- cmd.add("--height"); -- cmd.add(height.toString()); -+ if (annotate) { -+ cmd.add("-a"); - } -+ if (prependLib) { -+ cmd.add("-l"); -+ } -+ cmd.add("-o"); -+ cmd.add(output.name().toLowerCase()); -+ cmd.add("-f"); -+ cmd.add(outputFile.getAbsolutePath()); - if (minwidth != null) { - cmd.add("--minwidth"); - cmd.add(minwidth.toString()); -@@ -247,6 +280,7 @@ public class ProfileServlet extends HttpServlet { - if (reverse) { - cmd.add("--reverse"); - } -+ - cmd.add(pid.toString()); - process = ProcessUtils.runCmdAsync(cmd); - -@@ -257,7 +291,10 @@ public class ProfileServlet extends HttpServlet { - resp.getWriter() - .write("Started [" + event.getInternalName() - + "] profiling. This page will automatically redirect to " + relativeUrl + " after " -- + duration + " seconds.\n\nCommand:\n" + Joiner.on(" ").join(cmd)); -+ + duration + " seconds. " -+ + "If empty diagram and Linux 4.6+, see 'Basic Usage' section on the Async " -+ + "Profiler Home Page, https://github.com/jvm-profiling-tools/async-profiler." -+ + "\n\nCommand:\n" + Joiner.on(" ").join(cmd)); - - // to avoid auto-refresh by ProfileOutputServlet, refreshDelay can be specified - // via url param -@@ -275,8 +312,9 @@ public class ProfileServlet extends HttpServlet { - resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); - resp.getWriter() - .write("Unable to acquire lock. Another instance of profiler might be running."); -- LOG.warn("Unable to acquire lock in " + lockTimeoutSecs -- + " seconds. Another instance of profiler might be running."); -+ LOG.warn( -+ "Unable to acquire lock in {} seconds. Another instance of profiler might be running.", -+ lockTimeoutSecs); - } - } catch (InterruptedException e) { - LOG.warn("Interrupted while acquiring profile lock.", e); -@@ -289,9 +327,9 @@ public class ProfileServlet extends HttpServlet { - } - } - -- private Integer getInteger(final HttpServletRequest req, final String param, -+ private static Integer getInteger(final HttpServletRequest req, final String param, - final Integer defaultValue) { -- final String value = req.getParameter(param); -+ String value = req.getParameter(param); - if (value != null) { - try { - return Integer.valueOf(value); -@@ -302,8 +340,8 @@ public class ProfileServlet extends HttpServlet { - return defaultValue; - } - -- private Long getLong(final HttpServletRequest req, final String param) { -- final String value = req.getParameter(param); -+ private static Long getLong(final HttpServletRequest req, final String param) { -+ String value = req.getParameter(param); - if (value != null) { - try { - return Long.valueOf(value); -@@ -314,8 +352,8 @@ public class ProfileServlet extends HttpServlet { - return null; - } - -- private Double getMinWidth(final HttpServletRequest req) { -- final String value = req.getParameter("minwidth"); -+ private static Double getMinWidth(final HttpServletRequest req) { -+ String value = req.getParameter("minwidth"); - if (value != null) { - try { - return Double.valueOf(value); -@@ -326,8 +364,8 @@ public class ProfileServlet extends HttpServlet { - return null; - } - -- private Event getEvent(final HttpServletRequest req) { -- final String eventArg = req.getParameter("event"); -+ private static Event getEvent(final HttpServletRequest req) { -+ String eventArg = req.getParameter("event"); - if (eventArg != null) { - Event event = Event.fromInternalName(eventArg); - return event == null ? Event.CPU : event; -@@ -335,16 +373,16 @@ public class ProfileServlet extends HttpServlet { - return Event.CPU; - } - -- private Output getOutput(final HttpServletRequest req) { -- final String outputArg = req.getParameter("output"); -+ private static Output getOutput(final HttpServletRequest req) { -+ String outputArg = req.getParameter("output"); - if (req.getParameter("output") != null) { - try { - return Output.valueOf(outputArg.trim().toUpperCase()); - } catch (IllegalArgumentException e) { -- return Output.HTML; -+ return Output.FLAMEGRAPH; - } - } -- return Output.HTML; -+ return Output.FLAMEGRAPH; - } - - static void setResponseHeader(final HttpServletResponse response) { -@@ -376,8 +414,7 @@ public class ProfileServlet extends HttpServlet { - .write("The profiler servlet was disabled at startup.\n\n" - + "Please ensure the prerequisites for the Profiler Servlet have been installed and the\n" - + "environment is properly configured. For more information please see\n" -- + "http://hbase.apache.org/book.html#profiler\n"); -- return; -+ + "https://hbase.apache.org/book.html#profiler\n"); - } - - } diff --git a/hbase/hbase/stackable/patches/2.6.4/0007-HBASE-29797-Should-not-create-wal-directory-when-cre.patch b/hbase/hbase/stackable/patches/2.6.4/0007-HBASE-29797-Should-not-create-wal-directory-when-cre.patch deleted file mode 100644 index 7368c6f0b..000000000 --- a/hbase/hbase/stackable/patches/2.6.4/0007-HBASE-29797-Should-not-create-wal-directory-when-cre.patch +++ /dev/null @@ -1,672 +0,0 @@ -From 4eb9a746551f3567cd80cdeb25205ef4a86c6668 Mon Sep 17 00:00:00 2001 -From: Sebastian Bernauer -Date: Thu, 5 Feb 2026 14:32:03 +0100 -Subject: HBASE-29797 Should not create wal directory when creating WAL - instance - -Cherry-picked from https://github.com/apache/hbase/pull/7589 ---- - hbase-server/pom.xml | 5 ++ - .../hbase/master/region/MasterRegion.java | 2 +- - .../hbase/regionserver/HRegionServer.java | 4 +- - .../hbase/regionserver/wal/AbstractFSWAL.java | 6 +- - .../apache/hadoop/hbase/wal/WALFactory.java | 34 +++++++- - .../hadoop/hbase/HBaseTestingUtility.java | 5 +- - .../hadoop/hbase/master/TestWALFencing.java | 81 +++++++++++++++++++ - .../regionserver/TestCompactionPolicy.java | 1 + - .../regionserver/TestFailedAppendAndSync.java | 1 + - .../hbase/regionserver/TestHRegion.java | 22 ++--- - .../hbase/regionserver/TestLogRoller.java | 5 +- - .../hbase/regionserver/TestWALLockup.java | 2 + - .../regionserver/wal/AbstractTestFSWAL.java | 7 +- - .../wal/AbstractTestWALReplay.java | 1 + - .../wal/TestAsyncFSWALDurability.java | 2 +- - .../wal/TestAsyncFSWALRollStuck.java | 1 + - .../regionserver/wal/TestAsyncWALReplay.java | 4 +- - .../hbase/regionserver/wal/TestFSHLog.java | 3 + - .../wal/TestFSHLogDurability.java | 2 +- - .../hbase/regionserver/wal/TestWALReplay.java | 4 +- - .../wal/WALDurabilityTestBase.java | 8 +- - .../hadoop/hbase/wal/TestWALFactory.java | 15 ++-- - .../hadoop/hbase/wal/TestWALSplitToHFile.java | 5 +- - pom.xml | 7 ++ - 24 files changed, 190 insertions(+), 37 deletions(-) - create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestWALFencing.java - -diff --git a/hbase-server/pom.xml b/hbase-server/pom.xml -index 6585117cc5..64e3c3a477 100644 ---- a/hbase-server/pom.xml -+++ b/hbase-server/pom.xml -@@ -312,6 +312,11 @@ - junit-vintage-engine - test - -+ -+ org.awaitility -+ awaitility -+ test -+ - - org.mockito - mockito-core -diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegion.java -index 7fd7f6fa5c..0cef601d5b 100644 ---- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegion.java -+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegion.java -@@ -468,7 +468,7 @@ public final class MasterRegion { - params.archivedWalSuffix(), params.rollPeriodMs(), params.flushSize()); - walRoller.start(); - -- WALFactory walFactory = new WALFactory(conf, server.getServerName().toString()); -+ WALFactory walFactory = new WALFactory(conf, server.getServerName().toString(), server); - Path tableDir = CommonFSUtils.getTableDir(rootDir, td.getTableName()); - Path initializingFlag = new Path(tableDir, INITIALIZING_FLAG); - Path initializedFlag = new Path(tableDir, INITIALIZED_FLAG); -diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java -index f9ef9d8473..ec0f94975e 100644 ---- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java -+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java -@@ -2126,8 +2126,8 @@ public class HRegionServer extends Thread - throw new RegionServerRunningException( - "Region server has already created directory at " + this.serverName.toString()); - } -- // Always create wal directory as now we need this when master restarts to find out the live -- // region servers. -+ // Create wal directory here and we will never create it again in other places. This is -+ // important to make sure that our fencing way takes effect. See HBASE-29797 for more details. - if (!this.walFs.mkdirs(logDir)) { - throw new IOException("Can not create wal directory " + logDir); - } -diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java -index 76e292bd85..efa151dd6f 100644 ---- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java -+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java -@@ -424,10 +424,8 @@ public abstract class AbstractFSWAL implements WAL { - this.conf = conf; - this.abortable = abortable; - -- if (!fs.exists(walDir) && !fs.mkdirs(walDir)) { -- throw new IOException("Unable to mkdir " + walDir); -- } -- -+ // Here we only crate archive dir, without wal dir. This is to make sure that our fencing way -+ // takes effect. See HBASE-29797 for more details. - if (!fs.exists(this.walArchiveDir)) { - if (!fs.mkdirs(this.walArchiveDir)) { - throw new IOException("Unable to mkdir " + this.walArchiveDir); -diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java -index 11a6c18042..cedeb46f43 100644 ---- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java -+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java -@@ -17,6 +17,7 @@ - */ - package org.apache.hadoop.hbase.wal; - -+import com.google.errorprone.annotations.RestrictedApi; - import java.io.IOException; - import java.io.InterruptedIOException; - import java.util.List; -@@ -31,6 +32,7 @@ import org.apache.hadoop.hbase.regionserver.wal.MetricsWAL; - import org.apache.hadoop.hbase.regionserver.wal.ProtobufWALStreamReader; - import org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader; - import org.apache.hadoop.hbase.util.CancelableProgressable; -+import org.apache.hadoop.hbase.util.CommonFSUtils; - import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; - import org.apache.hadoop.hbase.util.LeaseNotRecoveredException; - import org.apache.hadoop.hbase.wal.WALProvider.Writer; -@@ -203,8 +205,10 @@ public class WALFactory { - return provider; - } - -+ @RestrictedApi(explanation = "Should only be called in tests", link = "", -+ allowedOnPath = ".*/src/test/.*|.*/HBaseTestingUtility.java|.*/WALPerformanceEvaluation.java") - public WALFactory(Configuration conf, String factoryId) throws IOException { -- this(conf, factoryId, null); -+ this(conf, factoryId, null, true); - } - - /** -@@ -213,6 +217,30 @@ public class WALFactory { - * @param abortable the server to abort - */ - public WALFactory(Configuration conf, String factoryId, Abortable abortable) throws IOException { -+ this(conf, factoryId, abortable, false); -+ } -+ -+ private static void createWALDirectory(Configuration conf, String factoryId) throws IOException { -+ FileSystem walFs = CommonFSUtils.getWALFileSystem(conf); -+ Path walRootDir = CommonFSUtils.getWALRootDir(conf); -+ Path walDir = new Path(walRootDir, AbstractFSWALProvider.getWALDirectoryName(factoryId)); -+ if (!walFs.exists(walDir) && !walFs.mkdirs(walDir)) { -+ throw new IOException("Can not create wal directory " + walDir); -+ } -+ } -+ -+ /** -+ * @param conf must not be null, will keep a reference to read params in later -+ * reader/writer instances. -+ * @param factoryId a unique identifier for this factory. used i.e. by filesystem -+ * implementations to make a directory -+ * @param abortable the server associated with this WAL file -+ * @param createWalDirectory pass {@code true} for testing purpose, to create the wal directory -+ * automatically. In normal code path, we should create it in -+ * HRegionServer setup. -+ */ -+ private WALFactory(Configuration conf, String factoryId, Abortable abortable, -+ boolean createWalDirectory) throws IOException { - // until we've moved reader/writer construction down into providers, this initialization must - // happen prior to provider initialization, in case they need to instantiate a reader/writer. - timeoutMillis = conf.getInt("hbase.hlog.open.timeout", 300000); -@@ -229,6 +257,10 @@ public class WALFactory { - this.abortable = abortable; - // end required early initialization - if (conf.getBoolean(WAL_ENABLED, true)) { -+ if (createWalDirectory) { -+ // for testing only -+ createWALDirectory(conf, factoryId); -+ } - provider = getProvider(WAL_PROVIDER, DEFAULT_WAL_PROVIDER, null); - } else { - // special handling of existing configuration behavior. -diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java -index 7cb985aee9..51cce84e6f 100644 ---- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java -+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java -@@ -2582,8 +2582,9 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { - // The WAL subsystem will use the default rootDir rather than the passed in rootDir - // unless I pass along via the conf. - Configuration confForWAL = new Configuration(conf); -- confForWAL.set(HConstants.HBASE_DIR, rootDir.toString()); -- return new WALFactory(confForWAL, "hregion-" + RandomStringUtils.randomNumeric(8)).getWAL(hri); -+ CommonFSUtils.setRootDir(confForWAL, rootDir); -+ return new WALFactory(confForWAL, "hregion-" + RandomStringUtils.insecure().nextNumeric(8)) -+ .getWAL(hri); - } - - /** -diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestWALFencing.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestWALFencing.java -new file mode 100644 -index 0000000000..9341dcd5d9 ---- /dev/null -+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestWALFencing.java -@@ -0,0 +1,81 @@ -+/* -+ * Licensed to the Apache Software Foundation (ASF) under one -+ * or more contributor license agreements. See the NOTICE file -+ * distributed with this work for additional information -+ * regarding copyright ownership. The ASF licenses this file -+ * to you under the Apache License, Version 2.0 (the -+ * "License"); you may not use this file except in compliance -+ * with the License. You may obtain a copy of the License at -+ * -+ * http://www.apache.org/licenses/LICENSE-2.0 -+ * -+ * Unless required by applicable law or agreed to in writing, software -+ * distributed under the License is distributed on an "AS IS" BASIS, -+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -+ * See the License for the specific language governing permissions and -+ * limitations under the License. -+ */ -+package org.apache.hadoop.hbase.master; -+ -+import static org.awaitility.Awaitility.await; -+ -+import java.io.IOException; -+import java.time.Duration; -+import java.util.Collections; -+import org.apache.hadoop.fs.FileStatus; -+import org.apache.hadoop.fs.Path; -+import org.apache.hadoop.hbase.HBaseTestingUtility; -+import org.apache.hadoop.hbase.TableName; -+import org.apache.hadoop.hbase.client.RegionInfo; -+import org.apache.hadoop.hbase.regionserver.HRegionServer; -+import org.apache.hadoop.hbase.regionserver.Region; -+import org.apache.hadoop.hbase.testclassification.MasterTests; -+import org.apache.hadoop.hbase.testclassification.MediumTests; -+import org.apache.hadoop.hbase.util.RecoverLeaseFSUtils; -+import org.junit.jupiter.api.AfterAll; -+import org.junit.jupiter.api.BeforeAll; -+import org.junit.jupiter.api.Tag; -+import org.junit.jupiter.api.Test; -+ -+/** -+ * Testcase for HBASE-29797, where the lazy initialized WALProvider may recreate the WAL directory -+ * and cause our fencing way loses efficacy. -+ */ -+@Tag(MasterTests.TAG) -+@Tag(MediumTests.TAG) -+public class TestWALFencing { -+ -+ private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); -+ -+ @BeforeAll -+ public static void setUp() throws Exception { -+ UTIL.startMiniCluster(3); -+ UTIL.getAdmin().balancerSwitch(false, true); -+ } -+ -+ @AfterAll -+ public static void tearDown() throws IOException { -+ UTIL.shutdownMiniCluster(); -+ } -+ -+ @Test -+ public void testMoveMeta() throws Exception { -+ HRegionServer metaRs = UTIL.getRSForFirstRegionInTable(TableName.META_TABLE_NAME); -+ HRegionServer otherRs = UTIL.getOtherRegionServer(metaRs); -+ // do fencing here, i.e, kill otherRs -+ Path splittingDir = UTIL.getMiniHBaseCluster().getMaster().getMasterWalManager() -+ .getLogDirs(Collections.singleton(otherRs.getServerName())).get(0); -+ for (FileStatus walFile : otherRs.getWALFileSystem().listStatus(splittingDir)) { -+ RecoverLeaseFSUtils.recoverFileLease(otherRs.getWALFileSystem(), walFile.getPath(), -+ otherRs.getConfiguration()); -+ } -+ // move meta region to otherRs, which should fail and crash otherRs, and then master will try to -+ // assign meta region to another rs -+ RegionInfo metaRegionInfo = metaRs.getRegions().stream().map(Region::getRegionInfo) -+ .filter(RegionInfo::isMetaRegion).findAny().get(); -+ UTIL.getAdmin().move(metaRegionInfo.getEncodedNameAsBytes(), otherRs.getServerName()); -+ // make sure that meta region is not on otherRs -+ await().during(Duration.ofSeconds(5)).atMost(Duration.ofSeconds(6)) -+ .until(() -> otherRs.getRegions(TableName.META_TABLE_NAME).isEmpty()); -+ } -+} -diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionPolicy.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionPolicy.java -index 5d764df9eb..a9d48e2360 100644 ---- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionPolicy.java -+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionPolicy.java -@@ -99,6 +99,7 @@ public class TestCompactionPolicy { - htd.addFamily(hcd); - HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false); - -+ fs.mkdirs(new Path(basedir, logName)); - hlog = new FSHLog(fs, basedir, logName, conf); - ChunkCreator.initialize(MemStoreLAB.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null, - MemStoreLAB.INDEX_CHUNK_SIZE_PERCENTAGE_DEFAULT); -diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestFailedAppendAndSync.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestFailedAppendAndSync.java -index 676151f3eb..a460aa4922 100644 ---- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestFailedAppendAndSync.java -+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestFailedAppendAndSync.java -@@ -194,6 +194,7 @@ public class TestFailedAppendAndSync { - // the test. - FileSystem fs = FileSystem.get(CONF); - Path rootDir = new Path(dir + getName()); -+ fs.mkdirs(new Path(rootDir, getName())); - DodgyFSLog dodgyWAL = new DodgyFSLog(fs, services, rootDir, getName(), CONF); - dodgyWAL.init(); - LogRoller logRoller = new LogRoller(services); -diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java -index 1d53a7d652..68b6c4919a 100644 ---- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java -+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java -@@ -368,8 +368,9 @@ public class TestHRegion { - } - - FileSystem fs = FileSystem.get(CONF); -- Path rootDir = new Path(dir + "testMemstoreSnapshotSize"); -- MyFaultyFSLog faultyLog = new MyFaultyFSLog(fs, rootDir, "testMemstoreSnapshotSize", CONF); -+ Path rootDir = new Path(dir + method); -+ fs.mkdirs(new Path(rootDir, method)); -+ MyFaultyFSLog faultyLog = new MyFaultyFSLog(fs, rootDir, method, CONF); - region = initHRegion(tableName, null, null, CONF, false, Durability.SYNC_WAL, faultyLog, - COLUMN_FAMILY_BYTES); - -@@ -412,10 +413,10 @@ public class TestHRegion { - - @Test - public void testMemstoreSizeAccountingWithFailedPostBatchMutate() throws IOException { -- String testName = "testMemstoreSizeAccountingWithFailedPostBatchMutate"; - FileSystem fs = FileSystem.get(CONF); -- Path rootDir = new Path(dir + testName); -- FSHLog hLog = new FSHLog(fs, rootDir, testName, CONF); -+ Path rootDir = new Path(dir + method); -+ fs.mkdirs(new Path(rootDir, method)); -+ FSHLog hLog = new FSHLog(fs, rootDir, method, CONF); - hLog.init(); - HRegion region = initHRegion(tableName, null, null, CONF, false, Durability.SYNC_WAL, hLog, - COLUMN_FAMILY_BYTES); -@@ -1251,8 +1252,10 @@ public class TestHRegion { - }; - } - } -- FailAppendFlushMarkerWAL wal = new FailAppendFlushMarkerWAL(FileSystem.get(walConf), -- CommonFSUtils.getRootDir(walConf), method, walConf); -+ FileSystem fs = FileSystem.get(walConf); -+ Path rootDir = CommonFSUtils.getRootDir(walConf); -+ fs.mkdirs(new Path(rootDir, method)); -+ FailAppendFlushMarkerWAL wal = new FailAppendFlushMarkerWAL(fs, rootDir, method, walConf); - wal.init(); - this.region = initHRegion(tableName, HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, CONF, - false, Durability.USE_DEFAULT, wal, family); -@@ -3353,8 +3356,9 @@ public class TestHRegion { - @Test - public void testDataInMemoryWithoutWAL() throws IOException { - FileSystem fs = FileSystem.get(CONF); -- Path rootDir = new Path(dir + "testDataInMemoryWithoutWAL"); -- FSHLog hLog = new FSHLog(fs, rootDir, "testDataInMemoryWithoutWAL", CONF); -+ Path rootDir = new Path(dir + method); -+ fs.mkdirs(new Path(rootDir, method)); -+ FSHLog hLog = new FSHLog(fs, rootDir, method, CONF); - hLog.init(); - // This chunk creation is done throughout the code base. Do we want to move it into core? - // It is missing from this test. W/o it we NPE. -diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestLogRoller.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestLogRoller.java -index 154b2a839b..f1d0aa4c6f 100644 ---- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestLogRoller.java -+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestLogRoller.java -@@ -58,12 +58,13 @@ public class TestLogRoller { - private static FileSystem FS; - - @Before -- public void setup() throws Exception { -+ public void setUp() throws Exception { - CONF = TEST_UTIL.getConfiguration(); - CONF.setInt("hbase.regionserver.logroll.period", LOG_ROLL_PERIOD); - CONF.setInt(HConstants.THREAD_WAKE_FREQUENCY, 300); - ROOT_DIR = TEST_UTIL.getRandomDir(); - FS = FileSystem.get(CONF); -+ FS.mkdirs(new Path(ROOT_DIR, LOG_DIR)); - RegionServerServices services = Mockito.mock(RegionServerServices.class); - Mockito.when(services.getConfiguration()).thenReturn(CONF); - ROLLER = new LogRoller(services); -@@ -74,7 +75,7 @@ public class TestLogRoller { - public void tearDown() throws Exception { - ROLLER.close(); - FS.close(); -- TEST_UTIL.shutdownMiniCluster(); -+ TEST_UTIL.cleanupTestDir(); - } - - /** -diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALLockup.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALLockup.java -index 7925fe0073..731781bfc1 100644 ---- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALLockup.java -+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALLockup.java -@@ -216,6 +216,7 @@ public class TestWALLockup { - // OK. Now I have my mocked up Server & RegionServerServices and dodgy WAL, go ahead with test. - FileSystem fs = FileSystem.get(CONF); - Path rootDir = new Path(dir + getName()); -+ fs.mkdirs(new Path(rootDir, getName())); - DodgyFSLog dodgyWAL = new DodgyFSLog(fs, rootDir, getName(), CONF); - dodgyWAL.init(); - Path originalWAL = dodgyWAL.getCurrentFileName(); -@@ -394,6 +395,7 @@ public class TestWALLockup { - // OK. Now I have my mocked up Server & RegionServerServices and dodgy WAL, go ahead with test. - FileSystem fs = FileSystem.get(CONF); - Path rootDir = new Path(dir + getName()); -+ fs.mkdirs(new Path(rootDir, getName())); - final DodgyFSLog dodgyWAL = new DodgyFSLog(fs, rootDir, getName(), CONF); - dodgyWAL.init(); - // I need a log roller running. -diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestFSWAL.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestFSWAL.java -index 1f04e2718b..ea1d6e6de0 100644 ---- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestFSWAL.java -+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestFSWAL.java -@@ -120,6 +120,7 @@ public abstract class AbstractTestFSWAL { - final Path hbaseWALDir = TEST_UTIL.createWALRootDir(); - DIR = new Path(hbaseWALDir, currentTest.getMethodName()); - assertNotEquals(hbaseDir, hbaseWALDir); -+ FS.mkdirs(DIR); - } - - @BeforeClass -@@ -392,9 +393,8 @@ public abstract class AbstractTestFSWAL { - @Test(expected = IOException.class) - public void testFailedToCreateWALIfParentRenamed() - throws IOException, CommonFSUtils.StreamLacksCapabilityException { -- final String name = "testFailedToCreateWALIfParentRenamed"; -- AbstractFSWAL wal = newWAL(FS, CommonFSUtils.getWALRootDir(CONF), name, -- HConstants.HREGION_OLDLOGDIR_NAME, CONF, null, true, null, null); -+ AbstractFSWAL wal = newWAL(FS, CommonFSUtils.getWALRootDir(CONF), -+ currentTest.getMethodName(), HConstants.HREGION_OLDLOGDIR_NAME, CONF, null, true, null, null); - long filenum = EnvironmentEdgeManager.currentTime(); - Path path = wal.computeFilename(filenum); - wal.createWriterInstance(path); -@@ -535,6 +535,7 @@ public abstract class AbstractTestFSWAL { - - private AbstractFSWAL createHoldingWAL(String testName, AtomicBoolean startHoldingForAppend, - CountDownLatch holdAppend) throws IOException { -+ FS.mkdirs(new Path(CommonFSUtils.getRootDir(CONF), testName)); - AbstractFSWAL wal = newWAL(FS, CommonFSUtils.getRootDir(CONF), testName, - HConstants.HREGION_OLDLOGDIR_NAME, CONF, null, true, null, null); - wal.init(); -diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java -index d93970c620..4bf3d93dda 100644 ---- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java -+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java -@@ -1030,6 +1030,7 @@ public abstract class AbstractTestWALReplay { - } - - private MockWAL createMockWAL() throws IOException { -+ fs.mkdirs(new Path(hbaseRootDir, logName)); - MockWAL wal = new MockWAL(fs, hbaseRootDir, logName, conf); - wal.init(); - // Set down maximum recovery so we dfsclient doesn't linger retrying something -diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncFSWALDurability.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncFSWALDurability.java -index 8402617c44..3e4a58ce84 100644 ---- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncFSWALDurability.java -+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncFSWALDurability.java -@@ -57,7 +57,7 @@ public class TestAsyncFSWALDurability extends WALDurabilityTestBase { - HBaseClassTestRule.forClass(TestFSHLogDurability.class); - - @Override -- protected CustomFSHLog getWAL(FileSystem fs, Path root, String logDir, Configuration conf) -+ protected CustomFSHLog getWAL0(FileSystem fs, Path root, String logDir, Configuration conf) - throws IOException { - CustomFSHLog wal = new CustomFSHLog(fs, root, logDir, conf); - wal.init(); -diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java -index 8adf15876a..ef2932d4c4 100644 ---- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java -+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java -@@ -47,7 +47,9 @@ public class TestWALReplay extends AbstractTestWALReplay { - - @Override - protected WAL createWAL(Configuration c, Path hbaseRootDir, String logName) throws IOException { -- FSHLog wal = new FSHLog(FileSystem.get(c), hbaseRootDir, logName, c); -+ FileSystem fs = hbaseRootDir.getFileSystem(c); -+ fs.mkdirs(new Path(hbaseRootDir, logName)); -+ FSHLog wal = new FSHLog(fs, hbaseRootDir, logName, c); - wal.init(); - // Set down maximum recovery so we dfsclient doesn't linger retrying something - // long gone. -diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/WALDurabilityTestBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/WALDurabilityTestBase.java -index 81ac0bfbfd..e731fabbea 100644 ---- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/WALDurabilityTestBase.java -+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/WALDurabilityTestBase.java -@@ -69,7 +69,13 @@ public abstract class WALDurabilityTestBase { - TEST_UTIL.cleanupTestDir(); - } - -- protected abstract T getWAL(FileSystem fs, Path root, String logDir, Configuration conf) -+ protected final T getWAL(FileSystem fs, Path root, String logDir, Configuration conf) -+ throws IOException { -+ fs.mkdirs(new Path(root, logDir)); -+ return getWAL0(fs, root, logDir, conf); -+ } -+ -+ protected abstract T getWAL0(FileSystem fs, Path root, String logDir, Configuration conf) - throws IOException; - - protected abstract void resetSyncFlag(T wal); -diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALFactory.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALFactory.java -index 1afdc6e0fa..11c1667466 100644 ---- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALFactory.java -+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALFactory.java -@@ -621,7 +621,7 @@ public class TestWALFactory { - @Test - public void testWALProviders() throws IOException { - Configuration conf = new Configuration(); -- WALFactory walFactory = new WALFactory(conf, this.currentServername.toString()); -+ WALFactory walFactory = new WALFactory(conf, this.currentServername.toString(), null); - assertEquals(walFactory.getWALProvider().getClass(), walFactory.getMetaProvider().getClass()); - } - -@@ -629,7 +629,7 @@ public class TestWALFactory { - public void testOnlySetWALProvider() throws IOException { - Configuration conf = new Configuration(); - conf.set(WAL_PROVIDER, WALFactory.Providers.multiwal.name()); -- WALFactory walFactory = new WALFactory(conf, this.currentServername.toString()); -+ WALFactory walFactory = new WALFactory(conf, this.currentServername.toString(), null); - - assertEquals(WALFactory.Providers.multiwal.clazz, walFactory.getWALProvider().getClass()); - assertEquals(WALFactory.Providers.multiwal.clazz, walFactory.getMetaProvider().getClass()); -@@ -639,7 +639,7 @@ public class TestWALFactory { - public void testOnlySetMetaWALProvider() throws IOException { - Configuration conf = new Configuration(); - conf.set(META_WAL_PROVIDER, WALFactory.Providers.asyncfs.name()); -- WALFactory walFactory = new WALFactory(conf, this.currentServername.toString()); -+ WALFactory walFactory = new WALFactory(conf, this.currentServername.toString(), null); - - assertEquals(WALFactory.Providers.defaultProvider.clazz, - walFactory.getWALProvider().getClass()); -@@ -650,14 +650,15 @@ public class TestWALFactory { - public void testDefaultProvider() throws IOException { - final Configuration conf = new Configuration(); - // AsyncFSWal is the default, we should be able to request any WAL. -- final WALFactory normalWalFactory = new WALFactory(conf, this.currentServername.toString()); -+ final WALFactory normalWalFactory = -+ new WALFactory(conf, this.currentServername.toString(), null); - Class fshLogProvider = - normalWalFactory.getProviderClass(WALFactory.WAL_PROVIDER, Providers.filesystem.name()); - assertEquals(Providers.filesystem.clazz, fshLogProvider); - - // Imagine a world where MultiWAL is the default - final WALFactory customizedWalFactory = -- new WALFactory(conf, this.currentServername.toString()) { -+ new WALFactory(conf, this.currentServername.toString(), null) { - @Override - Providers getDefaultProvider() { - return Providers.multiwal; -@@ -673,7 +674,7 @@ public class TestWALFactory { - public void testCustomProvider() throws IOException { - final Configuration config = new Configuration(); - config.set(WALFactory.WAL_PROVIDER, IOTestProvider.class.getName()); -- final WALFactory walFactory = new WALFactory(config, this.currentServername.toString()); -+ final WALFactory walFactory = new WALFactory(config, this.currentServername.toString(), null); - Class walProvider = - walFactory.getProviderClass(WALFactory.WAL_PROVIDER, Providers.filesystem.name()); - assertEquals(IOTestProvider.class, walProvider); -@@ -685,7 +686,7 @@ public class TestWALFactory { - public void testCustomMetaProvider() throws IOException { - final Configuration config = new Configuration(); - config.set(WALFactory.META_WAL_PROVIDER, IOTestProvider.class.getName()); -- final WALFactory walFactory = new WALFactory(config, this.currentServername.toString()); -+ final WALFactory walFactory = new WALFactory(config, this.currentServername.toString(), null); - Class walProvider = - walFactory.getProviderClass(WALFactory.WAL_PROVIDER, Providers.filesystem.name()); - assertEquals(Providers.filesystem.clazz, walProvider); -diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplitToHFile.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplitToHFile.java -index 9bd4e32279..cac63289ed 100644 ---- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplitToHFile.java -+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplitToHFile.java -@@ -171,12 +171,15 @@ public class TestWALSplitToHFile { - } - - private WAL createWAL(Configuration c, Path hbaseRootDir, String logName) throws IOException { -- FSHLog wal = new FSHLog(FileSystem.get(c), hbaseRootDir, logName, c); -+ FileSystem fs = hbaseRootDir.getFileSystem(c); -+ fs.mkdirs(new Path(hbaseRootDir, logName)); -+ FSHLog wal = new FSHLog(fs, hbaseRootDir, logName, c); - wal.init(); - return wal; - } - - private WAL createWAL(FileSystem fs, Path hbaseRootDir, String logName) throws IOException { -+ fs.mkdirs(new Path(hbaseRootDir, logName)); - FSHLog wal = new FSHLog(fs, hbaseRootDir, logName, this.conf); - wal.init(); - return wal; -diff --git a/pom.xml b/pom.xml -index a0f1dea8fd..568bc41540 100644 ---- a/pom.xml -+++ b/pom.xml -@@ -598,6 +598,7 @@ - 9.3.15.0 - 5.13.4 - 5.13.4 -+ 4.3.0 - 1.3 - 1.49.0 - 1.29.0-alpha -@@ -1416,6 +1417,12 @@ - ${junit.vintage.version} - test - -+ -+ org.awaitility -+ awaitility -+ ${awaitility.version} -+ test -+ - - org.hamcrest - hamcrest-core diff --git a/hbase/hbase/stackable/patches/2.6.4/0008-Add-jcip-annotations-dependency-for-Hadoop-3.4.3-com.patch b/hbase/hbase/stackable/patches/2.6.4/0008-Add-jcip-annotations-dependency-for-Hadoop-3.4.3-com.patch deleted file mode 100644 index 95548e97e..000000000 --- a/hbase/hbase/stackable/patches/2.6.4/0008-Add-jcip-annotations-dependency-for-Hadoop-3.4.3-com.patch +++ /dev/null @@ -1,73 +0,0 @@ -From aecdd61fa521da9a3683b9e03c76f7019996b86a Mon Sep 17 00:00:00 2001 -From: Maxi Wittich -Date: Fri, 19 Jun 2026 12:22:37 +0200 -Subject: Add jcip-annotations dependency for Hadoop 3.4.3 compatibility - -When upgrading Hadoop from 3.4.2 to 3.4.3, jcip-annotations is no longer - pulled in transitively. BuiltInProviderSelector.java imports - net.jcip.annotations.NotThreadSafe, so we must declare the dependency - explicitly. Matches the fix HBase 2.6.6 upstream. ---- - hbase-client/pom.xml | 6 ++++++ - hbase-server/pom.xml | 6 ++++++ - pom.xml | 6 ++++++ - 3 files changed, 18 insertions(+) - -diff --git a/hbase-client/pom.xml b/hbase-client/pom.xml -index 54a3b2baed..165526b7de 100644 ---- a/hbase-client/pom.xml -+++ b/hbase-client/pom.xml -@@ -89,6 +89,12 @@ - compile - true - -+ -+ com.github.stephenc.jcip -+ jcip-annotations -+ compile -+ true -+ - - commons-codec - commons-codec -diff --git a/hbase-server/pom.xml b/hbase-server/pom.xml -index 64e3c3a477..2bf30fdb5d 100644 ---- a/hbase-server/pom.xml -+++ b/hbase-server/pom.xml -@@ -197,6 +197,12 @@ - compile - true - -+ -+ com.github.stephenc.jcip -+ jcip-annotations -+ compile -+ true -+ - - com.github.ben-manes.caffeine - caffeine -diff --git a/pom.xml b/pom.xml -index 568bc41540..ce780c04c9 100644 ---- a/pom.xml -+++ b/pom.xml -@@ -658,6 +658,7 @@ - 12.1.0 - 1.7.1 - 1.3.9-1 -+ 1.0-1 - 4.7.3 - 4.7.3.6 - 3.5.3 -@@ -1142,6 +1143,11 @@ - findbugs-annotations - ${findbugs-annotations.version} - -+ -+ com.github.stephenc.jcip -+ jcip-annotations -+ ${jcip-annotations.version} -+ - - 0.13.0 - 1.11.4 +@@ -821,7 +821,7 @@ + 0.15.0 + 0.15.0 + 1.11.5 - 2.8.1 + 2.8.8 1.15 + 1.7 2.18.0 - 3.18.0 +@@ -829,8 +829,8 @@ 3.6.1 1.5.0 3.4.4 @@ -30,37 +31,27 @@ index 68409106bb..a874b731ae 100644 3.2.6 -- 2.2.1 -- 1.0.58 +- 2.2.5 + 2.2.6 -+ 1.0.63 + 1.0.63 2.12.3 - 1.81 + 1.84 - 1.5.1 + 1.5.3 - 1.0.1 1.1.0 - 5.7.1 -@@ -647,29 +647,29 @@ - 1.0.0 - 1.8 - 3.3.0 -- 3.1.0 -+ 3.1.2 + + 3.0.0 +@@ -890,16 +890,16 @@ + 3.6.0 2.10 3.0.1 - 3.4.0 @@ -68,23 +59,21 @@ index 68409106bb..a874b731ae 100644 3.8.1 - 1.1.0 + 1.1.2 - 3.1.2 + 3.9.0 12.1.0 1.7.1 1.3.9-1 + 1.0-1 4.7.3 - 4.7.2.1 + 4.7.3.6 3.5.3 2.12 1.0.1 - 2.44.4 -- 3.12.0 -+ 3.12.1 - - 0.27 +@@ -910,8 +910,8 @@ + 2.0.3 1.11.0 - 1.8.0 + 1.10.4 - 1.1.10.4 - 1.5.7-2 + 1.1.10.7 diff --git a/hbase/hbase/stackable/patches/2.6.4/0003-Include-jackson-dataformat-xml.patch b/hbase/hbase/stackable/patches/3.0.0/0002-Include-jackson-dataformat-xml.patch similarity index 65% rename from hbase/hbase/stackable/patches/2.6.4/0003-Include-jackson-dataformat-xml.patch rename to hbase/hbase/stackable/patches/3.0.0/0002-Include-jackson-dataformat-xml.patch index f211fc171..115e3e621 100644 --- a/hbase/hbase/stackable/patches/2.6.4/0003-Include-jackson-dataformat-xml.patch +++ b/hbase/hbase/stackable/patches/3.0.0/0002-Include-jackson-dataformat-xml.patch @@ -1,4 +1,4 @@ -From c23733bc2fb0e47a14d1132a1dfb1a6c59035959 Mon Sep 17 00:00:00 2001 +From 70f30876a4df406f9fa46cc44f19467998720e5e Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Fri, 12 Jul 2024 17:30:12 -0400 Subject: Include jackson-dataformat-xml. @@ -7,15 +7,15 @@ This is needed for XmlLayout to work so our structured logging works. It is an optional dependency of log4j2 so we need to make sure to include it. --- - hbase-assembly/pom.xml | 5 +++++ - pom.xml | 12 ++++++++++++ - 2 files changed, 17 insertions(+) + hbase-assembly/pom.xml | 5 +++++ + pom.xml | 7 +++++++ + 2 files changed, 12 insertions(+) diff --git a/hbase-assembly/pom.xml b/hbase-assembly/pom.xml -index f2080226f0..b4013a6203 100644 +index ba121e9680..e6dd9f6542 100644 --- a/hbase-assembly/pom.xml +++ b/hbase-assembly/pom.xml -@@ -221,6 +221,11 @@ +@@ -193,6 +193,11 @@ org.apache.logging.log4j log4j-core @@ -28,22 +28,10 @@ index f2080226f0..b4013a6203 100644 org.apache.logging.log4j log4j-slf4j-impl diff --git a/pom.xml b/pom.xml -index a874b731ae..b2523efedf 100644 +index 8267221deb..0a90f04581 100644 --- a/pom.xml +++ b/pom.xml -@@ -1182,6 +1182,11 @@ - log4j-core - ${log4j2.version} - -+ -+ org.apache.logging.log4j -+ log4j-core -+ ${log4j2.version} -+ - - org.apache.logging.log4j - log4j-slf4j-impl -@@ -1192,6 +1197,13 @@ +@@ -1434,6 +1434,13 @@ log4j-1.2-api ${log4j2.version} @@ -51,7 +39,7 @@ index a874b731ae..b2523efedf 100644 + + com.fasterxml.jackson.dataformat + jackson-dataformat-xml -+ ${jackson.databind.version} ++ ${jackson.version} + + diff --git a/hbase/hbase/stackable/patches/2.6.4/0004-Bump-cyclonedx-maven-plugin-to-version-2.9.1-and-twe.patch b/hbase/hbase/stackable/patches/3.0.0/0003-Bump-cyclonedx-maven-plugin-to-version-2.9.1-and-twe.patch similarity index 86% rename from hbase/hbase/stackable/patches/2.6.4/0004-Bump-cyclonedx-maven-plugin-to-version-2.9.1-and-twe.patch rename to hbase/hbase/stackable/patches/3.0.0/0003-Bump-cyclonedx-maven-plugin-to-version-2.9.1-and-twe.patch index cfecabb5a..08d15fdab 100644 --- a/hbase/hbase/stackable/patches/2.6.4/0004-Bump-cyclonedx-maven-plugin-to-version-2.9.1-and-twe.patch +++ b/hbase/hbase/stackable/patches/3.0.0/0003-Bump-cyclonedx-maven-plugin-to-version-2.9.1-and-twe.patch @@ -1,4 +1,4 @@ -From 64e6ee9a23aa3d5943c259347943a19b01499028 Mon Sep 17 00:00:00 2001 +From 84d450405b3177e9fe97dbddbb161fe53b422756 Mon Sep 17 00:00:00 2001 From: Lukas Voetmand Date: Fri, 6 Sep 2024 17:53:52 +0200 Subject: Bump cyclonedx-maven-plugin to version 2.9.1 and tweak its @@ -9,10 +9,10 @@ Subject: Bump cyclonedx-maven-plugin to version 2.9.1 and tweak its 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml -index b2523efedf..a0f1dea8fd 100644 +index 0a90f04581..8e37531651 100644 --- a/pom.xml +++ b/pom.xml -@@ -3336,7 +3336,11 @@ +@@ -3544,7 +3544,11 @@ org.cyclonedx cyclonedx-maven-plugin diff --git a/hbase/hbase/stackable/patches/2.6.4/0005-Allow-overriding-ipc-bind-port-and-use-alternative-p.patch b/hbase/hbase/stackable/patches/3.0.0/0004-Allow-overriding-ipc-bind-port-and-use-alternative-p.patch similarity index 56% rename from hbase/hbase/stackable/patches/2.6.4/0005-Allow-overriding-ipc-bind-port-and-use-alternative-p.patch rename to hbase/hbase/stackable/patches/3.0.0/0004-Allow-overriding-ipc-bind-port-and-use-alternative-p.patch index c737e218a..08b1f0c16 100644 --- a/hbase/hbase/stackable/patches/2.6.4/0005-Allow-overriding-ipc-bind-port-and-use-alternative-p.patch +++ b/hbase/hbase/stackable/patches/3.0.0/0004-Allow-overriding-ipc-bind-port-and-use-alternative-p.patch @@ -1,20 +1,23 @@ -From 98ae371a451ad67b5fc5b73bd367afef89b68c01 Mon Sep 17 00:00:00 2001 +From b6967065ea272bb285609e0eb8bfb2a34da62b7d Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Mon, 16 Jun 2025 14:44:32 +0200 Subject: Allow overriding ipc bind port and use alternative port from listener --- - .../org/apache/hadoop/hbase/HConstants.java | 29 ++++++++++-- - .../apache/hadoop/hbase/master/HMaster.java | 20 +++++++-- - .../hbase/regionserver/HRegionServer.java | 45 +++++++++++++------ - .../hbase/regionserver/RSRpcServices.java | 8 +++- - 4 files changed, 80 insertions(+), 22 deletions(-) + .../org/apache/hadoop/hbase/HConstants.java | 29 ++++++++++++++++--- + .../hadoop/hbase/HBaseRpcServicesBase.java | 5 +++- + .../apache/hadoop/hbase/HBaseServerBase.java | 29 ++++++++++++++----- + .../apache/hadoop/hbase/master/HMaster.java | 20 +++++++++++-- + .../hbase/master/MasterRpcServices.java | 9 +++++- + .../hbase/regionserver/HRegionServer.java | 25 ++++++++++------ + .../hbase/regionserver/RSRpcServices.java | 9 +++++- + 7 files changed, 100 insertions(+), 26 deletions(-) diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java -index c677e610c7..890a23e19f 100644 +index 1051686d32..47a7884755 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java -@@ -195,18 +195,27 @@ public final class HConstants { +@@ -190,18 +190,27 @@ public final class HConstants { /** default host address */ public static final String DEFAULT_HOST = "0.0.0.0"; @@ -44,7 +47,7 @@ index c677e610c7..890a23e19f 100644 /** Configuration key for the list of master host:ports **/ public static final String MASTER_ADDRS_KEY = "hbase.masters"; -@@ -314,18 +323,27 @@ public final class HConstants { +@@ -284,18 +293,27 @@ public final class HConstants { /** Default value for ZooKeeper session timeout */ public static final int DEFAULT_ZK_SESSION_TIMEOUT = 90 * 1000; @@ -74,7 +77,7 @@ index c677e610c7..890a23e19f 100644 /** A flag that enables automatic selection of regionserver info port */ public static final String REGIONSERVER_INFO_PORT_AUTO = REGIONSERVER_INFO_PORT + ".auto"; -@@ -1403,6 +1421,9 @@ public final class HConstants { +@@ -1308,6 +1326,9 @@ public final class HConstants { /** Configuration key for setting RPC codec class name */ public static final String RPC_CODEC_CONF_KEY = "hbase.client.rpc.codec"; @@ -84,8 +87,121 @@ index c677e610c7..890a23e19f 100644 /** Configuration key for setting replication codec class name */ public static final String REPLICATION_CODEC_CONF_KEY = "hbase.replication.rpc.codec"; +diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/HBaseRpcServicesBase.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/HBaseRpcServicesBase.java +index b2a0e78036..82be143e3c 100644 +--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/HBaseRpcServicesBase.java ++++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/HBaseRpcServicesBase.java +@@ -129,7 +129,8 @@ public abstract class HBaseRpcServicesBase> + int port = conf.getInt(getPortConfigName(), getDefaultPort()); + // Creation of a HSA will force a resolve. + final InetSocketAddress initialIsa = new InetSocketAddress(hostname, port); +- final InetSocketAddress bindAddress = new InetSocketAddress(getHostname(conf, hostname), port); ++ final InetSocketAddress bindAddress = ++ new InetSocketAddress(getHostname(conf, hostname), getBindPort(conf, port)); + if (initialIsa.getAddress() == null) { + throw new IllegalArgumentException("Failed resolve of " + initialIsa); + } +@@ -166,6 +167,8 @@ public abstract class HBaseRpcServicesBase> + + protected abstract String getHostname(Configuration conf, String defaultHostname); + ++ protected abstract int getBindPort(Configuration conf, int defaultPort); ++ + protected abstract String getPortConfigName(); + + protected abstract int getDefaultPort(); +diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/HBaseServerBase.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/HBaseServerBase.java +index 5beeceadf0..d7a88b93c9 100644 +--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/HBaseServerBase.java ++++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/HBaseServerBase.java +@@ -21,6 +21,7 @@ import static org.apache.hadoop.hbase.ChoreService.CHORE_SERVICE_INITIAL_POOL_SI + import static org.apache.hadoop.hbase.ChoreService.DEFAULT_CHORE_SERVICE_INITIAL_POOL_SIZE; + import static org.apache.hadoop.hbase.HConstants.DEFAULT_HBASE_SPLIT_COORDINATED_BY_ZK; + import static org.apache.hadoop.hbase.HConstants.HBASE_SPLIT_WAL_COORDINATED_BY_ZK; ++import static org.apache.hadoop.hbase.HConstants.RPC_CLIENT_BIND_ADDRESS; + + import com.google.errorprone.annotations.RestrictedApi; + import io.opentelemetry.api.trace.Span; +@@ -128,6 +129,10 @@ public abstract class HBaseServerBase> extends + */ + protected final String useThisHostnameInstead; + ++ protected int useThisPortInstead; ++ ++ protected int useThisInfoPortInstead; ++ + /** + * Provide online slow log responses from ringbuffer + */ +@@ -205,14 +210,18 @@ public abstract class HBaseServerBase> extends + */ + protected final synchronized void setupClusterConnection() throws IOException { + if (asyncClusterConnection == null) { +- InetSocketAddress localAddress = +- new InetSocketAddress(rpcServices.getSocketAddress().getAddress(), 0); ++ InetSocketAddress localAddress = getInetSocketAddress(this.conf); + User user = userProvider.getCurrent(); + asyncClusterConnection = + ClusterConnectionFactory.createAsyncClusterConnection(this, conf, localAddress, user); + } + } + ++ private InetSocketAddress getInetSocketAddress(Configuration conf) { ++ return conf.getBoolean(RPC_CLIENT_BIND_ADDRESS, true) ? ++ new InetSocketAddress(this.rpcServices.getSocketAddress().getAddress(), 0) : new InetSocketAddress(0); ++ } ++ + protected final void initializeFileSystem() throws IOException { + // Get fs instance used by this RS. Do we use checksum verification in the hbase? If hbase + // checksum verification enabled, then automatically switch off hdfs checksum verification. +@@ -261,19 +270,21 @@ public abstract class HBaseServerBase> extends + // has not logged in yet, UGI bootstraps from the ticket cache and spawns a TGT renewer + // for whichever principal happens to be there. + String hostName = resolveHostName(conf, useThisHostnameInstead); ++ + // login the zookeeper client principal (if using security) + ZKAuthentication.loginClient(this.conf, HConstants.ZK_CLIENT_KEYTAB_FILE, + HConstants.ZK_CLIENT_KERBEROS_PRINCIPAL, hostName); + // login the server principal (if using secure Hadoop) + login(userProvider, hostName); + this.rpcServices = createRpcServices(); +- InetSocketAddress addr = rpcServices.getSocketAddress(); +- serverName = ServerName.valueOf(hostName, addr.getPort(), this.startcode); ++ useThisPortInstead = getUseThisPortInstead(conf); ++ useThisInfoPortInstead = getUseThisInfoPortInstead(conf); ++ serverName = ServerName.valueOf(hostName, useThisPortInstead, this.startcode); + // init superusers and add the server principal (if using security) + // or process owner as default super user. + Superusers.initialize(conf); +- zooKeeper = +- new ZKWatcher(conf, getProcessName() + ":" + addr.getPort(), this, canCreateBaseZNode()); ++ zooKeeper = new ZKWatcher(conf, getProcessName() + ":" + useThisPortInstead, this, ++ canCreateBaseZNode()); + + this.configurationManager = new ConfigurationManager(); + setupSignalHandlers(); +@@ -358,7 +369,7 @@ public abstract class HBaseServerBase> extends + LOG.info("Retry starting http info server with port: " + port); + } + } +- port = this.infoServer.getPort(); ++ port = useThisInfoPortInstead; + conf.setInt(HConstants.REGIONSERVER_INFO_PORT, port); + int masterInfoPort = + conf.getInt(HConstants.MASTER_INFO_PORT, HConstants.DEFAULT_MASTER_INFOPORT); +@@ -658,6 +669,10 @@ public abstract class HBaseServerBase> extends + + protected abstract String getUseThisHostnameInstead(Configuration conf) throws IOException; + ++ protected abstract int getUseThisPortInstead(Configuration conf); ++ ++ protected abstract int getUseThisInfoPortInstead(Configuration conf); ++ + protected abstract void login(UserProvider user, String host) throws IOException; + + protected abstract DNS.ServerType getDNSServerType(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java -index 35f212b445..d2b787ddef 100644 +index c8e70db5b9..29138a2ddd 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -17,6 +17,8 @@ @@ -97,8 +213,8 @@ index 35f212b445..d2b787ddef 100644 import static org.apache.hadoop.hbase.HConstants.DEFAULT_HBASE_SPLIT_COORDINATED_BY_ZK; import static org.apache.hadoop.hbase.HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS; import static org.apache.hadoop.hbase.HConstants.HBASE_SPLIT_WAL_COORDINATED_BY_ZK; -@@ -571,6 +573,18 @@ public class HMaster extends HRegionServer implements MasterServices { - return conf.get(MASTER_HOSTNAME_KEY); +@@ -614,6 +616,18 @@ public class HMaster extends HBaseServerBase implements Maste + return DNS.ServerType.MASTER; } + @Override @@ -116,9 +232,9 @@ index 35f212b445..d2b787ddef 100644 private void registerConfigurationObservers() { configurationManager.registerObserver(this.rpcServices); configurationManager.registerObserver(this); -@@ -598,8 +612,8 @@ public class HMaster extends HRegionServer implements MasterServices { +@@ -628,8 +642,8 @@ public class HMaster extends HBaseServerBase implements Maste registerConfigurationObservers(); - Threads.setDaemonThreadRunning(new Thread(() -> TraceUtil.trace(() -> { + Threads.setDaemonThreadRunning(new Thread(TraceUtil.tracedRunnable(() -> { try { - int infoPort = putUpJettyServer(); - startActiveMasterManager(infoPort); @@ -127,7 +243,7 @@ index 35f212b445..d2b787ddef 100644 } catch (Throwable t) { // Make sure we log the exception. String error = "Failed to become Active Master"; -@@ -3024,7 +3038,7 @@ public class HMaster extends HRegionServer implements MasterServices { +@@ -3115,7 +3129,7 @@ public class HMaster extends HBaseServerBase implements Maste } case MASTER_INFO_PORT: { if (infoServer != null) { @@ -136,65 +252,53 @@ index 35f212b445..d2b787ddef 100644 } break; } +diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java +index 790ff28d28..9c63ef649a 100644 +--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java ++++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java +@@ -450,6 +450,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos.Updat + import org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos.UpdateReplicationPeerConfigResponse; + import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription; + import org.apache.hadoop.hbase.shaded.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsService; ++import static org.apache.hadoop.hbase.HConstants.MASTER_IPC_ADDRESS; ++import static org.apache.hadoop.hbase.HConstants.MASTER_IPC_PORT; + + /** + * Implements the master RPC services. +@@ -502,7 +504,12 @@ public class MasterRpcServices extends HBaseRpcServicesBase + + @Override + protected String getHostname(Configuration conf, String defaultHostname) { +- return conf.get("hbase.master.ipc.address", defaultHostname); ++ return conf.get(MASTER_IPC_ADDRESS, defaultHostname); ++ } ++ ++ @Override ++ protected int getBindPort(Configuration conf, int defaultPort) { ++ return conf.getInt(MASTER_IPC_PORT, defaultPort); + } + + @Override diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java -index 4085315ea8..c51f66678c 100644 +index 0874b808e6..0901f75d0c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java -@@ -24,6 +24,9 @@ import static org.apache.hadoop.hbase.HConstants.DEFAULT_HBASE_SPLIT_WAL_MAX_SPL +@@ -22,6 +22,8 @@ import static org.apache.hadoop.hbase.HConstants.DEFAULT_HBASE_SPLIT_WAL_MAX_SPL import static org.apache.hadoop.hbase.HConstants.DEFAULT_SLOW_LOG_SYS_TABLE_CHORE_DURATION; import static org.apache.hadoop.hbase.HConstants.HBASE_SPLIT_WAL_COORDINATED_BY_ZK; import static org.apache.hadoop.hbase.HConstants.HBASE_SPLIT_WAL_MAX_SPLITTER; +import static org.apache.hadoop.hbase.HConstants.REGIONSERVER_BOUND_INFO_PORT; +import static org.apache.hadoop.hbase.HConstants.REGIONSERVER_PORT; -+import static org.apache.hadoop.hbase.HConstants.RPC_CLIENT_BIND_ADDRESS; import static org.apache.hadoop.hbase.master.waleventtracker.WALEventTrackerTableCreator.WAL_EVENT_TRACKER_ENABLED_DEFAULT; import static org.apache.hadoop.hbase.master.waleventtracker.WALEventTrackerTableCreator.WAL_EVENT_TRACKER_ENABLED_KEY; import static org.apache.hadoop.hbase.namequeues.NamedQueueServiceChore.NAMED_QUEUE_CHORE_DURATION_DEFAULT; -@@ -505,6 +508,10 @@ public class HRegionServer extends Thread - */ - protected String useThisHostnameInstead; - -+ protected int useThisPortInstead; -+ -+ protected int useThisInfoPortInstead; -+ - /** - * @deprecated since 2.4.0 and will be removed in 4.0.0. Use - * {@link HRegionServer#UNSAFE_RS_HOSTNAME_DISABLE_MASTER_REVERSEDNS_KEY} instead. -@@ -669,6 +676,8 @@ public class HRegionServer extends Thread - this.namedQueueRecorder = NamedQueueRecorder.getInstance(this.conf); - rpcServices = createRpcServices(); - useThisHostnameInstead = getUseThisHostnameInstead(conf); -+ useThisPortInstead = getUseThisPortInstead(conf); -+ useThisInfoPortInstead = getUseThisInfoPortInstead(conf); - - // if use-ip is enabled, we will use ip to expose Master/RS service for client, - // see HBASE-27304 for details. -@@ -678,7 +687,7 @@ public class HRegionServer extends Thread - useIp ? rpcServices.isa.getAddress().getHostAddress() : rpcServices.isa.getHostName(); - String hostName = - StringUtils.isBlank(useThisHostnameInstead) ? isaHostName : useThisHostnameInstead; -- serverName = ServerName.valueOf(hostName, this.rpcServices.isa.getPort(), this.startcode); -+ serverName = ServerName.valueOf(hostName, useThisPortInstead, this.startcode); - - rpcControllerFactory = RpcControllerFactory.instantiate(this.conf); - rpcRetryingCallerFactory = RpcRetryingCallerFactory.instantiate(this.conf, -@@ -718,7 +727,7 @@ public class HRegionServer extends Thread - - // Some unit tests don't need a cluster, so no zookeeper at all - // Open connection to zookeeper and set primary watcher -- zooKeeper = new ZKWatcher(conf, getProcessName() + ":" + rpcServices.isa.getPort(), this, -+ zooKeeper = new ZKWatcher(conf, getProcessName() + ":" + useThisPortInstead, this, - canCreateBaseZNode()); - // If no master in cluster, skip trying to track one or look for a cluster status. - if (!this.masterless) { -@@ -779,6 +788,16 @@ public class HRegionServer extends Thread - } +@@ -588,6 +590,16 @@ public class HRegionServer extends HBaseServerBase + return DNS.ServerType.REGIONSERVER; } + protected int getUseThisPortInstead(Configuration conf) { + int port = conf.getInt(REGIONSERVER_PORT, 0); -+ return port != 0 ? port : this.rpcServices.isa.getPort(); ++ return port != 0 ? port : this.rpcServices.getSocketAddress().getPort(); + } + + protected int getUseThisInfoPortInstead(Configuration conf) { @@ -202,32 +306,11 @@ index 4085315ea8..c51f66678c 100644 + return port != 0 ? port : this.infoServer != null ? this.infoServer.getPort() : -1; + } + - private void setupSignalHandlers() { - if (!SystemUtils.IS_OS_WINDOWS) { - HBasePlatformDependent.handle("HUP", (number, name) -> { -@@ -961,7 +980,7 @@ public class HRegionServer extends Thread - } - // Setup RPC client for master communication - this.rpcClient = RpcClientFactory.createClient(conf, clusterId, -- new InetSocketAddress(this.rpcServices.isa.getAddress(), 0), -+ getInetSocketAddress(this.conf), - clusterConnection.getConnectionMetrics(), Collections.emptyMap()); - span.setStatus(StatusCode.OK); - } catch (Throwable t) { -@@ -975,6 +994,11 @@ public class HRegionServer extends Thread + @Override + protected void login(UserProvider user, String host) throws IOException { + user.login(SecurityConstants.REGIONSERVER_KRB_KEYTAB_FILE, +@@ -1251,11 +1263,7 @@ public class HRegionServer extends HBaseServerBase } - } - -+ private InetSocketAddress getInetSocketAddress(Configuration conf) { -+ return conf.getBoolean(RPC_CLIENT_BIND_ADDRESS, true) ? -+ new InetSocketAddress(this.rpcServices.isa.getAddress(), 0) : new InetSocketAddress(0); -+ } -+ - /** - * Bring up connection to zk ensemble and then wait until a master for this cluster and then after - * that, wait until cluster 'up' flag has been set. This is the order in which master does things. -@@ -1531,11 +1555,7 @@ public class HRegionServer extends Thread - serverLoad.setReportStartTime(reportStartTime); serverLoad.setReportEndTime(reportEndTime); - if (this.infoServer != null) { @@ -239,7 +322,7 @@ index 4085315ea8..c51f66678c 100644 MetricsUserAggregateSource userSource = metricsRegionServer.getMetricsUserAggregate().getSource(); if (userSource != null) { -@@ -1691,7 +1711,7 @@ public class HRegionServer extends Thread +@@ -1416,7 +1424,7 @@ public class HRegionServer extends HBaseServerBase if (key.equals(HConstants.KEY_FOR_HOSTNAME_SEEN_BY_MASTER)) { String hostnameFromMasterPOV = e.getValue(); this.serverName = ServerName.valueOf(hostnameFromMasterPOV, @@ -248,7 +331,7 @@ index 4085315ea8..c51f66678c 100644 String expectedHostName = rpcServices.getSocketAddress().getHostName(); // if Master use-ip is enabled, RegionServer use-ip will be enabled by default even if it // is set to disable. so we will use the ip of the RegionServer to compare with the -@@ -1817,7 +1837,7 @@ public class HRegionServer extends Thread +@@ -1520,7 +1528,7 @@ public class HRegionServer extends HBaseServerBase private void createMyEphemeralNode() throws KeeperException { RegionServerInfo.Builder rsInfo = RegionServerInfo.newBuilder(); @@ -257,20 +340,11 @@ index 4085315ea8..c51f66678c 100644 rsInfo.setVersionInfo(ProtobufUtil.getVersionInfo()); byte[] data = ProtobufUtil.prependPBMagic(rsInfo.build().toByteArray()); ZKUtil.createEphemeralNodeAndWatch(this.zooKeeper, getMyEphemeralNodePath(), data); -@@ -2490,7 +2510,7 @@ public class HRegionServer extends Thread - LOG.info("Retry starting http info server with port: " + port); - } - } -- port = this.infoServer.getPort(); -+ port = useThisInfoPortInstead; - conf.setInt(HConstants.REGIONSERVER_INFO_PORT, port); - int masterInfoPort = - conf.getInt(HConstants.MASTER_INFO_PORT, HConstants.DEFAULT_MASTER_INFOPORT); -@@ -3084,12 +3104,11 @@ public class HRegionServer extends Thread - LOG.info("reportForDuty to master=" + masterServerName + " with isa=" + rpcServices.isa - + ", startcode=" + this.startcode); +@@ -2688,12 +2696,11 @@ public class HRegionServer extends HBaseServerBase + LOG.info("reportForDuty to master=" + masterServerName + " with port=" + + rpcServices.getSocketAddress().getPort() + ", startcode=" + this.startcode); long now = EnvironmentEdgeManager.currentTime(); -- int port = rpcServices.isa.getPort(); +- int port = rpcServices.getSocketAddress().getPort(); RegionServerStartupRequest.Builder request = RegionServerStartupRequest.newBuilder(); if (!StringUtils.isBlank(useThisHostnameInstead)) { request.setUseThisHostnameInstead(useThisHostnameInstead); @@ -281,34 +355,29 @@ index 4085315ea8..c51f66678c 100644 request.setServerCurrentTime(now); result = rss.regionServerStartup(null, request.build()); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java -index e246da4bd8..d8408bb3d0 100644 +index fdfea375e0..378bb8c761 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java -@@ -283,6 +283,10 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.BulkLoadDescr +@@ -247,6 +247,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.BulkLoadDescr import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.CompactionDescriptor; import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor; import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.RegionEventDescriptor; -+import static org.apache.hadoop.hbase.HConstants.MASTER_IPC_ADDRESS; -+import static org.apache.hadoop.hbase.HConstants.MASTER_IPC_PORT; +import static org.apache.hadoop.hbase.HConstants.REGIONSERVER_IPC_ADDRESS; +import static org.apache.hadoop.hbase.HConstants.REGIONSERVER_IPC_PORT; /** * Implements the regionserver RPC services. -@@ -1273,14 +1277,14 @@ public class RSRpcServices implements HBaseRPCErrorHandler, AdminService.Blockin - int port = conf.getInt(HConstants.MASTER_PORT, HConstants.DEFAULT_MASTER_PORT); - // Creation of a HSA will force a resolve. - initialIsa = new InetSocketAddress(hostname, port); -- bindAddress = new InetSocketAddress(conf.get("hbase.master.ipc.address", hostname), port); -+ bindAddress = new InetSocketAddress(conf.get(MASTER_IPC_ADDRESS, hostname), conf.getInt(MASTER_IPC_PORT, port)); - } else { - String hostname = DNS.getHostname(conf, DNS.ServerType.REGIONSERVER); - int port = conf.getInt(HConstants.REGIONSERVER_PORT, HConstants.DEFAULT_REGIONSERVER_PORT); - // Creation of a HSA will force a resolve. - initialIsa = new InetSocketAddress(hostname, port); - bindAddress = -- new InetSocketAddress(conf.get("hbase.regionserver.ipc.address", hostname), port); -+ new InetSocketAddress(conf.get(REGIONSERVER_IPC_ADDRESS, hostname), conf.getInt(REGIONSERVER_IPC_PORT, port)); - } - if (initialIsa.getAddress() == null) { - throw new IllegalArgumentException("Failed resolve of " + initialIsa); +@@ -1229,7 +1231,12 @@ public class RSRpcServices extends HBaseRpcServicesBase + + @Override + protected String getHostname(Configuration conf, String defaultHostname) { +- return conf.get("hbase.regionserver.ipc.address", defaultHostname); ++ return conf.get(REGIONSERVER_IPC_ADDRESS, defaultHostname); ++ } ++ ++ @Override ++ protected int getBindPort(Configuration conf, int defaultPort) { ++ return conf.getInt(REGIONSERVER_IPC_PORT, defaultPort); + } + + @Override diff --git a/hbase/hbase/stackable/patches/2.6.4/0006-Update-property-usage-for-bound-ports.patch b/hbase/hbase/stackable/patches/3.0.0/0005-Update-property-usage-for-bound-ports.patch similarity index 75% rename from hbase/hbase/stackable/patches/2.6.4/0006-Update-property-usage-for-bound-ports.patch rename to hbase/hbase/stackable/patches/3.0.0/0005-Update-property-usage-for-bound-ports.patch index 869be9cf3..b42ad1df7 100644 --- a/hbase/hbase/stackable/patches/2.6.4/0006-Update-property-usage-for-bound-ports.patch +++ b/hbase/hbase/stackable/patches/3.0.0/0005-Update-property-usage-for-bound-ports.patch @@ -1,20 +1,21 @@ -From ee0bb32192f4e233638034aa1d4806507fac915f Mon Sep 17 00:00:00 2001 +From 9405bebf3d9d91034c592c4478a2712e9671d287 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Thu, 26 Jun 2025 16:58:47 +0200 Subject: Update property usage for bound ports --- .../org/apache/hadoop/hbase/HConstants.java | 4 ++-- + .../apache/hadoop/hbase/HBaseServerBase.java | 20 ++++++++++++++----- .../hadoop/hbase/LocalHBaseCluster.java | 12 +++++------ .../apache/hadoop/hbase/master/HMaster.java | 6 +++--- - .../hbase/regionserver/HRegionServer.java | 21 +++++++++++++------ - 4 files changed, 26 insertions(+), 17 deletions(-) + .../hbase/regionserver/HRegionServer.java | 4 ++-- + 5 files changed, 28 insertions(+), 18 deletions(-) diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java -index 890a23e19f..6a6f33c9e7 100644 +index 47a7884755..ffe093ac11 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java -@@ -213,7 +213,7 @@ public final class HConstants { +@@ -208,7 +208,7 @@ public final class HConstants { /** Configuration key for advertised master web API port */ public static final String MASTER_INFO_PORT = "hbase.master.info.port"; @@ -23,7 +24,7 @@ index 890a23e19f..6a6f33c9e7 100644 public static final String MASTER_BOUND_INFO_PORT = "hbase.master.bound.info.port"; /** Configuration key for the list of master host:ports **/ -@@ -341,7 +341,7 @@ public final class HConstants { +@@ -311,7 +311,7 @@ public final class HConstants { /** Configuration key for advertised region server web API port */ public static final String REGIONSERVER_INFO_PORT = "hbase.regionserver.info.port"; @@ -32,11 +33,63 @@ index 890a23e19f..6a6f33c9e7 100644 public static final String REGIONSERVER_BOUND_INFO_PORT = "hbase.regionserver.bound.info.port"; /** A flag that enables automatic selection of regionserver info port */ +diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/HBaseServerBase.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/HBaseServerBase.java +index d7a88b93c9..2c7d66d9c1 100644 +--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/HBaseServerBase.java ++++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/HBaseServerBase.java +@@ -21,6 +21,8 @@ import static org.apache.hadoop.hbase.ChoreService.CHORE_SERVICE_INITIAL_POOL_SI + import static org.apache.hadoop.hbase.ChoreService.DEFAULT_CHORE_SERVICE_INITIAL_POOL_SIZE; + import static org.apache.hadoop.hbase.HConstants.DEFAULT_HBASE_SPLIT_COORDINATED_BY_ZK; + import static org.apache.hadoop.hbase.HConstants.HBASE_SPLIT_WAL_COORDINATED_BY_ZK; ++import static org.apache.hadoop.hbase.HConstants.REGIONSERVER_BOUND_INFO_PORT; ++import static org.apache.hadoop.hbase.HConstants.REGIONSERVER_INFO_PORT; + import static org.apache.hadoop.hbase.HConstants.RPC_CLIENT_BIND_ADDRESS; + + import com.google.errorprone.annotations.RestrictedApi; +@@ -327,12 +329,14 @@ public abstract class HBaseServerBase> extends + */ + private void putUpWebUI() throws IOException { + int port = +- this.conf.getInt(HConstants.REGIONSERVER_INFO_PORT, HConstants.DEFAULT_REGIONSERVER_INFOPORT); ++ this.conf.getInt(REGIONSERVER_BOUND_INFO_PORT, ++ this.conf.getInt(REGIONSERVER_INFO_PORT, HConstants.DEFAULT_REGIONSERVER_INFOPORT)); + String addr = this.conf.get("hbase.regionserver.info.bindAddress", "0.0.0.0"); + + boolean isMaster = false; + if (this instanceof HMaster) { +- port = conf.getInt(HConstants.MASTER_INFO_PORT, HConstants.DEFAULT_MASTER_INFOPORT); ++ port = conf.getInt(HConstants.MASTER_BOUND_INFO_PORT, ++ this.conf.getInt(HConstants.MASTER_INFO_PORT, HConstants.DEFAULT_MASTER_INFOPORT)); + addr = this.conf.get("hbase.master.info.bindAddress", "0.0.0.0"); + isMaster = true; + } +@@ -369,12 +373,18 @@ public abstract class HBaseServerBase> extends + LOG.info("Retry starting http info server with port: " + port); + } + } +- port = useThisInfoPortInstead; +- conf.setInt(HConstants.REGIONSERVER_INFO_PORT, port); ++ ++ // update bound ports ++ port = this.infoServer.getPort(); ++ conf.setInt(REGIONSERVER_BOUND_INFO_PORT, port); ++ conf.setInt(HConstants.MASTER_BOUND_INFO_PORT, port); ++ ++ // set advertised ports ++ conf.setInt(REGIONSERVER_INFO_PORT, useThisInfoPortInstead); + int masterInfoPort = + conf.getInt(HConstants.MASTER_INFO_PORT, HConstants.DEFAULT_MASTER_INFOPORT); + conf.setInt("hbase.master.info.port.orig", masterInfoPort); +- conf.setInt(HConstants.MASTER_INFO_PORT, port); ++ conf.setInt(HConstants.MASTER_INFO_PORT, useThisInfoPortInstead); + } + + /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java -index 816ef997cb..2114725986 100644 +index 0839a23f42..58a2935e2b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java -@@ -144,20 +144,20 @@ public class LocalHBaseCluster { +@@ -146,20 +146,20 @@ public class LocalHBaseCluster { // treat info ports special; expressly don't change '-1' (keep off) // in case we make that the default behavior. if ( @@ -64,7 +117,7 @@ index 816ef997cb..2114725986 100644 } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java -index d2b787ddef..5d1a547338 100644 +index 29138a2ddd..3e2408a0c1 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -17,7 +17,7 @@ @@ -76,7 +129,7 @@ index d2b787ddef..5d1a547338 100644 import static org.apache.hadoop.hbase.HConstants.MASTER_PORT; import static org.apache.hadoop.hbase.HConstants.DEFAULT_HBASE_SPLIT_COORDINATED_BY_ZK; import static org.apache.hadoop.hbase.HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS; -@@ -581,7 +581,7 @@ public class HMaster extends HRegionServer implements MasterServices { +@@ -624,7 +624,7 @@ public class HMaster extends HBaseServerBase implements Maste @Override protected int getUseThisInfoPortInstead(Configuration conf) { @@ -85,7 +138,7 @@ index d2b787ddef..5d1a547338 100644 return port != 0 ? port : this.infoServer != null ? this.infoServer.getPort() : -1; } -@@ -3176,7 +3176,7 @@ public class HMaster extends HRegionServer implements MasterServices { +@@ -3258,7 +3258,7 @@ public class HMaster extends HBaseServerBase implements Maste public int getRegionServerInfoPort(final ServerName sn) { int port = this.serverManager.getInfoPort(sn); return port == 0 @@ -95,18 +148,19 @@ index d2b787ddef..5d1a547338 100644 } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java -index c51f66678c..f9ef9d8473 100644 +index 0901f75d0c..b3886eb828 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java -@@ -25,6 +25,7 @@ import static org.apache.hadoop.hbase.HConstants.DEFAULT_SLOW_LOG_SYS_TABLE_CHOR +@@ -22,7 +22,7 @@ import static org.apache.hadoop.hbase.HConstants.DEFAULT_HBASE_SPLIT_WAL_MAX_SPL + import static org.apache.hadoop.hbase.HConstants.DEFAULT_SLOW_LOG_SYS_TABLE_CHORE_DURATION; import static org.apache.hadoop.hbase.HConstants.HBASE_SPLIT_WAL_COORDINATED_BY_ZK; import static org.apache.hadoop.hbase.HConstants.HBASE_SPLIT_WAL_MAX_SPLITTER; - import static org.apache.hadoop.hbase.HConstants.REGIONSERVER_BOUND_INFO_PORT; +-import static org.apache.hadoop.hbase.HConstants.REGIONSERVER_BOUND_INFO_PORT; +import static org.apache.hadoop.hbase.HConstants.REGIONSERVER_INFO_PORT; import static org.apache.hadoop.hbase.HConstants.REGIONSERVER_PORT; - import static org.apache.hadoop.hbase.HConstants.RPC_CLIENT_BIND_ADDRESS; import static org.apache.hadoop.hbase.master.waleventtracker.WALEventTrackerTableCreator.WAL_EVENT_TRACKER_ENABLED_DEFAULT; -@@ -794,7 +795,7 @@ public class HRegionServer extends Thread + import static org.apache.hadoop.hbase.master.waleventtracker.WALEventTrackerTableCreator.WAL_EVENT_TRACKER_ENABLED_KEY; +@@ -596,7 +596,7 @@ public class HRegionServer extends HBaseServerBase } protected int getUseThisInfoPortInstead(Configuration conf) { @@ -115,42 +169,3 @@ index c51f66678c..f9ef9d8473 100644 return port != 0 ? port : this.infoServer != null ? this.infoServer.getPort() : -1; } -@@ -2468,12 +2469,14 @@ public class HRegionServer extends Thread - */ - private void putUpWebUI() throws IOException { - int port = -- this.conf.getInt(HConstants.REGIONSERVER_INFO_PORT, HConstants.DEFAULT_REGIONSERVER_INFOPORT); -+ this.conf.getInt(REGIONSERVER_BOUND_INFO_PORT, -+ this.conf.getInt(REGIONSERVER_INFO_PORT, HConstants.DEFAULT_REGIONSERVER_INFOPORT)); - String addr = this.conf.get("hbase.regionserver.info.bindAddress", "0.0.0.0"); - - boolean isMaster = false; - if (this instanceof HMaster) { -- port = conf.getInt(HConstants.MASTER_INFO_PORT, HConstants.DEFAULT_MASTER_INFOPORT); -+ port = conf.getInt(HConstants.MASTER_BOUND_INFO_PORT, -+ this.conf.getInt(HConstants.MASTER_INFO_PORT, HConstants.DEFAULT_MASTER_INFOPORT)); - addr = this.conf.get("hbase.master.info.bindAddress", "0.0.0.0"); - isMaster = true; - } -@@ -2510,12 +2513,18 @@ public class HRegionServer extends Thread - LOG.info("Retry starting http info server with port: " + port); - } - } -- port = useThisInfoPortInstead; -- conf.setInt(HConstants.REGIONSERVER_INFO_PORT, port); -+ -+ // update bound ports -+ port = this.infoServer.getPort(); -+ conf.setInt(REGIONSERVER_BOUND_INFO_PORT, port); -+ conf.setInt(HConstants.MASTER_BOUND_INFO_PORT, port); -+ -+ // set advertised ports -+ conf.setInt(REGIONSERVER_INFO_PORT, useThisInfoPortInstead); - int masterInfoPort = - conf.getInt(HConstants.MASTER_INFO_PORT, HConstants.DEFAULT_MASTER_INFOPORT); - conf.setInt("hbase.master.info.port.orig", masterInfoPort); -- conf.setInt(HConstants.MASTER_INFO_PORT, port); -+ conf.setInt(HConstants.MASTER_INFO_PORT, useThisInfoPortInstead); - } - - /* diff --git a/hbase/hbase/stackable/patches/2.6.4/patchable.toml b/hbase/hbase/stackable/patches/3.0.0/patchable.toml similarity index 51% rename from hbase/hbase/stackable/patches/2.6.4/patchable.toml rename to hbase/hbase/stackable/patches/3.0.0/patchable.toml index 34de3ef44..5702f173f 100644 --- a/hbase/hbase/stackable/patches/2.6.4/patchable.toml +++ b/hbase/hbase/stackable/patches/3.0.0/patchable.toml @@ -1,2 +1,2 @@ mirror = "https://github.com/stackabletech/hbase.git" -base = "67f414cfed130bf6325570463ed36ea7d45aebc5" +base = "da418afa6973bf4222231c1d233410d8c18ac7ec" diff --git a/hbase/phoenix/boil-config.toml b/hbase/phoenix/boil-config.toml index 35027a1cf..1bfdd8dec 100644 --- a/hbase/phoenix/boil-config.toml +++ b/hbase/phoenix/boil-config.toml @@ -1,13 +1,3 @@ -[versions."5.3.0-hbase2.6.4".local-images] -"hbase/hbase" = "2.6.4" -"hadoop/hadoop" = "3.4.3" -java-devel = "11" - -[versions."5.3.0-hbase2.6.4".build-arguments] -phoenix-version = "5.3.0" -hbase-profile = "2.6" -delete-caches = "true" - [versions."5.3.1-hbase2.6.6".local-images] "hbase/hbase" = "2.6.6" "hadoop/hadoop" = "3.4.3" diff --git a/hbase/phoenix/stackable/patches/5.3.2/patchable.toml b/hbase/phoenix/stackable/patches/5.3.2/patchable.toml new file mode 100644 index 000000000..636e0d8a0 --- /dev/null +++ b/hbase/phoenix/stackable/patches/5.3.2/patchable.toml @@ -0,0 +1,2 @@ +mirror = "https://github.com/stackabletech/phoenix.git" +base = "77213611b442f599729045bda2cffc6c1de2c168"