Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
89 changes: 5 additions & 84 deletions dev-support/Jenkinsfile
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,6 @@ pipeline {
OUTPUT_DIR_RELATIVE_JDK11_HADOOP3 = 'output-jdk11-hadoop3'
OUTPUT_DIR_RELATIVE_JDK17_HADOOP3 = 'output-jdk17-hadoop3'
OUTPUT_DIR_RELATIVE_JDK21_HADOOP3 = 'output-jdk21-hadoop3'
OUTPUT_DIR_RELATIVE_READ_REPLICA = 'output-read-replica'

PROJECT = 'hbase'
PROJECT_PERSONALITY = 'https://raw.githubusercontent.com/apache/hbase/master/dev-support/hbase-personality.sh'
Expand Down Expand Up @@ -138,7 +137,6 @@ pipeline {
stash name: 'jdk11-hadoop3-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_JDK11_HADOOP3}/doesn't-match"
stash name: 'jdk17-hadoop3-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_JDK17_HADOOP3}/doesn't-match"
stash name: 'jdk21-hadoop3-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_JDK21_HADOOP3}/doesn't-match"
stash name: 'read-replica-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_READ_REPLICA}/doesn't-match"
}
}
stage ('health checks') {
Expand Down Expand Up @@ -323,7 +321,7 @@ pipeline {
else
echo "No archiver directory, skipping compressing."
fi
'''
'''
sshPublisher(publishers: [
sshPublisherDesc(configName: 'Nightlies',
transfers: [
Expand All @@ -342,7 +340,7 @@ pipeline {
else
echo "No test_logs.zip, skipping"
fi
'''
'''
// Has to be relative to WORKSPACE.
archiveArtifacts artifacts: "${env.OUTPUT_DIR_RELATIVE}/*"
archiveArtifacts artifacts: "${env.OUTPUT_DIR_RELATIVE}/**/*"
Expand Down Expand Up @@ -436,7 +434,7 @@ pipeline {
else
echo "No archiver directory, skipping compressing."
fi
'''
'''
sshPublisher(publishers: [
sshPublisherDesc(configName: 'Nightlies',
transfers: [
Expand All @@ -455,7 +453,7 @@ pipeline {
else
echo "No test_logs.zip, skipping"
fi
'''
'''
// Has to be relative to WORKSPACE.
archiveArtifacts artifacts: "${env.OUTPUT_DIR_RELATIVE}/*"
archiveArtifacts artifacts: "${env.OUTPUT_DIR_RELATIVE}/**/*"
Expand Down Expand Up @@ -584,6 +582,7 @@ pipeline {
}
}
}

stage ('yetus jdk17 hadoop3 checks') {
agent {
node {
Expand Down Expand Up @@ -804,81 +803,6 @@ pipeline {
}
}
}
stage ('hbase read-replica feature checks') {
agent {
node {
label 'hbase'
}
}
when {
anyOf {
branch 'master'
branch 'branch-3'
}
}
environment {
BASEDIR = "${env.WORKSPACE}/component"
OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_READ_REPLICA}"
OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_READ_REPLICA}"
}
steps {
sh '''#!/usr/bin/env bash
set -e
rm -rf "${OUTPUT_DIR}" && mkdir -p "${OUTPUT_DIR}"
echo '(x) {color:red}-1 read-replica checks{color}' >"${OUTPUT_DIR}/commentfile"
echo "-- Something went wrong running this stage, please [check relevant console output|${BUILD_URL}/console]." >> "${OUTPUT_DIR}/commentfile"
'''
dir('component') {
checkout scm
}
sh '''#!/usr/bin/env bash
set -e
rm -rf "${OUTPUT_DIR}/machine" && mkdir -p "${OUTPUT_DIR}/machine"
"${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine"
echo "got the following saved stats in '${OUTPUT_DIR_RELATIVE}/machine'"
ls -lh "${OUTPUT_DIR_RELATIVE}/machine"
'''
script {
def ret = sh(
returnStatus: true,
script: '''#!/usr/bin/env bash
set -e
declare -i status=0
if "${BASEDIR}/dev-support/hbase_nightly_read_replica_test.sh" ; then
echo '(/) {color:green}+1 read-replica checks{color}' > "${OUTPUT_DIR}/commentfile"
else
echo '(x) {color:red}-1 read-replica checks{color}' > "${OUTPUT_DIR}/commentfile"
status=1
fi
echo "-- For more information [see read-replica test report|${BUILD_URL}Read_20Replica_20Nightly_20Test_20Report/]" >> "${OUTPUT_DIR}/commentfile"
exit "${status}"
'''
)
if (ret != 0) {
// mark the build as UNSTABLE instead of FAILURE, to avoid skipping the later publish of
// test output. See HBASE-26339 for more details.
currentBuild.result = 'UNSTABLE'
}
}
}
post {
always {
junit testResults: "${env.OUTPUT_DIR_RELATIVE}/read-replica-nightly-test-results.xml", allowEmptyResults: true

publishHTML target: [
allowMissing : true,
keepAll : true,
alwaysLinkToLastBuild: true,
reportDir : "${env.OUTPUT_DIR_RELATIVE}",
reportFiles : 'read-replica-nightly-test-report.html',
reportName : 'Read Replica Nightly Test Report'
]

stash name: 'read-replica-result', includes: "${OUTPUT_DIR_RELATIVE}/commentfile"
archiveArtifacts artifacts: "${env.OUTPUT_DIR_RELATIVE}/**/*", allowEmptyArchive: true
}
}
}
} // parallel
} //stage:_health checks
} //stages
Expand All @@ -896,23 +820,20 @@ pipeline {
rm -rf ${OUTPUT_DIR_RELATIVE_JDK11_HADOOP3}
rm -rf ${OUTPUT_DIR_RELATIVE_JDK17_HADOOP3}
rm -rf ${OUTPUT_DIR_RELATIVE_JDK21_HADOOP3}
rm -rf ${OUTPUT_DIR_RELATIVE_READ_REPLICA}
'''
unstash 'general-result'
unstash 'jdk8-hadoop2-result'
unstash 'jdk8-hadoop3-result'
unstash 'jdk11-hadoop3-result'
unstash 'jdk17-hadoop3-result'
unstash 'jdk21-hadoop3-result'
unstash 'read-replica-result'

def results = ["${env.OUTPUT_DIR_RELATIVE_GENERAL}/commentfile",
"${env.OUTPUT_DIR_RELATIVE_JDK8_HADOOP2}/commentfile",
"${env.OUTPUT_DIR_RELATIVE_JDK8_HADOOP3}/commentfile",
"${env.OUTPUT_DIR_RELATIVE_JDK11_HADOOP3}/commentfile",
"${env.OUTPUT_DIR_RELATIVE_JDK17_HADOOP3}/commentfile",
"${env.OUTPUT_DIR_RELATIVE_JDK21_HADOOP3}/commentfile"]
"${env.OUTPUT_DIR_RELATIVE_READ_REPLICA}/commentfile"]
echo env.BRANCH_NAME
echo env.BUILD_URL
echo currentBuild.result
Expand Down
9 changes: 3 additions & 6 deletions dev-support/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -60,12 +60,9 @@ configurations in this directory:

- `docker/` -- Dockerfile for CI build environment
- `hbase_docker/`, `hbase_docker.sh` -- Docker-based local test cluster
- `hbase_nightly_read_replica_test.sh` -- Builds a Docker image and runs two
HBase clusters (primary + read-replica) to execute end-to-end replication
integration tests via pytest. Optional `--keep-containers` and `--keep-image`
flags skip cleanup on exit.
- `read-replica/` -- Python test suite, Docker Compose configuration, and
cluster config used by `hbase_nightly_read_replica_test.sh`
- `read-replica/` -- Standalone Jenkins pipeline, test driver script
(`hbase_nightly_read_replica_test.sh`), Python test suite, Docker Compose
configuration, and cluster config for read-replica integration tests
- `adhoc_run_tests/` -- Scripts for running test suites outside CI
- `integration-test/` -- Integration test support

Expand Down
2 changes: 1 addition & 1 deletion dev-support/read-replica/.env
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ HBASE_CONTAINER_NAME=hbase-docker
# This is the host running the hbase-docker containers. Use localhost if the containers
# are running locally. If they are started by another container, such as a Jenkins
# container in a Docker-out-of-Docker setup, then try setting this to host.docker.internal.
HBASE_HOST=${HBASE_HOST:-localhost}
HBASE_HOST=${HBASE_HOST:-host.docker.internal}

@kgeisz kgeisz Sep 17, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@taklwu this needs to be set to host.docker.internal in order to get it running in my local Jenkins setup. We will see how this goes with the upstream Jenkins.

HBASE_HOST is used for curling the HBase UI to see if it's up. It's part of the process of verifying HBase has started. (here it is in the code)

The only problem with this is if I want to run the tests locally with hbase_nightly_read_replica_test.sh (without Jenkins) then HBASE_HOST should be localhost. We may want to think of a way to make it easier for a user to run this without having to know they should modify HBASE_HOST to make it localhost.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can we use something like hostname -a then it could auto detect the hostname ?

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

seems like you can use it in build-images.sh line #21 and overwrite the hostname? or is it how the .env is being used?

. ./.env

# The directory within the docker container that contains the config files
HBASE_CONF_DIR=/opt/hbase/conf
# The directory containing the 'data-store/<hbase.rootdir>' directory.
Expand Down
144 changes: 144 additions & 0 deletions dev-support/read-replica/Jenkinsfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,144 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
pipeline {
agent any
options {
buildDiscarder(logRotator(numToKeepStr: '10'))
timeout (time: 2, unit: 'HOURS')
timestamps()
skipDefaultCheckout()
disableConcurrentBuilds()
}
environment {
OUTPUT_DIR_RELATIVE_READ_REPLICA = 'output-read-replica'
}
stages {
stage ('scm-checkout') {
steps {
dir('component') {
checkout scm
}
}
}
stage ('init health results') {
steps {
stash name: 'read-replica-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_READ_REPLICA}/doesn't-match"
}
}
stage ('hbase read-replica feature checks') {
environment {
BASEDIR = "${env.WORKSPACE}/component"
OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_READ_REPLICA}"
OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_READ_REPLICA}"
}
steps {
sh '''#!/usr/bin/env bash
set -e
rm -rf "${OUTPUT_DIR}" && mkdir -p "${OUTPUT_DIR}"
echo '(x) {color:red}-1 read-replica checks{color}' >"${OUTPUT_DIR}/commentfile"
echo "-- Something went wrong running this stage, please [check relevant console output|${BUILD_URL}/console]." >> "${OUTPUT_DIR}/commentfile"
'''
dir('component') {
checkout scm
}
sh '''#!/usr/bin/env bash
set -e
rm -rf "${OUTPUT_DIR}/machine" && mkdir -p "${OUTPUT_DIR}/machine"
"${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine"
echo "got the following saved stats in '${OUTPUT_DIR_RELATIVE}/machine'"
ls -lh "${OUTPUT_DIR_RELATIVE}/machine"
'''
script {
def ret = sh(
returnStatus: true,
script: '''#!/usr/bin/env bash
set -e
declare -i status=0
if "${BASEDIR}/dev-support/read-replica/hbase_nightly_read_replica_test.sh" ; then
echo '(/) {color:green}+1 read-replica checks{color}' > "${OUTPUT_DIR}/commentfile"
else
echo '(x) {color:red}-1 read-replica checks{color}' > "${OUTPUT_DIR}/commentfile"
status=1
fi
echo "-- For more information [see read-replica test report|${BUILD_URL}Read_20Replica_20Nightly_20Test_20Report/]" >> "${OUTPUT_DIR}/commentfile"
exit "${status}"
'''
)
if (ret != 0) {
// mark the build as UNSTABLE instead of FAILURE to allow post processing of test reports (HBASE-26339)
currentBuild.result = 'UNSTABLE'
}
}
}
post {
always {
// 1. Publish JUnit XML metrics for Jenkins trend charts
junit testResults: "${env.OUTPUT_DIR_RELATIVE}/read-replica-nightly-test-results.xml", allowEmptyResults: true

// 2. Render interactive HTML test report in the Jenkins UI navigation bar
publishHTML target: [
allowMissing : true,
keepAll : true,
alwaysLinkToLastBuild: true,
reportDir : "${env.OUTPUT_DIR_RELATIVE}",
reportFiles : 'read-replica-nightly-test-report.html',
reportName : 'Read Replica Nightly Test Report'
]

stash name: 'read-replica-result', includes: "${OUTPUT_DIR_RELATIVE}/commentfile"
archiveArtifacts artifacts: "${env.OUTPUT_DIR_RELATIVE}/**/*", allowEmptyArchive: true
}
}
}
}
post {
always {
script {
try {
sh "printenv"
sh '''
echo "Clean up result directories"
rm -rf ${OUTPUT_DIR_RELATIVE_READ_REPLICA}
'''
unstash 'read-replica-result'

def results = ["${env.OUTPUT_DIR_RELATIVE_READ_REPLICA}/commentfile"]
echo env.BRANCH_NAME
echo env.BUILD_URL
echo currentBuild.result
echo currentBuild.durationString
def comment = "Results for branch ${env.BRANCH_NAME}\n"
comment += "\t[build ${currentBuild.displayName} on builds.a.o|${env.BUILD_URL}]: "
if (currentBuild.result == null || currentBuild.result == "SUCCESS") {
comment += "(/) *{color:green}+1 overall{color}*\n"
} else {
comment += "(x) *{color:red}-1 overall{color}*\n"
}
comment += "----\ndetails (if available):\n\n"
echo ""
echo "[DEBUG] trying to aggregate step-wise results"
comment += results.collect { fileExists(file: it) ? readFile(file: it) : "" }.join("\n\n")
echo "[INFO] Final aggregated comment (would be posted to JIRA in production):"
echo comment
} catch (Exception exception) {
echo "Got exception: ${exception}"
echo " ${exception.getStackTrace()}"
}
}
}
}
}
12 changes: 6 additions & 6 deletions dev-support/read-replica/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -113,13 +113,13 @@ read-replica/
## CI: Jenkins Nightly Pipeline

**Files:**
- `dev-support/Jenkinsfile` — stage definition (`hbase read-replica feature checks`)
- `dev-support/hbase_nightly_read_replica_test.sh` — test driver script
- `dev-support/read-replica/Jenkinsfile` — pipeline definition (`hbase read-replica feature checks`)
- `dev-support/read-replica/hbase_nightly_read_replica_test.sh` — test driver script

### When It Runs

The read-replica stage runs as part of the HBase nightly build on the `master` and `branch-3`
branches. It executes in parallel alongside the other nightly check stages (Yetus, JDK8/11/17).
The read-replica tests run as their own standalone nightly pipeline on the `master` and `branch-3`
branches, separate from the main HBase nightly build.

### What the Test Driver Does

Expand Down Expand Up @@ -208,7 +208,7 @@ useful for reproducing test failures seen in CI.

```bash
# From the repo root — run the full suite
dev-support/hbase_nightly_read_replica_test.sh
dev-support/read-replica/hbase_nightly_read_replica_test.sh
```

The script accepts two flags for local debugging:
Expand All @@ -220,7 +220,7 @@ The script accepts two flags for local debugging:

```bash
# Keep the image and containers for debugging
dev-support/hbase_nightly_read_replica_test.sh --keep-image --keep-containers
dev-support/read-replica/hbase_nightly_read_replica_test.sh --keep-image --keep-containers
```

### Running Manually
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,9 @@

set -e

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPLICA_DIR="${SCRIPT_DIR}/read-replica"
REPLICA_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
OUTPUT_DIR="${OUTPUT_DIR:-${REPLICA_DIR}/output}"
export HBASE_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
export HBASE_ROOT="$(cd "${REPLICA_DIR}/../.." && pwd)"

export HBASE_IMAGE="hbase-read-replica:${BUILD_NUMBER:-local}"

Expand All @@ -48,12 +47,11 @@ while [[ $# -gt 0 ]]; do
esac
done

echo "Script dir: ${SCRIPT_DIR}"
echo "Replica dir: ${REPLICA_DIR}"
echo "Output dir: ${OUTPUT_DIR}"
echo "HBase root: ${HBASE_ROOT}"

echo "Changing to replica dir: REPLICA_DIR"
echo "Changing to replica dir: ${REPLICA_DIR}"
cd "${REPLICA_DIR}"

echo "Sourcing environment file: $(pwd)/.env"
Expand Down
Loading