diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile index 9200f5aa5593..8b4e05f47d92 100644 --- a/dev-support/Jenkinsfile +++ b/dev-support/Jenkinsfile @@ -39,7 +39,6 @@ pipeline { OUTPUT_DIR_RELATIVE_JDK11_HADOOP3 = 'output-jdk11-hadoop3' OUTPUT_DIR_RELATIVE_JDK17_HADOOP3 = 'output-jdk17-hadoop3' OUTPUT_DIR_RELATIVE_JDK21_HADOOP3 = 'output-jdk21-hadoop3' - OUTPUT_DIR_RELATIVE_READ_REPLICA = 'output-read-replica' PROJECT = 'hbase' PROJECT_PERSONALITY = 'https://raw.githubusercontent.com/apache/hbase/master/dev-support/hbase-personality.sh' @@ -138,7 +137,6 @@ pipeline { stash name: 'jdk11-hadoop3-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_JDK11_HADOOP3}/doesn't-match" stash name: 'jdk17-hadoop3-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_JDK17_HADOOP3}/doesn't-match" stash name: 'jdk21-hadoop3-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_JDK21_HADOOP3}/doesn't-match" - stash name: 'read-replica-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_READ_REPLICA}/doesn't-match" } } stage ('health checks') { @@ -323,7 +321,7 @@ pipeline { else echo "No archiver directory, skipping compressing." fi - ''' +''' sshPublisher(publishers: [ sshPublisherDesc(configName: 'Nightlies', transfers: [ @@ -342,7 +340,7 @@ pipeline { else echo "No test_logs.zip, skipping" fi - ''' +''' // Has to be relative to WORKSPACE. archiveArtifacts artifacts: "${env.OUTPUT_DIR_RELATIVE}/*" archiveArtifacts artifacts: "${env.OUTPUT_DIR_RELATIVE}/**/*" @@ -436,7 +434,7 @@ pipeline { else echo "No archiver directory, skipping compressing." fi - ''' +''' sshPublisher(publishers: [ sshPublisherDesc(configName: 'Nightlies', transfers: [ @@ -455,7 +453,7 @@ pipeline { else echo "No test_logs.zip, skipping" fi - ''' +''' // Has to be relative to WORKSPACE. archiveArtifacts artifacts: "${env.OUTPUT_DIR_RELATIVE}/*" archiveArtifacts artifacts: "${env.OUTPUT_DIR_RELATIVE}/**/*" @@ -584,6 +582,7 @@ pipeline { } } } + stage ('yetus jdk17 hadoop3 checks') { agent { node { @@ -804,81 +803,6 @@ pipeline { } } } - stage ('hbase read-replica feature checks') { - agent { - node { - label 'hbase' - } - } - when { - anyOf { - branch 'master' - branch 'branch-3' - } - } - environment { - BASEDIR = "${env.WORKSPACE}/component" - OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_READ_REPLICA}" - OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_READ_REPLICA}" - } - steps { - sh '''#!/usr/bin/env bash - set -e - rm -rf "${OUTPUT_DIR}" && mkdir -p "${OUTPUT_DIR}" - echo '(x) {color:red}-1 read-replica checks{color}' >"${OUTPUT_DIR}/commentfile" - echo "-- Something went wrong running this stage, please [check relevant console output|${BUILD_URL}/console]." >> "${OUTPUT_DIR}/commentfile" - ''' - dir('component') { - checkout scm - } - sh '''#!/usr/bin/env bash - set -e - rm -rf "${OUTPUT_DIR}/machine" && mkdir -p "${OUTPUT_DIR}/machine" - "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine" - echo "got the following saved stats in '${OUTPUT_DIR_RELATIVE}/machine'" - ls -lh "${OUTPUT_DIR_RELATIVE}/machine" - ''' - script { - def ret = sh( - returnStatus: true, - script: '''#!/usr/bin/env bash - set -e - declare -i status=0 - if "${BASEDIR}/dev-support/hbase_nightly_read_replica_test.sh" ; then - echo '(/) {color:green}+1 read-replica checks{color}' > "${OUTPUT_DIR}/commentfile" - else - echo '(x) {color:red}-1 read-replica checks{color}' > "${OUTPUT_DIR}/commentfile" - status=1 - fi - echo "-- For more information [see read-replica test report|${BUILD_URL}Read_20Replica_20Nightly_20Test_20Report/]" >> "${OUTPUT_DIR}/commentfile" - exit "${status}" - ''' - ) - if (ret != 0) { - // mark the build as UNSTABLE instead of FAILURE, to avoid skipping the later publish of - // test output. See HBASE-26339 for more details. - currentBuild.result = 'UNSTABLE' - } - } - } - post { - always { - junit testResults: "${env.OUTPUT_DIR_RELATIVE}/read-replica-nightly-test-results.xml", allowEmptyResults: true - - publishHTML target: [ - allowMissing : true, - keepAll : true, - alwaysLinkToLastBuild: true, - reportDir : "${env.OUTPUT_DIR_RELATIVE}", - reportFiles : 'read-replica-nightly-test-report.html', - reportName : 'Read Replica Nightly Test Report' - ] - - stash name: 'read-replica-result', includes: "${OUTPUT_DIR_RELATIVE}/commentfile" - archiveArtifacts artifacts: "${env.OUTPUT_DIR_RELATIVE}/**/*", allowEmptyArchive: true - } - } - } } // parallel } //stage:_health checks } //stages @@ -896,7 +820,6 @@ pipeline { rm -rf ${OUTPUT_DIR_RELATIVE_JDK11_HADOOP3} rm -rf ${OUTPUT_DIR_RELATIVE_JDK17_HADOOP3} rm -rf ${OUTPUT_DIR_RELATIVE_JDK21_HADOOP3} - rm -rf ${OUTPUT_DIR_RELATIVE_READ_REPLICA} ''' unstash 'general-result' unstash 'jdk8-hadoop2-result' @@ -904,7 +827,6 @@ pipeline { unstash 'jdk11-hadoop3-result' unstash 'jdk17-hadoop3-result' unstash 'jdk21-hadoop3-result' - unstash 'read-replica-result' def results = ["${env.OUTPUT_DIR_RELATIVE_GENERAL}/commentfile", "${env.OUTPUT_DIR_RELATIVE_JDK8_HADOOP2}/commentfile", @@ -912,7 +834,6 @@ pipeline { "${env.OUTPUT_DIR_RELATIVE_JDK11_HADOOP3}/commentfile", "${env.OUTPUT_DIR_RELATIVE_JDK17_HADOOP3}/commentfile", "${env.OUTPUT_DIR_RELATIVE_JDK21_HADOOP3}/commentfile"] - "${env.OUTPUT_DIR_RELATIVE_READ_REPLICA}/commentfile"] echo env.BRANCH_NAME echo env.BUILD_URL echo currentBuild.result diff --git a/dev-support/README.md b/dev-support/README.md index eb6cec921b43..d47780b0c2c3 100644 --- a/dev-support/README.md +++ b/dev-support/README.md @@ -60,12 +60,9 @@ configurations in this directory: - `docker/` -- Dockerfile for CI build environment - `hbase_docker/`, `hbase_docker.sh` -- Docker-based local test cluster -- `hbase_nightly_read_replica_test.sh` -- Builds a Docker image and runs two - HBase clusters (primary + read-replica) to execute end-to-end replication - integration tests via pytest. Optional `--keep-containers` and `--keep-image` - flags skip cleanup on exit. -- `read-replica/` -- Python test suite, Docker Compose configuration, and - cluster config used by `hbase_nightly_read_replica_test.sh` +- `read-replica/` -- Standalone Jenkins pipeline, test driver script + (`hbase_nightly_read_replica_test.sh`), Python test suite, Docker Compose + configuration, and cluster config for read-replica integration tests - `adhoc_run_tests/` -- Scripts for running test suites outside CI - `integration-test/` -- Integration test support diff --git a/dev-support/read-replica/.env b/dev-support/read-replica/.env index d6edb145d58f..d738d324bcb9 100644 --- a/dev-support/read-replica/.env +++ b/dev-support/read-replica/.env @@ -22,7 +22,7 @@ HBASE_CONTAINER_NAME=hbase-docker # This is the host running the hbase-docker containers. Use localhost if the containers # are running locally. If they are started by another container, such as a Jenkins # container in a Docker-out-of-Docker setup, then try setting this to host.docker.internal. -HBASE_HOST=${HBASE_HOST:-localhost} +HBASE_HOST=${HBASE_HOST:-host.docker.internal} # The directory within the docker container that contains the config files HBASE_CONF_DIR=/opt/hbase/conf # The directory containing the 'data-store/' directory. diff --git a/dev-support/read-replica/Jenkinsfile b/dev-support/read-replica/Jenkinsfile new file mode 100644 index 000000000000..fa39bbca3086 --- /dev/null +++ b/dev-support/read-replica/Jenkinsfile @@ -0,0 +1,144 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +pipeline { + agent any + options { + buildDiscarder(logRotator(numToKeepStr: '10')) + timeout (time: 2, unit: 'HOURS') + timestamps() + skipDefaultCheckout() + disableConcurrentBuilds() + } + environment { + OUTPUT_DIR_RELATIVE_READ_REPLICA = 'output-read-replica' + } + stages { + stage ('scm-checkout') { + steps { + dir('component') { + checkout scm + } + } + } + stage ('init health results') { + steps { + stash name: 'read-replica-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_READ_REPLICA}/doesn't-match" + } + } + stage ('hbase read-replica feature checks') { + environment { + BASEDIR = "${env.WORKSPACE}/component" + OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_READ_REPLICA}" + OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_READ_REPLICA}" + } + steps { + sh '''#!/usr/bin/env bash + set -e + rm -rf "${OUTPUT_DIR}" && mkdir -p "${OUTPUT_DIR}" + echo '(x) {color:red}-1 read-replica checks{color}' >"${OUTPUT_DIR}/commentfile" + echo "-- Something went wrong running this stage, please [check relevant console output|${BUILD_URL}/console]." >> "${OUTPUT_DIR}/commentfile" + ''' + dir('component') { + checkout scm + } + sh '''#!/usr/bin/env bash + set -e + rm -rf "${OUTPUT_DIR}/machine" && mkdir -p "${OUTPUT_DIR}/machine" + "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine" + echo "got the following saved stats in '${OUTPUT_DIR_RELATIVE}/machine'" + ls -lh "${OUTPUT_DIR_RELATIVE}/machine" + ''' + script { + def ret = sh( + returnStatus: true, + script: '''#!/usr/bin/env bash + set -e + declare -i status=0 + if "${BASEDIR}/dev-support/read-replica/hbase_nightly_read_replica_test.sh" ; then + echo '(/) {color:green}+1 read-replica checks{color}' > "${OUTPUT_DIR}/commentfile" + else + echo '(x) {color:red}-1 read-replica checks{color}' > "${OUTPUT_DIR}/commentfile" + status=1 + fi + echo "-- For more information [see read-replica test report|${BUILD_URL}Read_20Replica_20Nightly_20Test_20Report/]" >> "${OUTPUT_DIR}/commentfile" + exit "${status}" + ''' + ) + if (ret != 0) { + // mark the build as UNSTABLE instead of FAILURE to allow post processing of test reports (HBASE-26339) + currentBuild.result = 'UNSTABLE' + } + } + } + post { + always { + // 1. Publish JUnit XML metrics for Jenkins trend charts + junit testResults: "${env.OUTPUT_DIR_RELATIVE}/read-replica-nightly-test-results.xml", allowEmptyResults: true + + // 2. Render interactive HTML test report in the Jenkins UI navigation bar + publishHTML target: [ + allowMissing : true, + keepAll : true, + alwaysLinkToLastBuild: true, + reportDir : "${env.OUTPUT_DIR_RELATIVE}", + reportFiles : 'read-replica-nightly-test-report.html', + reportName : 'Read Replica Nightly Test Report' + ] + + stash name: 'read-replica-result', includes: "${OUTPUT_DIR_RELATIVE}/commentfile" + archiveArtifacts artifacts: "${env.OUTPUT_DIR_RELATIVE}/**/*", allowEmptyArchive: true + } + } + } + } + post { + always { + script { + try { + sh "printenv" + sh ''' + echo "Clean up result directories" + rm -rf ${OUTPUT_DIR_RELATIVE_READ_REPLICA} + ''' + unstash 'read-replica-result' + + def results = ["${env.OUTPUT_DIR_RELATIVE_READ_REPLICA}/commentfile"] + echo env.BRANCH_NAME + echo env.BUILD_URL + echo currentBuild.result + echo currentBuild.durationString + def comment = "Results for branch ${env.BRANCH_NAME}\n" + comment += "\t[build ${currentBuild.displayName} on builds.a.o|${env.BUILD_URL}]: " + if (currentBuild.result == null || currentBuild.result == "SUCCESS") { + comment += "(/) *{color:green}+1 overall{color}*\n" + } else { + comment += "(x) *{color:red}-1 overall{color}*\n" + } + comment += "----\ndetails (if available):\n\n" + echo "" + echo "[DEBUG] trying to aggregate step-wise results" + comment += results.collect { fileExists(file: it) ? readFile(file: it) : "" }.join("\n\n") + echo "[INFO] Final aggregated comment (would be posted to JIRA in production):" + echo comment + } catch (Exception exception) { + echo "Got exception: ${exception}" + echo " ${exception.getStackTrace()}" + } + } + } + } +} diff --git a/dev-support/read-replica/README.md b/dev-support/read-replica/README.md index e4b5c3b436a8..f446b1fab96d 100644 --- a/dev-support/read-replica/README.md +++ b/dev-support/read-replica/README.md @@ -113,13 +113,13 @@ read-replica/ ## CI: Jenkins Nightly Pipeline **Files:** -- `dev-support/Jenkinsfile` — stage definition (`hbase read-replica feature checks`) -- `dev-support/hbase_nightly_read_replica_test.sh` — test driver script +- `dev-support/read-replica/Jenkinsfile` — pipeline definition (`hbase read-replica feature checks`) +- `dev-support/read-replica/hbase_nightly_read_replica_test.sh` — test driver script ### When It Runs -The read-replica stage runs as part of the HBase nightly build on the `master` and `branch-3` -branches. It executes in parallel alongside the other nightly check stages (Yetus, JDK8/11/17). +The read-replica tests run as their own standalone nightly pipeline on the `master` and `branch-3` +branches, separate from the main HBase nightly build. ### What the Test Driver Does @@ -208,7 +208,7 @@ useful for reproducing test failures seen in CI. ```bash # From the repo root — run the full suite -dev-support/hbase_nightly_read_replica_test.sh +dev-support/read-replica/hbase_nightly_read_replica_test.sh ``` The script accepts two flags for local debugging: @@ -220,7 +220,7 @@ The script accepts two flags for local debugging: ```bash # Keep the image and containers for debugging -dev-support/hbase_nightly_read_replica_test.sh --keep-image --keep-containers +dev-support/read-replica/hbase_nightly_read_replica_test.sh --keep-image --keep-containers ``` ### Running Manually diff --git a/dev-support/hbase_nightly_read_replica_test.sh b/dev-support/read-replica/hbase_nightly_read_replica_test.sh similarity index 95% rename from dev-support/hbase_nightly_read_replica_test.sh rename to dev-support/read-replica/hbase_nightly_read_replica_test.sh index 528f8fa8e514..62860ee32b4b 100755 --- a/dev-support/hbase_nightly_read_replica_test.sh +++ b/dev-support/read-replica/hbase_nightly_read_replica_test.sh @@ -20,10 +20,9 @@ set -e -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPLICA_DIR="${SCRIPT_DIR}/read-replica" +REPLICA_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" OUTPUT_DIR="${OUTPUT_DIR:-${REPLICA_DIR}/output}" -export HBASE_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +export HBASE_ROOT="$(cd "${REPLICA_DIR}/../.." && pwd)" export HBASE_IMAGE="hbase-read-replica:${BUILD_NUMBER:-local}" @@ -48,12 +47,11 @@ while [[ $# -gt 0 ]]; do esac done -echo "Script dir: ${SCRIPT_DIR}" echo "Replica dir: ${REPLICA_DIR}" echo "Output dir: ${OUTPUT_DIR}" echo "HBase root: ${HBASE_ROOT}" -echo "Changing to replica dir: REPLICA_DIR" +echo "Changing to replica dir: ${REPLICA_DIR}" cd "${REPLICA_DIR}" echo "Sourcing environment file: $(pwd)/.env"