Skip to content

HDDS-14897. Add multiple S3 gateways to the rolling-upgrade suite #10028

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
dombizita wants to merge 3 commits into
base: HDDS-14496-zdu
Choose a base branch
from
Open

HDDS-14897. Add multiple S3 gateways to the rolling-upgrade suite #10028

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
023286a
HDDS-14897. Add multiple S3 gateways to the rolling-upgrade suite
dombizita Apr 1, 2026
7e2c25e
Comment out rolling upgrade run
dombizita Apr 1, 2026
215088b
Remove unnecessary s3g data directory
dombizita Apr 2, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion hadoop-ozone/dist/src/main/compose/testlib.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -278,7 +278,7 @@ reorder_om_nodes() {
"if [[ -f /etc/hadoop/ozone-site.xml ]]; then \
sed -i -e 's/om1,om2,om3/${new_order}/' /etc/hadoop/ozone-site.xml; \
echo 'Replaced OM order with ${new_order} in ${c}'; \
fi"
fi" || true
Copy link
Copy Markdown
Contributor

@adoroszlai adoroszlai Apr 2, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

silently swallow genuine bash failures

This is fine. In the worst case OM client will contact follower first with the original order.

done
fi
}
Expand Down
51 changes: 46 additions & 5 deletions hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-compose.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,16 @@ x-volumes:
- &ozone-dir ../../../..:${OZONE_DIR}
- &transformation ../../../../libexec/transformation.py:/opt/hadoop/libexec/transformation.py

x-s3-worker:
&s3-worker
command: ["ozone","s3g"]
<<: *common-config
environment:
<<: *environment
ports:
- 9878
- 19878

services:
kdc:
command: ["/opt/hadoop/compose/common/init-kdc.sh"]
Expand Down Expand Up @@ -265,18 +275,49 @@ services:
- *ozone-dir
- *transformation
s3g:
command: ["ozone","s3g"]
<<: *common-config
environment:
<<: *environment
image: haproxy:lts-alpine
hostname: s3g
networks:
net:
ipv4_address: 10.9.0.23
ports:
- 9878:9878
- 19878:19878
volumes:
- ../../../common/s3-haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg
command: ["haproxy", "-f", "/usr/local/etc/haproxy/haproxy.cfg"]
s3g1:
<<: *s3-worker
hostname: s3g1
networks:
net:
ipv4_address: 10.9.0.24
volumes:
- ${OZONE_VOLUME}/s3g1:/data
- *keytabs
- *krb5conf
- *ozone-dir
- *transformation
s3g2:
<<: *s3-worker
hostname: s3g2
networks:
net:
ipv4_address: 10.9.0.25
volumes:
- ${OZONE_VOLUME}/s3g2:/data
- *keytabs
- *krb5conf
- *ozone-dir
- *transformation
s3g3:
<<: *s3-worker
hostname: s3g3
networks:
net:
ipv4_address: 10.9.0.26
volumes:
- ${OZONE_VOLUME}/s3g:/data
- ${OZONE_VOLUME}/s3g3:/data
- *keytabs
- *krb5conf
- *ozone-dir
Expand Down
2 changes: 1 addition & 1 deletion hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/load.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,6 @@ export COMPOSE_FILE="$TEST_DIR/compose/ha/docker-compose.yaml"
export OM_SERVICE_ID=omservice
export SECURITY_ENABLED="true"

create_data_dirs dn{1..5} kms om{1..3} recon s3g scm{1..3}
create_data_dirs dn{1..5} kms om{1..3} recon s3g{1..3} scm{1..3}

echo "Using docker cluster defined in $COMPOSE_FILE"
20 changes: 8 additions & 12 deletions hadoop-ozone/dist/src/main/compose/upgrade/upgrades/rolling-upgrade/driver.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,20 +54,13 @@ rolling_restart_service() {
fi
fi

# The data generation/validation is doing S3 API tests, so skip it in case the S3 gateway is updated
# TODO find a better solution
if [[ ${SERVICE} != "s3g" ]]; then
callback before_service_restart
fi
callback before_service_restart

# Restart service with new image.
prepare_for_image "${OZONE_UPGRADE_TO}"
create_containers "${SERVICE}"

# The data generation/validation is doing S3 API tests, so skip it in case the S3 gateway is updated
if [[ ${SERVICE} != "s3g" ]]; then
callback after_service_restart
fi
callback after_service_restart

# Service-specific readiness checks.
case "${SERVICE}" in
Expand Down Expand Up @@ -112,14 +105,17 @@ for s in dn1 dn2 dn3 dn4 dn5; do
rolling_restart_service "$s" "$OZONE_UPGRADE_TO"
done

# OMs
for s in om1 om2 om3; do
OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-2-${s}"
rolling_restart_service "$s" "$OZONE_UPGRADE_TO"
done

# S3 Gateway
OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-2-s3g"
rolling_restart_service "s3g" "$OZONE_UPGRADE_TO"
# S3 Gateways (s3g is HAProxy and does not need to be upgraded)
for s in s3g1 s3g2 s3g3; do
OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-2-${s}"
rolling_restart_service "$s" "$OZONE_UPGRADE_TO"
done

# TODO Add downgrade scenario

Expand Down
Loading