diff --git a/charts/redis-ha/Chart.yaml b/charts/redis-ha/Chart.yaml index d4e21a86..5c4e920a 100644 --- a/charts/redis-ha/Chart.yaml +++ b/charts/redis-ha/Chart.yaml @@ -5,7 +5,7 @@ keywords: - redis - keyvalue - database -version: 4.39.0 +version: 4.40.0 appVersion: 8.8.0 description: This Helm chart provides a highly available Redis implementation with a master/slave configuration and uses Sentinel sidecars for failover management icon: https://img.icons8.com/external-tal-revivo-shadow-tal-revivo/24/external-redis-an-in-memory-data-structure-project-implementing-a-distributed-logo-shadow-tal-revivo.png diff --git a/charts/redis-ha/templates/_configs.tpl b/charts/redis-ha/templates/_configs.tpl index 450d1293..fef3594e 100644 --- a/charts/redis-ha/templates/_configs.tpl +++ b/charts/redis-ha/templates/_configs.tpl @@ -493,6 +493,199 @@ set -e } + master_actual_role() { + set +e + if [ "$REDIS_PORT" -eq 0 ]; then + MASTER_ACTUAL_ROLE=$(redis-cli {{ if .Values.auth }} -a "${AUTH}" --no-auth-warning{{ end }} -h "${MASTER}" -p "${REDIS_TLS_PORT}" --tls --cacert /tls-certs/{{ .Values.tls.caCertFile }} {{ if ne (default "yes" .Values.sentinel.authClients) "no"}} --cert /tls-certs/{{ .Values.tls.certFile }} --key /tls-certs/{{ .Values.tls.keyFile }}{{ end }} info | grep role | sed 's/role://' | sed 's/\r//') + else + MASTER_ACTUAL_ROLE=$(redis-cli {{ if .Values.auth }} -a "${AUTH}" --no-auth-warning{{ end }} -h "${MASTER}" -p "${REDIS_PORT}" info | grep role | sed 's/role://' | sed 's/\r//') + fi + set -e + } + + # Announce services publish not-ready addresses; the headless ${SERVICE} + # does not, so it stops resolving once every pod is unready. + REPLICA_COUNT={{ .Values.replicas }} + REDIS_ACTIVE_PORT="${REDIS_PORT}" + [ "$REDIS_PORT" -eq 0 ] && REDIS_ACTIVE_PORT="${REDIS_TLS_PORT}" + + announce_info() { + set +e + if [ "$REDIS_PORT" -eq 0 ]; then + redis-cli {{ if .Values.auth }} -a "${AUTH}" --no-auth-warning{{ end }} -h "$1" -p "${REDIS_TLS_PORT}" --tls --cacert /tls-certs/{{ .Values.tls.caCertFile }} {{ if ne (default "yes" .Values.sentinel.authClients) "no"}} --cert /tls-certs/{{ .Values.tls.certFile }} --key /tls-certs/{{ .Values.tls.keyFile }}{{ end }} info replication 2>/dev/null | sed 's/\r//' + else + redis-cli {{ if .Values.auth }} -a "${AUTH}" --no-auth-warning{{ end }} -h "$1" -p "${REDIS_PORT}" info replication 2>/dev/null | sed 's/\r//' + fi + set -e + } + + announce_cmd() { + set +e + host="$1"; shift + if [ "$REDIS_PORT" -eq 0 ]; then + redis-cli {{ if .Values.auth }} -a "${AUTH}" --no-auth-warning{{ end }} -h "$host" -p "${REDIS_TLS_PORT}" --tls --cacert /tls-certs/{{ .Values.tls.caCertFile }} {{ if ne (default "yes" .Values.sentinel.authClients) "no"}} --cert /tls-certs/{{ .Values.tls.certFile }} --key /tls-certs/{{ .Values.tls.keyFile }}{{ end }} "$@" + else + redis-cli {{ if .Values.auth }} -a "${AUTH}" --no-auth-warning{{ end }} -h "$host" -p "${REDIS_PORT}" "$@" + fi + set -e + } + + announce_sentinel_cmd() { + set +e + host="$1"; shift + if [ "$SENTINEL_PORT" -eq 0 ]; then + redis-cli {{ if .Values.sentinel.auth }} -a "${SENTINELAUTH}" --no-auth-warning{{ end }} -h "$host" -p "${SENTINEL_TLS_PORT}" --tls --cacert /tls-certs/{{ .Values.tls.caCertFile }} {{ if ne (default "yes" .Values.sentinel.authClients) "no"}} --cert /tls-certs/{{ .Values.tls.certFile }} --key /tls-certs/{{ .Values.tls.keyFile }}{{ end }} "$@" + else + redis-cli {{ if .Values.sentinel.auth }} -a "${SENTINELAUTH}" --no-auth-warning{{ end }} -h "$host" -p "${SENTINEL_PORT}" "$@" + fi + set -e + } + + # True if no member holds the master role. + masterless_now() { + _j=0 + while [ "$_j" -lt "$REPLICA_COUNT" ]; do + _r=$(announce_info "${SERVICE}-announce-${_j}" | awk -F: '/^role:/{print $2}') + [ "$_r" = "master" ] && return 1 + _j=$((_j + 1)) + done + return 0 + } + + # A masterless set never recovers on its own, so confirm it over seconds + # rather than over detection cycles. Any master seen aborts. + confirm_masterless_and_heal() { + _c=0 + while [ "$_c" -lt "$MASTERLESS_CONFIRMATIONS" ]; do + if ! masterless_now; then + echo "A member holds the master role; standing down after $_c/$MASTERLESS_CONFIRMATIONS confirmations." + return 1 + fi + _c=$((_c + 1)) + [ "$_c" -lt "$MASTERLESS_CONFIRMATIONS" ] && sleep "$MASTERLESS_CONFIRM_INTERVAL" + done + echo "No member held the master role across $MASTERLESS_CONFIRMATIONS consecutive checks." + promote_best_member + } + + # Promote the highest-offset member. Sentinel cannot: once every candidate + # looks stale it aborts with -failover-abort-no-good-slave / -NOGOODSLAVE. + promote_best_member() { + # Rescan: a liveness-killed redis is briefly unreachable. + _try=0 + while [ "$_try" -lt "$PROMOTE_SCAN_ATTEMPTS" ]; do + _best='' + _best_off=-1 + _i=0 + while [ "$_i" -lt "$REPLICA_COUNT" ]; do + _info=$(announce_info "${SERVICE}-announce-${_i}") + _role=$(echo "$_info" | awk -F: '/^role:/{print $2}') + if [ "$_role" = "master" ]; then + echo "A master is present at ${SERVICE}-announce-${_i}; no promotion needed." + return 1 + fi + _off=$(echo "$_info" | awk -F: '/_repl_offset:/{print $2; exit}') + case "$_off" in + ''|*[!0-9]*) ;; + *) if [ "$_off" -gt "$_best_off" ]; then _best_off="$_off"; _best="$_i"; fi ;; + esac + _i=$((_i + 1)) + done + [ -n "$_best" ] && break + _try=$((_try + 1)) + echo "Masterless, but no member answered (attempt $_try/$PROMOTE_SCAN_ATTEMPTS) — members may be restarting." + [ "$_try" -lt "$PROMOTE_SCAN_ATTEMPTS" ] && sleep "$MASTERLESS_CONFIRM_INTERVAL" + done + if [ -z "$_best" ]; then + echo "Masterless, but no member is reachable — not promoting." + return 1 + fi + _best_host="${SERVICE}-announce-${_best}" + _best_ip=$(getent hosts "$_best_host" | awk '{ print $1 }') + echo "ERROR: no member holds the master role. Promoting ${_best_host} (replication offset ${_best_off})." + announce_cmd "$_best_host" replicaof no one + _i=0 + while [ "$_i" -lt "$REPLICA_COUNT" ]; do + if [ "$_i" -ne "$_best" ] && [ -n "$_best_ip" ]; then + announce_cmd "${SERVICE}-announce-${_i}" replicaof "$_best_ip" "$REDIS_ACTIVE_PORT" + fi + _i=$((_i + 1)) + done + if [ -n "$_best_ip" ]; then + repoint_sentinels "$_best_ip" + else + echo "WARNING: could not resolve ${_best_host}; leaving sentinel config untouched." + fi + echo "Promotion complete: ${_best_host} is now master." + return 0 + } + + # Address and index of whichever member currently holds the master role + # ('' if none). + find_actual_master() { + ACTUAL_MASTER_INDEX='' + ACTUAL_MASTER_ADDR='' + _m=0 + while [ "$_m" -lt "$REPLICA_COUNT" ]; do + _r=$(announce_info "${SERVICE}-announce-${_m}" | awk -F: '/^role:/{print $2}') + if [ "$_r" = "master" ]; then + ACTUAL_MASTER_INDEX="$_m" + ACTUAL_MASTER_ADDR=$(getent_hosts "$_m" | awk '{ print $1 }') + return 0 + fi + _m=$((_m + 1)) + done + return 0 + } + + # Rewrite every sentinel's monitor entry to point at $1. 'sentinel reset' + # is not enough: it rediscovers from the stale configured address, so a + # wrong entry survives it. + repoint_sentinels() { + if [ "$SENTINEL_TLS_REPLICATION_ENABLED" = true ]; then + _monitor_port="${REDIS_TLS_PORT}" + else + _monitor_port="${REDIS_PORT}" + fi + _k=0 + while [ "$_k" -lt "$REPLICA_COUNT" ]; do + announce_sentinel_cmd "${SERVICE}-announce-${_k}" sentinel remove "${MASTER_GROUP}" + announce_sentinel_cmd "${SERVICE}-announce-${_k}" sentinel monitor "${MASTER_GROUP}" "$1" "$_monitor_port" "$QUORUM" + _k=$((_k + 1)) + done + echo "Repointed sentinels: monitor ${MASTER_GROUP} ${1}:${_monitor_port} quorum ${QUORUM}" + } + + # Sentinel's named master does not hold the master role. Trust the data + # plane instead of forcing a blind failover (which would demote a healthy + # master if sentinel's answer was merely stale): if some member really is + # master, repoint every sentinel at it; only if none is, force a failover + # and promote directly if sentinel cannot (-NOGOODSLAVE). + heal_stale_sentinel() { + find_actual_master + if [ -n "$ACTUAL_MASTER_INDEX" ]; then + if [ "$ACTUAL_MASTER_INDEX" = "$INDEX" ]; then + echo "This pod holds the master role after all; nothing to heal." + elif [ -n "$ACTUAL_MASTER_ADDR" ]; then + echo "ERROR: sentinel names ${MASTER} as master but ${SERVICE}-announce-${ACTUAL_MASTER_INDEX} (${ACTUAL_MASTER_ADDR}) holds the role. Repointing sentinels instead of failing over..." + repoint_sentinels "$ACTUAL_MASTER_ADDR" + else + echo "WARNING: ${SERVICE}-announce-${ACTUAL_MASTER_INDEX} holds the master role but its address did not resolve; leaving sentinel config untouched." + fi + return 0 + fi + echo "ERROR: no member holds the master role. Forcing a failover..." + if [ "$SENTINEL_PORT" -eq 0 ]; then + redis-cli -h "${SERVICE}" -p "${SENTINEL_TLS_PORT}" {{ if .Values.sentinel.auth }} -a "${SENTINELAUTH}" --no-auth-warning{{ end }} --tls --cacert /tls-certs/{{ .Values.tls.caCertFile }} {{ if ne (default "yes" .Values.sentinel.authClients) "no"}} --cert /tls-certs/{{ .Values.tls.certFile }} --key /tls-certs/{{ .Values.tls.keyFile }}{{ end }} sentinel failover "${MASTER_GROUP}" || true + else + redis-cli -h "${SERVICE}" -p "${SENTINEL_PORT}" {{ if .Values.sentinel.auth }} -a "${SENTINELAUTH}" --no-auth-warning{{ end }} sentinel failover "${MASTER_GROUP}" || true + fi + # SENTINEL FAILOVER returns -NOGOODSLAVE when no candidate looks + # fresh; promote directly instead. + sleep {{ .Values.splitBrainDetection.retryInterval }} + confirm_masterless_and_heal || true + } + identify_announce_ip while [ -z "${ANNOUNCE_IP}" ]; do @@ -503,6 +696,12 @@ QUORUM_FAIL_COUNT=0 MAX_QUORUM_FAILURES=${MAX_QUORUM_FAILURES:-5} + STALE_MASTER_COUNT=0 + MAX_STALE_MASTER_FAILURES=${MAX_STALE_MASTER_FAILURES:-3} + # Confirmations required before promoting, and the gap between them. + MASTERLESS_CONFIRMATIONS=${MASTERLESS_CONFIRMATIONS:-5} + MASTERLESS_CONFIRM_INTERVAL=${MASTERLESS_CONFIRM_INTERVAL:-5} + PROMOTE_SCAN_ATTEMPTS=${PROMOTE_SCAN_ATTEMPTS:-6} trap "exit 0" TERM while true; do @@ -521,8 +720,21 @@ redis_role echo "Redis role is $ROLE, expected role is master. No need to reinitialize." if [ "$ROLE" != "master" ]; then - echo "Redis role is $ROLE, expected role is master, reinitializing" - reinit + # Sentinel names this pod as master but it runs as a replica: + # reinit would re-derive `slaveof` and shutdown, restarting the + # pod straight back into a replica while sentinel keeps naming + # it (#383). This pod is also the fastest observer of the + # broken state, so heal here instead of waiting for peers' + # stale-master counters. Promotion stays offset-aware — a + # freshly replaced (empty) pod behind a stale sentinel record + # must not be promoted just because sentinel names it. + echo "Redis role is $ROLE but sentinel names this pod as master; healing from data-plane truth" + heal_stale_sentinel + if [ -n "$ACTUAL_MASTER_INDEX" ] && [ "$ACTUAL_MASTER_INDEX" != "$INDEX" ]; then + # Sentinels now point at the real master; reinit so + # init.sh reconfigures this pod as its replica. + reinit + fi fi fi elif [ "${MASTER}" ]; then @@ -542,10 +754,29 @@ reinit fi fi + else + # Agreeing with sentinel is not enough: the named master may + # itself be a replica, leaving the set with no master at all. + master_actual_role + if [ -z "${MASTER_ACTUAL_ROLE}" ] || [ "${MASTER_ACTUAL_ROLE}" = "master" ]; then + # healthy, or unreachable (sentinel's own down detection handles that) + STALE_MASTER_COUNT=0 + else + STALE_MASTER_COUNT=$((STALE_MASTER_COUNT + 1)) + echo "WARNING: Sentinel names ${MASTER} as master but its actual role is '${MASTER_ACTUAL_ROLE}' (set may be masterless). Failure count: $STALE_MASTER_COUNT/$MAX_STALE_MASTER_FAILURES" + if [ "$STALE_MASTER_COUNT" -ge "$MAX_STALE_MASTER_FAILURES" ]; then + echo "ERROR: Sentinel-named master has not held the master role for $MAX_STALE_MASTER_FAILURES consecutive checks. Healing..." + heal_stale_sentinel + STALE_MASTER_COUNT=0 + fi + fi fi else QUORUM_FAIL_COUNT=$((QUORUM_FAIL_COUNT + 1)) echo "WARNING: Sentinel returned no master (quorum may be broken). Failure count: $QUORUM_FAIL_COUNT/$MAX_QUORUM_FAILURES" + # No master named is also what a masterless set looks like; the + # reset below still runs on its own schedule for real quorum loss. + confirm_masterless_and_heal || true if [ "$QUORUM_FAIL_COUNT" -ge "$MAX_QUORUM_FAILURES" ]; then echo "ERROR: Quorum broken for $MAX_QUORUM_FAILURES consecutive checks. Attempting sentinel reset..." if [ "$SENTINEL_PORT" -eq 0 ]; then