From 0abcfa1b887b896f842adb1e21a537fc0ea4b13d Mon Sep 17 00:00:00 2001
From: Robert Moucha <robert.moucha@gooddata.com>
Date: Thu, 8 Oct 2026 16:25:21 +0200
Subject: [PATCH 1/8] fix(redis-ha): ask every sentinel through its announce
 service

init.sh and fix-split-brain.sh reached Sentinel through the headless
service, which lists only ready pods. A replica that loses its master
fails readiness, so once the master was gone long enough no pod could
find Sentinel. init.sh then fell back to following announce-0 even when
announce-0 was itself a replica, and the set ended with no master.

Query each sentinel through its announce service, which publishes
not-ready addresses, and use the most common answer. Send forced
failovers and resets the same way.

Signed-off-by: Robert Moucha <robert.moucha@gooddata.com>
Co-authored-by: Claude <noreply@anthropic.com>
---
 charts/redis-ha/templates/_configs.tpl | 100 ++++++++++++++-----------
 1 file changed, 57 insertions(+), 43 deletions(-)

diff --git a/charts/redis-ha/templates/_configs.tpl b/charts/redis-ha/templates/_configs.tpl
index 450d129..a34b25a 100644
--- a/charts/redis-ha/templates/_configs.tpl
+++ b/charts/redis-ha/templates/_configs.tpl
@@ -89,25 +89,51 @@
 {{- end }}
 
 {{- define "lib.sh" }}
+    sentinel_cli() {
+        _host="${1}"
+        shift
+        if [ "$SENTINEL_PORT" -eq 0 ]; then
+            redis-cli -h "${_host}" -p "${SENTINEL_TLS_PORT}" {{ if .Values.sentinel.auth }} -a "${SENTINELAUTH}" --no-auth-warning{{ end }} --tls --cacert /tls-certs/{{ .Values.tls.caCertFile }} {{ if ne (default "yes" .Values.sentinel.authClients) "no"}} --cert /tls-certs/{{ .Values.tls.certFile }} --key /tls-certs/{{ .Values.tls.keyFile }}{{ end }} "$@"
+        else
+            redis-cli -h "${_host}" -p "${SENTINEL_PORT}" {{ if .Values.sentinel.auth }} -a "${SENTINELAUTH}" --no-auth-warning{{ end }} "$@"
+        fi
+    }
+
+    # The headless service lists only ready pods, and a replica that lost its
+    # master is not ready. Ask every sentinel through its announce service,
+    # which also publishes not-ready pods, and take the most common answer.
     sentinel_get_master() {
     set +e
-        if [ "$SENTINEL_PORT" -eq 0 ]; then
-            if [ "$RESOLVE_HOSTNAMES" = true ]; then
-                redis-cli -h "${SERVICE}" -p "${SENTINEL_TLS_PORT}" {{ if .Values.sentinel.auth }} -a "${SENTINELAUTH}" --no-auth-warning{{ end }} --tls --cacert /tls-certs/{{ .Values.tls.caCertFile }} {{ if ne (default "yes" .Values.sentinel.authClients) "no"}} --cert /tls-certs/{{ .Values.tls.certFile }} --key /tls-certs/{{ .Values.tls.keyFile }}{{ end }} sentinel get-master-addr-by-name "${MASTER_GROUP}" |\
-                head -n 1 | grep -E '^\s*[a-zA-Z0-9.-]+\s*$'
-            else
-                redis-cli -h "${SERVICE}" -p "${SENTINEL_TLS_PORT}" {{ if .Values.sentinel.auth }} -a "${SENTINELAUTH}" --no-auth-warning{{ end }} --tls --cacert /tls-certs/{{ .Values.tls.caCertFile }} {{ if ne (default "yes" .Values.sentinel.authClients) "no"}} --cert /tls-certs/{{ .Values.tls.certFile }} --key /tls-certs/{{ .Values.tls.keyFile }}{{ end }} sentinel get-master-addr-by-name "${MASTER_GROUP}" |\
-                head -n 1 | grep -E '((^\s*((([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))\s*$)|(^\s*((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?s*$))'
-            fi
+        if [ "$RESOLVE_HOSTNAMES" = true ]; then
+            _addr_regex='^\s*[a-zA-Z0-9.-]+\s*$'
         else
-            if [ "$RESOLVE_HOSTNAMES" = true ]; then
-                redis-cli -h "${SERVICE}" -p "${SENTINEL_PORT}" {{ if .Values.sentinel.auth }} -a "${SENTINELAUTH}" --no-auth-warning{{ end }} sentinel get-master-addr-by-name "${MASTER_GROUP}" |\
-                head -n 1 | grep -E '^\s*[a-zA-Z0-9.-]+\s*$'
-            else
-                redis-cli -h "${SERVICE}" -p "${SENTINEL_PORT}" {{ if .Values.sentinel.auth }} -a "${SENTINELAUTH}" --no-auth-warning{{ end }} sentinel get-master-addr-by-name "${MASTER_GROUP}" |\
-                head -n 1 | grep -E '((^\s*((([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))\s*$)|(^\s*((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?s*$))'
-            fi
+            _addr_regex='((^\s*((([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))\s*$)|(^\s*((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?s*$))'
         fi
+        _answers=''
+        _i=0
+        while [ "${_i}" -lt "${REPLICA_COUNT}" ]; do
+            _answer=$(sentinel_cli "${SERVICE}-announce-${_i}" sentinel get-master-addr-by-name "${MASTER_GROUP}" 2>/dev/null | head -n 1 | grep -E "${_addr_regex}")
+            _answers="${_answers} ${_answer}"
+            _i=$((_i + 1))
+        done
+        echo "${_answers}" | awk '{ for (i = 1; i <= NF; i++) votes[$i]++ } END { for (a in votes) if (votes[a] > best) { best = votes[a]; master = a } print master }'
+    set -e
+    }
+
+    # Ask the sentinels in turn to fail over, until one accepts or refuses
+    # for lack of a good replica. Prints that sentinel's reply.
+    sentinel_failover() {
+    set +e
+        _reply=''
+        _i=0
+        while [ "${_i}" -lt "${REPLICA_COUNT}" ]; do
+            _reply=$(sentinel_cli "${SERVICE}-announce-${_i}" sentinel failover "${MASTER_GROUP}" 2>&1)
+            case "${_reply}" in
+                OK|*NOGOODSLAVE*) break ;;
+            esac
+            _i=$((_i + 1))
+        done
+        echo "${_reply}"
     set -e
     }
 
@@ -127,7 +153,7 @@
 
     identify_master() {
         echo "Identifying redis master (get-master-addr-by-name).."
-        echo "  using sentinel ({{ template "redis-ha.fullname" . }}), sentinel group name ({{ template "redis-ha.masterGroupName" . }})"
+        echo "  using sentinels ({{ template "redis-ha.fullname" . }}-announce-*), sentinel group name ({{ template "redis-ha.masterGroupName" . }})"
         MASTER="$(sentinel_get_master_retry 3)"
         if [ -n "${MASTER}" ]; then
             echo "  $(date) Found redis master (${MASTER})"
@@ -250,33 +276,18 @@
         if [ "$(redis_ping_retry 3)" != "PONG" ]; then
             echo "  $(date) Can't ping redis master (${MASTER})"
             echo "Attempting to force failover (sentinel failover).."
-
-            if [ "$SENTINEL_PORT" -eq 0 ]; then
-                echo "  on sentinel (${SERVICE}:${SENTINEL_TLS_PORT}), sentinel grp (${MASTER_GROUP})"
-                if redis-cli -h "${SERVICE}" -p "${SENTINEL_TLS_PORT}" {{ if .Values.sentinel.auth }} -a "${SENTINELAUTH}" --no-auth-warning{{ end }} --tls --cacert /tls-certs/{{ .Values.tls.caCertFile }} {{ if ne (default "yes" .Values.sentinel.authClients) "no"}} --cert /tls-certs/{{ .Values.tls.certFile }} --key /tls-certs/{{ .Values.tls.keyFile }}{{ end }} sentinel failover "${MASTER_GROUP}" | grep -q 'NOGOODSLAVE' ; then
-                    echo "  $(date) Failover returned with 'NOGOODSLAVE'"
-                    echo "Setting defaults for this pod.."
-                    setup_defaults
-                    return 0
-                fi
-            else
-                echo "  on sentinel (${SERVICE}:${SENTINEL_PORT}), sentinel grp (${MASTER_GROUP})"
-                if redis-cli -h "${SERVICE}" -p "${SENTINEL_PORT}" {{ if .Values.sentinel.auth }} -a "${SENTINELAUTH}" --no-auth-warning{{ end }} sentinel failover "${MASTER_GROUP}" | grep -q 'NOGOODSLAVE' ; then
-                    echo "  $(date) Failover returned with 'NOGOODSLAVE'"
-                    echo "Setting defaults for this pod.."
-                    setup_defaults
-                    return 0
-                fi
+            echo "  on sentinels (${SERVICE}-announce-*), sentinel grp (${MASTER_GROUP})"
+            if sentinel_failover | grep -q 'NOGOODSLAVE' ; then
+                echo "  $(date) Failover returned with 'NOGOODSLAVE'"
+                echo "Setting defaults for this pod.."
+                setup_defaults
+                return 0
             fi
 
             echo "Hold on for 10sec"
             sleep 10
             echo "We should get redis master's ip now. Asking (get-master-addr-by-name).."
-            if [ "$SENTINEL_PORT" -eq 0 ]; then
-                echo "  sentinel (${SERVICE}:${SENTINEL_TLS_PORT}), sentinel grp (${MASTER_GROUP})"
-            else
-                echo "  sentinel (${SERVICE}:${SENTINEL_PORT}), sentinel grp (${MASTER_GROUP})"
-            fi
+            echo "  sentinels (${SERVICE}-announce-*), sentinel grp (${MASTER_GROUP})"
             MASTER="$(sentinel_get_master)"
             if [ "${MASTER}" ]; then
                 echo "  $(date) Found redis master (${MASTER})"
@@ -343,6 +354,7 @@
     SENTINEL_CONF=/data/conf/sentinel.conf
     SENTINEL_TLS_PORT={{ .Values.sentinel.tlsPort }}
     SERVICE={{ template "redis-ha.fullname" . }}
+    REPLICA_COUNT={{ .Values.replicas }}
     NAMESPACE="{{ .Release.Namespace }}"
     SENTINEL_TLS_REPLICATION_ENABLED={{ default false .Values.sentinel.tlsReplication }}
     REDIS_TLS_REPLICATION_ENABLED={{ default false .Values.redis.tlsReplication }}
@@ -548,11 +560,13 @@
             echo "WARNING: Sentinel returned no master (quorum may be broken). Failure count: $QUORUM_FAIL_COUNT/$MAX_QUORUM_FAILURES"
             if [ "$QUORUM_FAIL_COUNT" -ge "$MAX_QUORUM_FAILURES" ]; then
                 echo "ERROR: Quorum broken for $MAX_QUORUM_FAILURES consecutive checks. Attempting sentinel reset..."
-                if [ "$SENTINEL_PORT" -eq 0 ]; then
-                    redis-cli -h "${SERVICE}" -p "${SENTINEL_TLS_PORT}" {{ if .Values.sentinel.auth }} -a "${SENTINELAUTH}" --no-auth-warning{{ end }} --tls --cacert /tls-certs/{{ .Values.tls.caCertFile }} {{ if ne (default "yes" .Values.sentinel.authClients) "no"}} --cert /tls-certs/{{ .Values.tls.certFile }} --key /tls-certs/{{ .Values.tls.keyFile }}{{ end }} sentinel reset "${MASTER_GROUP}" || true
-                else
-                    redis-cli -h "${SERVICE}" -p "${SENTINEL_PORT}" {{ if .Values.sentinel.auth }} -a "${SENTINELAUTH}" --no-auth-warning{{ end }} sentinel reset "${MASTER_GROUP}" || true
-                fi
+                _i=0
+                while [ "${_i}" -lt "${REPLICA_COUNT}" ]; do
+                    if sentinel_cli "${SERVICE}-announce-${_i}" sentinel reset "${MASTER_GROUP}"; then
+                        break
+                    fi
+                    _i=$((_i + 1))
+                done
                 QUORUM_FAIL_COUNT=0
             fi
         fi

From e2296f56f709b5d0108b72bb79d67215eecdda5e Mon Sep 17 00:00:00 2001
From: Robert Moucha <robert.moucha@gooddata.com>
Date: Thu, 8 Oct 2026 16:38:53 +0200
Subject: [PATCH 2/8] fix(redis-ha): follow only a member that holds the master
 role

When init.sh could not get a usable master from sentinel, a replica
followed announce-0 without checking it. If announce-0 was itself a
replica, the set formed a replication loop with no master, which
sentinel cannot break: it rejects every candidate with NOGOODSLAVE.

- Follow a member only when it reports role:master.
- Never accept this pod's own announce address as the master; the
  address may still reach the predecessor that is shutting down.
- Look for a master again right before forcing a failover, so a
  healthy new master is not moved a second time.
- With no master anywhere, wait for one. Pod 0 still makes itself
  master when no other member answers, or after MASTERLESS_WAIT_ROUNDS
  rounds of 10s (default 12).

Signed-off-by: Robert Moucha <robert.moucha@gooddata.com>
Co-authored-by: Claude <noreply@anthropic.com>
---
 charts/redis-ha/templates/_configs.tpl | 118 ++++++++++++++++++-------
 1 file changed, 87 insertions(+), 31 deletions(-)

diff --git a/charts/redis-ha/templates/_configs.tpl b/charts/redis-ha/templates/_configs.tpl
index a34b25a..6ef8573 100644
--- a/charts/redis-ha/templates/_configs.tpl
+++ b/charts/redis-ha/templates/_configs.tpl
@@ -208,37 +208,80 @@
         cp /readonly-config/sentinel.conf "${SENTINEL_CONF}"
     }
 
+    redis_cli_at() {
+        _host="${1}"
+        shift
+        if [ "$REDIS_PORT" -eq 0 ]; then
+            redis-cli -h "${_host}"{{ if .Values.auth }} -a "${AUTH}" --no-auth-warning{{ end }} -p "${REDIS_TLS_PORT}" --tls --cacert /tls-certs/{{ .Values.tls.caCertFile }} {{ if ne (default "yes" .Values.sentinel.authClients) "no"}} --cert /tls-certs/{{ .Values.tls.certFile }} --key /tls-certs/{{ .Values.tls.keyFile }}{{ end }} "$@"
+        else
+            redis-cli -h "${_host}"{{ if .Values.auth }} -a "${AUTH}" --no-auth-warning{{ end }} -p "${REDIS_PORT}" "$@"
+        fi
+    }
+
+    # Prints the role ("master" or "slave") the redis at $1 reports, or
+    # nothing when it does not answer.
+    redis_role_at() {
+    set +e
+        redis_cli_at "${1}" role 2>/dev/null | head -n 1
+    set -e
+    }
+
+    # Sets DEFAULT_MASTER to another member that really holds the master
+    # role: the one sentinel names if it does, otherwise the first member
+    # found. Sets REACHABLE_MEMBERS to the number of other members that
+    # answered.
+    find_role_master() {
+        DEFAULT_MASTER=''
+        REACHABLE_MEMBERS=0
+        _named="$(sentinel_get_master)"
+        if [ -n "${_named}" ] && [ "${_named}" != "${ANNOUNCE_IP}" ] && [ "$(redis_role_at "${_named}")" = "master" ]; then
+            DEFAULT_MASTER="${_named}"
+        fi
+        _i=0
+        while [ "${_i}" -lt "${REPLICA_COUNT}" ]; do
+            if [ "${_i}" != "${INDEX}" ]; then
+                _role="$(redis_role_at "${SERVICE}-announce-${_i}")"
+                if [ -n "${_role}" ]; then
+                    REACHABLE_MEMBERS=$((REACHABLE_MEMBERS + 1))
+                fi
+                if [ "${_role}" = "master" ] && [ -z "${DEFAULT_MASTER}" ]; then
+                    DEFAULT_MASTER="$(getent_hosts "${_i}" | awk '{ print $1 }')"
+                fi
+            fi
+            _i=$((_i + 1))
+        done
+    }
+
+    # Never follow a member that is not master: a replica of a replica
+    # forms a loop that sentinel cannot break. Wait for a master instead.
+    # Pod 0 makes itself master when no other member answers, as on a
+    # fresh install, or after MASTERLESS_WAIT_ROUNDS rounds of waiting.
     setup_defaults() {
         echo "Setting up defaults.."
         echo "  using statefulset index (${INDEX})"
-        if [ "${INDEX}" = "0" ]; then
-            echo "Setting this pod as master for redis and sentinel.."
-            echo "  using announce (${ANNOUNCE_IP})"
-            redis_update "${ANNOUNCE_IP}"
-            sentinel_update "${ANNOUNCE_IP}"
-            echo "  make sure ${ANNOUNCE_IP} is not a slave (slaveof no one)"
-            sed -i "s/^.*slaveof.*//" "${REDIS_CONF}"
-        else
-            if [ "$RESOLVE_HOSTNAMES" = true ]; then
-                echo "Getting redis master hostname.."
-                echo "  blindly assuming (${SERVICE}-announce-0.${NAMESPACE}.svc) is master"
-                DEFAULT_MASTER="${SERVICE}-announce-0.${NAMESPACE}.svc"
-                echo "  identified redis (may be redis master) hostname (${DEFAULT_MASTER})"
-            else
-                echo "Getting redis master ip.."
-                echo "  blindly assuming (${SERVICE}-announce-0) or (${SERVICE}-server-0) are master"
-                DEFAULT_MASTER="$(getent_hosts 0 | awk '{ print $1 }')"
-                if [ -z "${DEFAULT_MASTER}" ]; then
-                    echo "Error: Unable to resolve redis master (getent hosts)."
-                    exit 1
-                fi
-                echo "  identified redis (may be redis master) ip (${DEFAULT_MASTER})"
+        _round=0
+        while true; do
+            find_role_master
+            if [ -n "${DEFAULT_MASTER}" ]; then
+                echo "  $(date) Found member holding the master role (${DEFAULT_MASTER})"
+                echo "Setting default slave config for redis and sentinel.."
+                redis_update "${DEFAULT_MASTER}"
+                sentinel_update "${DEFAULT_MASTER}"
+                return 0
             fi
-            echo "Setting default slave config for redis and sentinel.."
-            echo "  using master address (${DEFAULT_MASTER})"
-            redis_update "${DEFAULT_MASTER}"
-            sentinel_update "${DEFAULT_MASTER}"
-        fi
+            if [ "${INDEX}" = "0" ] && { [ "${REACHABLE_MEMBERS}" -eq 0 ] || [ "${_round}" -ge "${MASTERLESS_WAIT_ROUNDS}" ]; }; then
+                echo "Setting this pod as master for redis and sentinel.."
+                echo "  using announce (${ANNOUNCE_IP})"
+                redis_update "${ANNOUNCE_IP}"
+                sentinel_update "${ANNOUNCE_IP}"
+                echo "  make sure ${ANNOUNCE_IP} is not a slave (slaveof no one)"
+                sed -i "s/^.*slaveof.*//" "${REDIS_CONF}"
+                return 0
+            fi
+            _round=$((_round + 1))
+            echo "  $(date) No member holds the master role (${REACHABLE_MEMBERS} other members answered). Waiting, round ${_round}.."
+            sleep 10
+        done
     }
 
     redis_ping() {
@@ -256,7 +299,9 @@
         retry=${1}
         sleep=3
         for i in $(seq 1 "${retry}"); do
-            if [ "$(redis_ping)" = "PONG" ]; then
+            # Our own announce address is not a master we can follow: this
+            # pod is starting, and the address may still reach its predecessor.
+            if [ "${MASTER}" != "${ANNOUNCE_IP}" ] && [ "$(redis_ping)" = "PONG" ] && [ "$(redis_role_at "${MASTER}")" = "master" ]; then
                ping='PONG'
                break
             fi
@@ -275,6 +320,16 @@
         fi
         if [ "$(redis_ping_retry 3)" != "PONG" ]; then
             echo "  $(date) Can't ping redis master (${MASTER})"
+            # Sentinel may have failed over meanwhile. Forcing another
+            # failover would move a healthy new master again.
+            find_role_master
+            if [ -n "${DEFAULT_MASTER}" ]; then
+                echo "  $(date) Found member holding the master role (${DEFAULT_MASTER})"
+                echo "Updating redis and sentinel config.."
+                sentinel_update "${DEFAULT_MASTER}"
+                redis_update "${DEFAULT_MASTER}"
+                return 0
+            fi
             echo "Attempting to force failover (sentinel failover).."
             echo "  on sentinels (${SERVICE}-announce-*), sentinel grp (${MASTER_GROUP})"
             if sentinel_failover | grep -q 'NOGOODSLAVE' ; then
@@ -289,14 +344,14 @@
             echo "We should get redis master's ip now. Asking (get-master-addr-by-name).."
             echo "  sentinels (${SERVICE}-announce-*), sentinel grp (${MASTER_GROUP})"
             MASTER="$(sentinel_get_master)"
-            if [ "${MASTER}" ]; then
+            if [ "${MASTER}" ] && [ "${MASTER}" != "${ANNOUNCE_IP}" ]; then
                 echo "  $(date) Found redis master (${MASTER})"
                 echo "Updating redis and sentinel config.."
                 sentinel_update "${MASTER}"
                 redis_update "${MASTER}"
             else
-                echo "$(date) Error: Could not failover, exiting..."
-                exit 1
+                echo "  $(date) Sentinel still names no other master"
+                setup_defaults
             fi
         else
             echo "  $(date) Found reachable redis master (${MASTER})"
@@ -355,6 +410,7 @@
     SENTINEL_TLS_PORT={{ .Values.sentinel.tlsPort }}
     SERVICE={{ template "redis-ha.fullname" . }}
     REPLICA_COUNT={{ .Values.replicas }}
+    MASTERLESS_WAIT_ROUNDS=${MASTERLESS_WAIT_ROUNDS:-12}
     NAMESPACE="{{ .Release.Namespace }}"
     SENTINEL_TLS_REPLICATION_ENABLED={{ default false .Values.sentinel.tlsReplication }}
     REDIS_TLS_REPLICATION_ENABLED={{ default false .Values.redis.tlsReplication }}

From 10642072576cc82aeaf8fe55ebfd824252d619f8 Mon Sep 17 00:00:00 2001
From: ShmuelOps <shmuel@tennasys.com>
Date: Thu, 8 Oct 2026 16:52:47 +0200
Subject: [PATCH 3/8] fix(redis-ha): self-heal a masterless set instead of
 waiting for sentinel

When the master pod is replaced behind the same announce address,
sentinel can keep naming that address while it runs as a replica. Every
member agrees with sentinel, so split-brain-fix never acts, and sentinel
rejects every candidate with NOGOODSLAVE.

Poll the announce services for the member that really holds the master
role. Repoint the sentinels at it when one exists. When none does over
MASTERLESS_CONFIRMATIONS checks, promote the reachable member with the
highest replication offset.

Squashed from https://github.com/DandyDeveloper/charts/pull/413
(commits 0a3bcb0 and ac939b0), without its chart version bump.

Signed-off-by: ShmuelOps <shmuel@tennasys.com>
Signed-off-by: Robert Moucha <robert.moucha@gooddata.com>
Co-authored-by: Claude <noreply@anthropic.com>
---
 charts/redis-ha/templates/_configs.tpl | 235 ++++++++++++++++++++++++-
 1 file changed, 233 insertions(+), 2 deletions(-)

diff --git a/charts/redis-ha/templates/_configs.tpl b/charts/redis-ha/templates/_configs.tpl
index 6ef8573..3ee6fdf 100644
--- a/charts/redis-ha/templates/_configs.tpl
+++ b/charts/redis-ha/templates/_configs.tpl
@@ -561,6 +561,199 @@
     set -e
     }
 
+    master_actual_role() {
+    set +e
+        if [ "$REDIS_PORT" -eq 0 ]; then
+            MASTER_ACTUAL_ROLE=$(redis-cli {{ if .Values.auth }} -a "${AUTH}" --no-auth-warning{{ end }} -h "${MASTER}" -p "${REDIS_TLS_PORT}" --tls --cacert /tls-certs/{{ .Values.tls.caCertFile }} {{ if ne (default "yes" .Values.sentinel.authClients) "no"}} --cert /tls-certs/{{ .Values.tls.certFile }} --key /tls-certs/{{ .Values.tls.keyFile }}{{ end }} info | grep role | sed 's/role://' | sed 's/\r//')
+        else
+            MASTER_ACTUAL_ROLE=$(redis-cli {{ if .Values.auth }} -a "${AUTH}" --no-auth-warning{{ end }} -h "${MASTER}" -p "${REDIS_PORT}" info | grep role | sed 's/role://' | sed 's/\r//')
+        fi
+    set -e
+    }
+
+    # Announce services publish not-ready addresses; the headless ${SERVICE}
+    # does not, so it stops resolving once every pod is unready.
+    REPLICA_COUNT={{ .Values.replicas }}
+    REDIS_ACTIVE_PORT="${REDIS_PORT}"
+    [ "$REDIS_PORT" -eq 0 ] && REDIS_ACTIVE_PORT="${REDIS_TLS_PORT}"
+
+    announce_info() {
+    set +e
+        if [ "$REDIS_PORT" -eq 0 ]; then
+            redis-cli {{ if .Values.auth }} -a "${AUTH}" --no-auth-warning{{ end }} -h "$1" -p "${REDIS_TLS_PORT}" --tls --cacert /tls-certs/{{ .Values.tls.caCertFile }} {{ if ne (default "yes" .Values.sentinel.authClients) "no"}} --cert /tls-certs/{{ .Values.tls.certFile }} --key /tls-certs/{{ .Values.tls.keyFile }}{{ end }} info replication 2>/dev/null | sed 's/\r//'
+        else
+            redis-cli {{ if .Values.auth }} -a "${AUTH}" --no-auth-warning{{ end }} -h "$1" -p "${REDIS_PORT}" info replication 2>/dev/null | sed 's/\r//'
+        fi
+    set -e
+    }
+
+    announce_cmd() {
+    set +e
+        host="$1"; shift
+        if [ "$REDIS_PORT" -eq 0 ]; then
+            redis-cli {{ if .Values.auth }} -a "${AUTH}" --no-auth-warning{{ end }} -h "$host" -p "${REDIS_TLS_PORT}" --tls --cacert /tls-certs/{{ .Values.tls.caCertFile }} {{ if ne (default "yes" .Values.sentinel.authClients) "no"}} --cert /tls-certs/{{ .Values.tls.certFile }} --key /tls-certs/{{ .Values.tls.keyFile }}{{ end }} "$@"
+        else
+            redis-cli {{ if .Values.auth }} -a "${AUTH}" --no-auth-warning{{ end }} -h "$host" -p "${REDIS_PORT}" "$@"
+        fi
+    set -e
+    }
+
+    announce_sentinel_cmd() {
+    set +e
+        host="$1"; shift
+        if [ "$SENTINEL_PORT" -eq 0 ]; then
+            redis-cli {{ if .Values.sentinel.auth }} -a "${SENTINELAUTH}" --no-auth-warning{{ end }} -h "$host" -p "${SENTINEL_TLS_PORT}" --tls --cacert /tls-certs/{{ .Values.tls.caCertFile }} {{ if ne (default "yes" .Values.sentinel.authClients) "no"}} --cert /tls-certs/{{ .Values.tls.certFile }} --key /tls-certs/{{ .Values.tls.keyFile }}{{ end }} "$@"
+        else
+            redis-cli {{ if .Values.sentinel.auth }} -a "${SENTINELAUTH}" --no-auth-warning{{ end }} -h "$host" -p "${SENTINEL_PORT}" "$@"
+        fi
+    set -e
+    }
+
+    # True if no member holds the master role.
+    masterless_now() {
+        _j=0
+        while [ "$_j" -lt "$REPLICA_COUNT" ]; do
+            _r=$(announce_info "${SERVICE}-announce-${_j}" | awk -F: '/^role:/{print $2}')
+            [ "$_r" = "master" ] && return 1
+            _j=$((_j + 1))
+        done
+        return 0
+    }
+
+    # A masterless set never recovers on its own, so confirm it over seconds
+    # rather than over detection cycles. Any master seen aborts.
+    confirm_masterless_and_heal() {
+        _c=0
+        while [ "$_c" -lt "$MASTERLESS_CONFIRMATIONS" ]; do
+            if ! masterless_now; then
+                echo "A member holds the master role; standing down after $_c/$MASTERLESS_CONFIRMATIONS confirmations."
+                return 1
+            fi
+            _c=$((_c + 1))
+            [ "$_c" -lt "$MASTERLESS_CONFIRMATIONS" ] && sleep "$MASTERLESS_CONFIRM_INTERVAL"
+        done
+        echo "No member held the master role across $MASTERLESS_CONFIRMATIONS consecutive checks."
+        promote_best_member
+    }
+
+    # Promote the highest-offset member. Sentinel cannot: once every candidate
+    # looks stale it aborts with -failover-abort-no-good-slave / -NOGOODSLAVE.
+    promote_best_member() {
+        # Rescan: a liveness-killed redis is briefly unreachable.
+        _try=0
+        while [ "$_try" -lt "$PROMOTE_SCAN_ATTEMPTS" ]; do
+            _best=''
+            _best_off=-1
+            _i=0
+            while [ "$_i" -lt "$REPLICA_COUNT" ]; do
+                _info=$(announce_info "${SERVICE}-announce-${_i}")
+                _role=$(echo "$_info" | awk -F: '/^role:/{print $2}')
+                if [ "$_role" = "master" ]; then
+                    echo "A master is present at ${SERVICE}-announce-${_i}; no promotion needed."
+                    return 1
+                fi
+                _off=$(echo "$_info" | awk -F: '/_repl_offset:/{print $2; exit}')
+                case "$_off" in
+                    ''|*[!0-9]*) ;;
+                    *) if [ "$_off" -gt "$_best_off" ]; then _best_off="$_off"; _best="$_i"; fi ;;
+                esac
+                _i=$((_i + 1))
+            done
+            [ -n "$_best" ] && break
+            _try=$((_try + 1))
+            echo "Masterless, but no member answered (attempt $_try/$PROMOTE_SCAN_ATTEMPTS) — members may be restarting."
+            [ "$_try" -lt "$PROMOTE_SCAN_ATTEMPTS" ] && sleep "$MASTERLESS_CONFIRM_INTERVAL"
+        done
+        if [ -z "$_best" ]; then
+            echo "Masterless, but no member is reachable — not promoting."
+            return 1
+        fi
+        _best_host="${SERVICE}-announce-${_best}"
+        _best_ip=$(getent hosts "$_best_host" | awk '{ print $1 }')
+        echo "ERROR: no member holds the master role. Promoting ${_best_host} (replication offset ${_best_off})."
+        announce_cmd "$_best_host" replicaof no one
+        _i=0
+        while [ "$_i" -lt "$REPLICA_COUNT" ]; do
+            if [ "$_i" -ne "$_best" ] && [ -n "$_best_ip" ]; then
+                announce_cmd "${SERVICE}-announce-${_i}" replicaof "$_best_ip" "$REDIS_ACTIVE_PORT"
+            fi
+            _i=$((_i + 1))
+        done
+        if [ -n "$_best_ip" ]; then
+            repoint_sentinels "$_best_ip"
+        else
+            echo "WARNING: could not resolve ${_best_host}; leaving sentinel config untouched."
+        fi
+        echo "Promotion complete: ${_best_host} is now master."
+        return 0
+    }
+
+    # Address and index of whichever member currently holds the master role
+    # ('' if none).
+    find_actual_master() {
+        ACTUAL_MASTER_INDEX=''
+        ACTUAL_MASTER_ADDR=''
+        _m=0
+        while [ "$_m" -lt "$REPLICA_COUNT" ]; do
+            _r=$(announce_info "${SERVICE}-announce-${_m}" | awk -F: '/^role:/{print $2}')
+            if [ "$_r" = "master" ]; then
+                ACTUAL_MASTER_INDEX="$_m"
+                ACTUAL_MASTER_ADDR=$(getent_hosts "$_m" | awk '{ print $1 }')
+                return 0
+            fi
+            _m=$((_m + 1))
+        done
+        return 0
+    }
+
+    # Rewrite every sentinel's monitor entry to point at $1. 'sentinel reset'
+    # is not enough: it rediscovers from the stale configured address, so a
+    # wrong entry survives it.
+    repoint_sentinels() {
+        if [ "$SENTINEL_TLS_REPLICATION_ENABLED" = true ]; then
+            _monitor_port="${REDIS_TLS_PORT}"
+        else
+            _monitor_port="${REDIS_PORT}"
+        fi
+        _k=0
+        while [ "$_k" -lt "$REPLICA_COUNT" ]; do
+            announce_sentinel_cmd "${SERVICE}-announce-${_k}" sentinel remove "${MASTER_GROUP}"
+            announce_sentinel_cmd "${SERVICE}-announce-${_k}" sentinel monitor "${MASTER_GROUP}" "$1" "$_monitor_port" "$QUORUM"
+            _k=$((_k + 1))
+        done
+        echo "Repointed sentinels: monitor ${MASTER_GROUP} ${1}:${_monitor_port} quorum ${QUORUM}"
+    }
+
+    # Sentinel's named master does not hold the master role. Trust the data
+    # plane instead of forcing a blind failover (which would demote a healthy
+    # master if sentinel's answer was merely stale): if some member really is
+    # master, repoint every sentinel at it; only if none is, force a failover
+    # and promote directly if sentinel cannot (-NOGOODSLAVE).
+    heal_stale_sentinel() {
+        find_actual_master
+        if [ -n "$ACTUAL_MASTER_INDEX" ]; then
+            if [ "$ACTUAL_MASTER_INDEX" = "$INDEX" ]; then
+                echo "This pod holds the master role after all; nothing to heal."
+            elif [ -n "$ACTUAL_MASTER_ADDR" ]; then
+                echo "ERROR: sentinel names ${MASTER} as master but ${SERVICE}-announce-${ACTUAL_MASTER_INDEX} (${ACTUAL_MASTER_ADDR}) holds the role. Repointing sentinels instead of failing over..."
+                repoint_sentinels "$ACTUAL_MASTER_ADDR"
+            else
+                echo "WARNING: ${SERVICE}-announce-${ACTUAL_MASTER_INDEX} holds the master role but its address did not resolve; leaving sentinel config untouched."
+            fi
+            return 0
+        fi
+        echo "ERROR: no member holds the master role. Forcing a failover..."
+        if [ "$SENTINEL_PORT" -eq 0 ]; then
+            redis-cli -h "${SERVICE}" -p "${SENTINEL_TLS_PORT}" {{ if .Values.sentinel.auth }} -a "${SENTINELAUTH}" --no-auth-warning{{ end }} --tls --cacert /tls-certs/{{ .Values.tls.caCertFile }} {{ if ne (default "yes" .Values.sentinel.authClients) "no"}} --cert /tls-certs/{{ .Values.tls.certFile }} --key /tls-certs/{{ .Values.tls.keyFile }}{{ end }} sentinel failover "${MASTER_GROUP}" || true
+        else
+            redis-cli -h "${SERVICE}" -p "${SENTINEL_PORT}" {{ if .Values.sentinel.auth }} -a "${SENTINELAUTH}" --no-auth-warning{{ end }} sentinel failover "${MASTER_GROUP}" || true
+        fi
+        # SENTINEL FAILOVER returns -NOGOODSLAVE when no candidate looks
+        # fresh; promote directly instead.
+        sleep {{ .Values.splitBrainDetection.retryInterval }}
+        confirm_masterless_and_heal || true
+    }
+
     identify_announce_ip
 
     while [ -z "${ANNOUNCE_IP}" ]; do
@@ -571,6 +764,12 @@
 
     QUORUM_FAIL_COUNT=0
     MAX_QUORUM_FAILURES=${MAX_QUORUM_FAILURES:-5}
+    STALE_MASTER_COUNT=0
+    MAX_STALE_MASTER_FAILURES=${MAX_STALE_MASTER_FAILURES:-3}
+    # Confirmations required before promoting, and the gap between them.
+    MASTERLESS_CONFIRMATIONS=${MASTERLESS_CONFIRMATIONS:-5}
+    MASTERLESS_CONFIRM_INTERVAL=${MASTERLESS_CONFIRM_INTERVAL:-5}
+    PROMOTE_SCAN_ATTEMPTS=${PROMOTE_SCAN_ATTEMPTS:-6}
 
     trap "exit 0" TERM
     while true; do
@@ -589,8 +788,21 @@
                 redis_role
                 echo "Redis role is $ROLE, expected role is master. No need to reinitialize."
                 if [ "$ROLE" != "master" ]; then
-                    echo "Redis role is $ROLE, expected role is master, reinitializing"
-                    reinit
+                    # Sentinel names this pod as master but it runs as a replica:
+                    # reinit would re-derive `slaveof` and shutdown, restarting the
+                    # pod straight back into a replica while sentinel keeps naming
+                    # it (#383). This pod is also the fastest observer of the
+                    # broken state, so heal here instead of waiting for peers'
+                    # stale-master counters. Promotion stays offset-aware — a
+                    # freshly replaced (empty) pod behind a stale sentinel record
+                    # must not be promoted just because sentinel names it.
+                    echo "Redis role is $ROLE but sentinel names this pod as master; healing from data-plane truth"
+                    heal_stale_sentinel
+                    if [ -n "$ACTUAL_MASTER_INDEX" ] && [ "$ACTUAL_MASTER_INDEX" != "$INDEX" ]; then
+                        # Sentinels now point at the real master; reinit so
+                        # init.sh reconfigures this pod as its replica.
+                        reinit
+                    fi
                 fi
             fi
         elif [ "${MASTER}" ]; then
@@ -610,10 +822,29 @@
                         reinit
                     fi
                 fi
+            else
+                # Agreeing with sentinel is not enough: the named master may
+                # itself be a replica, leaving the set with no master at all.
+                master_actual_role
+                if [ -z "${MASTER_ACTUAL_ROLE}" ] || [ "${MASTER_ACTUAL_ROLE}" = "master" ]; then
+                    # healthy, or unreachable (sentinel's own down detection handles that)
+                    STALE_MASTER_COUNT=0
+                else
+                    STALE_MASTER_COUNT=$((STALE_MASTER_COUNT + 1))
+                    echo "WARNING: Sentinel names ${MASTER} as master but its actual role is '${MASTER_ACTUAL_ROLE}' (set may be masterless). Failure count: $STALE_MASTER_COUNT/$MAX_STALE_MASTER_FAILURES"
+                    if [ "$STALE_MASTER_COUNT" -ge "$MAX_STALE_MASTER_FAILURES" ]; then
+                        echo "ERROR: Sentinel-named master has not held the master role for $MAX_STALE_MASTER_FAILURES consecutive checks. Healing..."
+                        heal_stale_sentinel
+                        STALE_MASTER_COUNT=0
+                    fi
+                fi
             fi
         else
             QUORUM_FAIL_COUNT=$((QUORUM_FAIL_COUNT + 1))
             echo "WARNING: Sentinel returned no master (quorum may be broken). Failure count: $QUORUM_FAIL_COUNT/$MAX_QUORUM_FAILURES"
+            # No master named is also what a masterless set looks like; the
+            # reset below still runs on its own schedule for real quorum loss.
+            confirm_masterless_and_heal || true
             if [ "$QUORUM_FAIL_COUNT" -ge "$MAX_QUORUM_FAILURES" ]; then
                 echo "ERROR: Quorum broken for $MAX_QUORUM_FAILURES consecutive checks. Attempting sentinel reset..."
                 _i=0

From 4e1739a319dc2ee86e92d4845c1ef627db48170d Mon Sep 17 00:00:00 2001
From: Robert Moucha <robert.moucha@gooddata.com>
Date: Thu, 8 Oct 2026 16:54:21 +0200
Subject: [PATCH 4/8] fix(redis-ha): heal a masterless set whose named master
 is down

Harden the masterless healer:

- Also count a named master that does not answer, once a quorum of
  sentinels flags it as down. A replaced master that cannot start, while
  sentinel finds no good replica, otherwise leaves the set without a
  master and the healer never acts.
- After repointing a sentinel, restore what sentinel.conf sets for the
  master: auth-pass and the sentinel.config values. A bare monitor entry
  falls back to sentinel defaults and cannot authenticate to the master.
- Resolve the promoted member through getent_hosts, so announceHostnames
  works.
- Force failovers through the announce services, and reuse the shared
  redis and sentinel client helpers.

Signed-off-by: Robert Moucha <robert.moucha@gooddata.com>
Co-authored-by: Claude <noreply@anthropic.com>
---
 charts/redis-ha/templates/_configs.tpl | 67 +++++++++++++++-----------
 1 file changed, 38 insertions(+), 29 deletions(-)

diff --git a/charts/redis-ha/templates/_configs.tpl b/charts/redis-ha/templates/_configs.tpl
index 3ee6fdf..35d71b0 100644
--- a/charts/redis-ha/templates/_configs.tpl
+++ b/charts/redis-ha/templates/_configs.tpl
@@ -571,44 +571,41 @@
     set -e
     }
 
-    # Announce services publish not-ready addresses; the headless ${SERVICE}
-    # does not, so it stops resolving once every pod is unready.
-    REPLICA_COUNT={{ .Values.replicas }}
     REDIS_ACTIVE_PORT="${REDIS_PORT}"
     [ "$REDIS_PORT" -eq 0 ] && REDIS_ACTIVE_PORT="${REDIS_TLS_PORT}"
 
     announce_info() {
     set +e
-        if [ "$REDIS_PORT" -eq 0 ]; then
-            redis-cli {{ if .Values.auth }} -a "${AUTH}" --no-auth-warning{{ end }} -h "$1" -p "${REDIS_TLS_PORT}" --tls --cacert /tls-certs/{{ .Values.tls.caCertFile }} {{ if ne (default "yes" .Values.sentinel.authClients) "no"}} --cert /tls-certs/{{ .Values.tls.certFile }} --key /tls-certs/{{ .Values.tls.keyFile }}{{ end }} info replication 2>/dev/null | sed 's/\r//'
-        else
-            redis-cli {{ if .Values.auth }} -a "${AUTH}" --no-auth-warning{{ end }} -h "$1" -p "${REDIS_PORT}" info replication 2>/dev/null | sed 's/\r//'
-        fi
+        redis_cli_at "$1" info replication 2>/dev/null | sed 's/\r//'
     set -e
     }
 
     announce_cmd() {
     set +e
-        host="$1"; shift
-        if [ "$REDIS_PORT" -eq 0 ]; then
-            redis-cli {{ if .Values.auth }} -a "${AUTH}" --no-auth-warning{{ end }} -h "$host" -p "${REDIS_TLS_PORT}" --tls --cacert /tls-certs/{{ .Values.tls.caCertFile }} {{ if ne (default "yes" .Values.sentinel.authClients) "no"}} --cert /tls-certs/{{ .Values.tls.certFile }} --key /tls-certs/{{ .Values.tls.keyFile }}{{ end }} "$@"
-        else
-            redis-cli {{ if .Values.auth }} -a "${AUTH}" --no-auth-warning{{ end }} -h "$host" -p "${REDIS_PORT}" "$@"
-        fi
+        redis_cli_at "$@"
     set -e
     }
 
     announce_sentinel_cmd() {
     set +e
-        host="$1"; shift
-        if [ "$SENTINEL_PORT" -eq 0 ]; then
-            redis-cli {{ if .Values.sentinel.auth }} -a "${SENTINELAUTH}" --no-auth-warning{{ end }} -h "$host" -p "${SENTINEL_TLS_PORT}" --tls --cacert /tls-certs/{{ .Values.tls.caCertFile }} {{ if ne (default "yes" .Values.sentinel.authClients) "no"}} --cert /tls-certs/{{ .Values.tls.certFile }} --key /tls-certs/{{ .Values.tls.keyFile }}{{ end }} "$@"
-        else
-            redis-cli {{ if .Values.sentinel.auth }} -a "${SENTINELAUTH}" --no-auth-warning{{ end }} -h "$host" -p "${SENTINEL_PORT}" "$@"
-        fi
+        sentinel_cli "$@"
     set -e
     }
 
+    # True when at least a quorum of sentinels flag the named master as down,
+    # so a master this pod cannot reach is not just cut off from this pod.
+    sentinels_see_master_down() {
+        _down=0
+        _k=0
+        while [ "$_k" -lt "$REPLICA_COUNT" ]; do
+            if announce_sentinel_cmd "${SERVICE}-announce-${_k}" sentinel master "${MASTER_GROUP}" 2>/dev/null | grep -q 's_down'; then
+                _down=$((_down + 1))
+            fi
+            _k=$((_k + 1))
+        done
+        [ "$_down" -ge "$QUORUM" ]
+    }
+
     # True if no member holds the master role.
     masterless_now() {
         _j=0
@@ -669,7 +666,7 @@
             return 1
         fi
         _best_host="${SERVICE}-announce-${_best}"
-        _best_ip=$(getent hosts "$_best_host" | awk '{ print $1 }')
+        _best_ip=$(getent_hosts "$_best" | awk '{ print $1 }')
         echo "ERROR: no member holds the master role. Promoting ${_best_host} (replication offset ${_best_off})."
         announce_cmd "$_best_host" replicaof no one
         _i=0
@@ -719,6 +716,18 @@
         while [ "$_k" -lt "$REPLICA_COUNT" ]; do
             announce_sentinel_cmd "${SERVICE}-announce-${_k}" sentinel remove "${MASTER_GROUP}"
             announce_sentinel_cmd "${SERVICE}-announce-${_k}" sentinel monitor "${MASTER_GROUP}" "$1" "$_monitor_port" "$QUORUM"
+            # A new monitor entry starts from sentinel's defaults. Put back what
+            # sentinel.conf sets, or the sentinel cannot authenticate to the master.
+            {{- if not .Values.sentinel.customConfig }}
+            {{- range $key, $value := .Values.sentinel.config }}
+            {{- if ne "maxclients" $key }}
+            announce_sentinel_cmd "${SERVICE}-announce-${_k}" sentinel set "${MASTER_GROUP}" {{ $key }} {{ $value }}
+            {{- end }}
+            {{- end }}
+            {{- end }}
+            {{- if .Values.auth }}
+            announce_sentinel_cmd "${SERVICE}-announce-${_k}" sentinel set "${MASTER_GROUP}" auth-pass "${AUTH}"
+            {{- end }}
             _k=$((_k + 1))
         done
         echo "Repointed sentinels: monitor ${MASTER_GROUP} ${1}:${_monitor_port} quorum ${QUORUM}"
@@ -743,11 +752,7 @@
             return 0
         fi
         echo "ERROR: no member holds the master role. Forcing a failover..."
-        if [ "$SENTINEL_PORT" -eq 0 ]; then
-            redis-cli -h "${SERVICE}" -p "${SENTINEL_TLS_PORT}" {{ if .Values.sentinel.auth }} -a "${SENTINELAUTH}" --no-auth-warning{{ end }} --tls --cacert /tls-certs/{{ .Values.tls.caCertFile }} {{ if ne (default "yes" .Values.sentinel.authClients) "no"}} --cert /tls-certs/{{ .Values.tls.certFile }} --key /tls-certs/{{ .Values.tls.keyFile }}{{ end }} sentinel failover "${MASTER_GROUP}" || true
-        else
-            redis-cli -h "${SERVICE}" -p "${SENTINEL_PORT}" {{ if .Values.sentinel.auth }} -a "${SENTINELAUTH}" --no-auth-warning{{ end }} sentinel failover "${MASTER_GROUP}" || true
-        fi
+        sentinel_failover || true
         # SENTINEL FAILOVER returns -NOGOODSLAVE when no candidate looks
         # fresh; promote directly instead.
         sleep {{ .Values.splitBrainDetection.retryInterval }}
@@ -826,12 +831,16 @@
                 # Agreeing with sentinel is not enough: the named master may
                 # itself be a replica, leaving the set with no master at all.
                 master_actual_role
-                if [ -z "${MASTER_ACTUAL_ROLE}" ] || [ "${MASTER_ACTUAL_ROLE}" = "master" ]; then
-                    # healthy, or unreachable (sentinel's own down detection handles that)
+                if [ "${MASTER_ACTUAL_ROLE}" = "master" ]; then
+                    STALE_MASTER_COUNT=0
+                elif [ -z "${MASTER_ACTUAL_ROLE}" ] && ! sentinels_see_master_down; then
+                    # Unreachable from this pod only: sentinel's own down detection decides.
                     STALE_MASTER_COUNT=0
                 else
+                    # A named master that stays down for several checks gets no
+                    # failover when sentinel finds no good replica either.
                     STALE_MASTER_COUNT=$((STALE_MASTER_COUNT + 1))
-                    echo "WARNING: Sentinel names ${MASTER} as master but its actual role is '${MASTER_ACTUAL_ROLE}' (set may be masterless). Failure count: $STALE_MASTER_COUNT/$MAX_STALE_MASTER_FAILURES"
+                    echo "WARNING: Sentinel names ${MASTER} as master but its actual role is '${MASTER_ACTUAL_ROLE:-unreachable}' (set may be masterless). Failure count: $STALE_MASTER_COUNT/$MAX_STALE_MASTER_FAILURES"
                     if [ "$STALE_MASTER_COUNT" -ge "$MAX_STALE_MASTER_FAILURES" ]; then
                         echo "ERROR: Sentinel-named master has not held the master role for $MAX_STALE_MASTER_FAILURES consecutive checks. Healing..."
                         heal_stale_sentinel

From 627071f1dd14d9823be6341570879100188d8eb9 Mon Sep 17 00:00:00 2001
From: Robert Moucha <robert.moucha@gooddata.com>
Date: Thu, 8 Oct 2026 17:25:55 +0200
Subject: [PATCH 5/8] fix(redis-ha): hand over the master without losing writes

Kubernetes stops all containers of a pod at once. The sentinel container
had no preStop hook and exited on SIGTERM, so the redis container's hook
could not ask it to fail over: it always failed with "Connection
refused" and the master was replaced only after down-after-milliseconds.
Add a default sentinel preStop hook that keeps sentinel running while
the local redis is master. With sentinel.auth, the redis container also
lacked SENTINELAUTH, so the hook could not authenticate; pass it in.

During a forced failover, the old master kept accepting writes until
sentinel demoted it on its next check, about 10s after the promotion.
Those writes were lost. The redis preStop hook now pauses writes
(CLIENT PAUSE WRITE) before it asks for the failover, follows the
promoted node as soon as sentinel names it, and then unpauses. A held
write fails with READONLY, and the client retries it on the new master.
When sentinel aborts the failover, the hook unpauses at once.

The hook is now POSIX sh, like the other scripts.

Signed-off-by: Robert Moucha <robert.moucha@gooddata.com>
Co-authored-by: Claude <noreply@anthropic.com>
---
 charts/redis-ha/README.md                     |   2 +-
 charts/redis-ha/templates/_configs.tpl        | 121 +++++++++++++-----
 .../templates/redis-ha-health-configmap.yaml  |   2 +
 .../templates/redis-ha-statefulset.yaml       |  16 ++-
 charts/redis-ha/values.yaml                   |   9 +-
 5 files changed, 117 insertions(+), 33 deletions(-)

diff --git a/charts/redis-ha/README.md b/charts/redis-ha/README.md
index 56eaf46..165005a 100644
--- a/charts/redis-ha/README.md
+++ b/charts/redis-ha/README.md
@@ -208,7 +208,7 @@ The following table lists the configurable parameters of the Redis chart and the
 | `sentinel.customConfig` | Allows for custom sentinel.conf files to be applied. If this is used then `sentinel.config` is ignored | string | `""` |
 | `sentinel.existingSecret` | An existing secret containing a key defined by `sentinel.authKey` that configures `requirepass` in the conf parameters (Requires `sentinel.auth: enabled`, cannot be used in conjunction with `.Values.sentinel.password`) | string | `""` |
 | `sentinel.extraVolumeMounts` | additional volumeMounts for Sentinel container | list | `[]` |
-| `sentinel.lifecycle` | Container Lifecycle Hooks for sentinel container. Ref: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/ | object | `{}` |
+| `sentinel.lifecycle` | Container Lifecycle Hooks for sentinel container. The default preStop hook keeps sentinel running while the redis container's preStop hook fails the master over. Without it, sentinel stops first and the failover request fails. Ref: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/ | object | see values.yaml |
 | `sentinel.livenessProbe.enabled` |  | bool | `true` |
 | `sentinel.livenessProbe.failureThreshold` | Failure threshold for liveness probe | int | `5` |
 | `sentinel.livenessProbe.initialDelaySeconds` | Initial delay in seconds for liveness probe | int | `30` |
diff --git a/charts/redis-ha/templates/_configs.tpl b/charts/redis-ha/templates/_configs.tpl
index 35d71b0..6b48e34 100644
--- a/charts/redis-ha/templates/_configs.tpl
+++ b/charts/redis-ha/templates/_configs.tpl
@@ -473,48 +473,78 @@
     {{- if or (eq (int .Values.redis.port) 0) (eq (int .Values.sentinel.port) 0) }}
     TLS_CLIENT_OPTION="--tls --cacert /tls-certs/{{ .Values.tls.caCertFile }}{{ if ne (default "yes" .Values.sentinel.authClients) "no"}} --cert /tls-certs/{{ .Values.tls.certFile }} --key /tls-certs/{{ .Values.tls.keyFile }}{{end}}"
     {{- end }}
+    {{- $masterGroupName := include "redis-ha.masterGroupName" . }}
+    redis_local() {
+      redis-cli \
+      {{- if .Values.auth }}
+        -a "${AUTH}" --no-auth-warning \
+      {{- end }}
+        -h localhost \
+      {{- if (int .Values.redis.port) }}
+        -p {{ .Values.redis.port }} \
+      {{- else }}
+        -p {{ .Values.redis.tlsPort }} ${TLS_CLIENT_OPTION} \
+      {{- end}}
+        "$@"
+    }
+    sentinel_local() {
+      redis-cli \
+      {{- if .Values.sentinel.auth }}
+        -a "${SENTINELAUTH}" --no-auth-warning \
+      {{- end }}
+        -h localhost \
+      {{- if (int .Values.sentinel.port) }}
+        -p {{ .Values.sentinel.port }} \
+      {{- else }}
+        -p {{ .Values.sentinel.tlsPort }} ${TLS_CLIENT_OPTION} \
+      {{- end}}
+        "$@"
+    }
     get_redis_role() {
-      is_master=$(
-        redis-cli \
-        {{- if .Values.auth }}
-          -a "${AUTH}" --no-auth-warning \
-        {{- end }}
-          -h localhost \
-        {{- if (int .Values.redis.port) }}
-          -p {{ .Values.redis.port }} \
-        {{- else }}
-          -p {{ .Values.redis.tlsPort }} ${TLS_CLIENT_OPTION} \
-        {{- end}}
-          info | grep -c 'role:master' || true
-      )
+      is_master=$(redis_local info replication | grep -c 'role:master' || true)
     }
     get_redis_role
-    if [[ "$is_master" -eq 1 ]]; then
+    if [ "$is_master" -eq 1 ]; then
       echo "This node is currently master, we trigger a failover."
-      {{- $masterGroupName := include "redis-ha.masterGroupName" . }}
-      response=$(
-        redis-cli \
-        {{- if .Values.sentinel.auth }}
-          -a "${SENTINELAUTH}" --no-auth-warning \
-        {{- end }}
-          -h localhost \
-        {{- if (int .Values.sentinel.port) }}
-          -p {{ .Values.sentinel.port }} \
-        {{- else }}
-          -p {{ .Values.sentinel.tlsPort }} ${TLS_CLIENT_OPTION} \
-        {{- end}}
-          SENTINEL failover {{ $masterGroupName }}
-      )
-      if [[ "$response" != "OK" ]] ; then
+      # Hold writes during the handoff. A write this node accepts after the
+      # promotion is lost. A held write fails with READONLY once this node is
+      # a replica, and the client retries it on the new master.
+      if [ "$(redis_local client pause 35000 write)" != "OK" ]; then
+        echo "Could not pause writes, failing over without the pause."
+      fi
+      old_master=$(sentinel_local sentinel get-master-addr-by-name {{ $masterGroupName }} | head -n 1)
+      response=$(sentinel_local sentinel failover {{ $masterGroupName }})
+      if [ "$response" != "OK" ] ; then
         echo "$response"
+        redis_local client unpause >/dev/null
         exit 1
       fi
       timeout=30
-      while [[ "$is_master" -eq 1 && $timeout -gt 0 ]]; do
+      while [ "$is_master" -eq 1 ] && [ "$timeout" -gt 0 ]; do
         sleep 1
+        # Follow the promoted node at once. Sentinel demotes this node only on
+        # its next check, which keeps the held writes waiting longer.
+        new_master=$(sentinel_local sentinel get-master-addr-by-name {{ $masterGroupName }} | tr '\n' ' ')
+        # shellcheck disable=SC2086
+        set -- $new_master
+        if [ -n "${1:-}" ] && [ "$1" != "$old_master" ]; then
+          redis_local replicaof "$1" "$2" >/dev/null
+        fi
         get_redis_role
         timeout=$((timeout - 1))
+        # Sentinel can accept the request and then abort it, for example
+        # when no replica qualifies. Stop holding writes when that happens.
+        if [ "$is_master" -eq 1 ] && [ "$timeout" -lt 27 ] && \
+           ! sentinel_local sentinel master {{ $masterGroupName }} | grep -q failover_in_progress; then
+          echo "Sentinel is no longer failing over."
+          break
+        fi
       done
+      redis_local client unpause >/dev/null
+      if [ "$is_master" -eq 1 ]; then
+        echo "Failover did not complete within 30s."
+        exit 1
+      fi
       echo "Failover successful"
     fi
 {{- end }}
@@ -1054,6 +1084,37 @@
     exit 0
 {{- end }}
 
+{{- define "sentinel_prestop.sh" }}
+    {{- if eq (int .Values.redis.port) 0 }}
+    TLS_CLIENT_OPTION="--tls --cacert /tls-certs/{{ .Values.tls.caCertFile }}{{ if ne (default "yes" .Values.sentinel.authClients) "no"}} --cert /tls-certs/{{ .Values.tls.certFile }} --key /tls-certs/{{ .Values.tls.keyFile }}{{end}}"
+    {{- end }}
+    redis_role() {
+      redis-cli \
+      {{- if .Values.auth }}
+        -a "${AUTH}" --no-auth-warning \
+      {{- end }}
+        -h localhost \
+      {{- if ne (int .Values.redis.port) 0 }}
+        -p {{ .Values.redis.port }} \
+      {{- else }}
+        -p {{ .Values.redis.tlsPort }} ${TLS_CLIENT_OPTION} \
+      {{- end}}
+        role 2>/dev/null | head -n 1
+    }
+    # The redis container's preStop hook asks this sentinel to fail over.
+    # Kubernetes stops both containers at once, so keep this sentinel running
+    # until the local redis is no longer master.
+    waited=0
+    while [ "$waited" -lt 40 ] && [ "$(redis_role)" = "master" ]; do
+      sleep 1
+      waited=$((waited + 1))
+    done
+    if [ "$waited" -gt 0 ]; then
+      # Give this sentinel time to publish the new configuration.
+      sleep 5
+    fi
+{{- end }}
+
 {{- define "redis_readiness.sh" }}
     {{- if not (ne (int .Values.sentinel.port) 0) }}
     TLS_CLIENT_OPTION="--tls --cacert /tls-certs/{{ .Values.tls.caCertFile }}{{ if ne (default "yes" .Values.sentinel.authClients) "no"}} --cert /tls-certs/{{ .Values.tls.certFile }} --key /tls-certs/{{ .Values.tls.keyFile }}{{end}}"
diff --git a/charts/redis-ha/templates/redis-ha-health-configmap.yaml b/charts/redis-ha/templates/redis-ha-health-configmap.yaml
index 7aa7002..ef79c70 100644
--- a/charts/redis-ha/templates/redis-ha-health-configmap.yaml
+++ b/charts/redis-ha/templates/redis-ha-health-configmap.yaml
@@ -18,3 +18,5 @@ data:
 {{- include "redis_readiness.sh" . }}
   sentinel_liveness.sh: |
 {{- include "sentinel_liveness.sh" . }}
+  sentinel_prestop.sh: |
+{{- include "sentinel_prestop.sh" . }}
diff --git a/charts/redis-ha/templates/redis-ha-statefulset.yaml b/charts/redis-ha/templates/redis-ha-statefulset.yaml
index bab5ea7..58a86fe 100644
--- a/charts/redis-ha/templates/redis-ha-statefulset.yaml
+++ b/charts/redis-ha/templates/redis-ha-statefulset.yaml
@@ -294,8 +294,9 @@ spec:
           - /data/conf/redis.conf
         {{- end }}
         securityContext: {{- include "compatibility.renderSecurityContext" (dict "secContext" .Values.containerSecurityContext "context" $) | nindent 10 }}
-        {{- if .Values.auth }}
+        {{- if or .Values.auth .Values.sentinel.auth }}
         env:
+        {{- if .Values.auth }}
         - name: AUTH
           valueFrom:
             secretKeyRef:
@@ -306,6 +307,19 @@ spec:
             {{- end }}
               key: {{ .Values.authKey }}
         {{- end }}
+        {{- if .Values.sentinel.auth }}
+        # The preStop hook asks sentinel to fail over, so it needs sentinel's password.
+        - name: SENTINELAUTH
+          valueFrom:
+            secretKeyRef:
+            {{- if tpl (.Values.sentinel.existingSecret | default "" ) . }}
+              name: {{ tpl (.Values.sentinel.existingSecret | default "" ) . }}
+            {{- else }}
+              name: {{ template "redis-ha.fullname" . }}-sentinel
+            {{- end }}
+              key: {{ .Values.sentinel.authKey }}
+        {{- end }}
+        {{- end }}
         {{- if .Values.redis.envFrom }}
         envFrom:
 {{ toYaml .Values.redis.envFrom | indent 10 }}
diff --git a/charts/redis-ha/values.yaml b/charts/redis-ha/values.yaml
index 95ad969..43e3eef 100644
--- a/charts/redis-ha/values.yaml
+++ b/charts/redis-ha/values.yaml
@@ -645,8 +645,15 @@ sentinel:
   #    memory: 200Mi
 
   # -- Container Lifecycle Hooks for sentinel container.
+  # The default preStop hook keeps sentinel running while the redis container's
+  # preStop hook fails the master over. Without it, sentinel stops first and the
+  # failover request fails.
   # Ref: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/
-  lifecycle: {}
+  # @default -- see values.yaml
+  lifecycle:
+    preStop:
+      exec:
+        command: ["/bin/sh", "/health/sentinel_prestop.sh"]
 
   # -- additional volumeMounts for Sentinel container
   extraVolumeMounts: []

From b427c228beef10e3c46c2b96f83d0304a953ec02 Mon Sep 17 00:00:00 2001
From: Robert Moucha <robert.moucha@gooddata.com>
Date: Fri, 9 Oct 2026 11:52:45 +0200
Subject: [PATCH 6/8] fix(redis-ha): do not restart a crashed master with stale
 data

When the redis container of the master restarts inside its pod, for
example after an OOM kill or a crash, it is back within seconds. Sentinel
has not noticed the outage, so the node returns as master with the data
it last saved. Every replica then resyncs from that older data, and the
whole set loses the writes made since the last save.

Start redis through redis-start.sh. config-init clears a marker on every
pod start, so the script can tell a container restart from a pod start.
On a container restart, if sentinel still names this node as master, it
asks sentinel to promote a replica and starts as that replica's replica.

Signed-off-by: Robert Moucha <robert.moucha@gooddata.com>
Co-authored-by: Claude <noreply@anthropic.com>
---
 charts/redis-ha/templates/_configs.tpl        | 39 +++++++++++++++++++
 .../templates/redis-ha-configmap.yaml         |  2 +
 .../templates/redis-ha-statefulset.yaml       |  3 +-
 3 files changed, 43 insertions(+), 1 deletion(-)

diff --git a/charts/redis-ha/templates/_configs.tpl b/charts/redis-ha/templates/_configs.tpl
index 6b48e34..4b3bd17 100644
--- a/charts/redis-ha/templates/_configs.tpl
+++ b/charts/redis-ha/templates/_configs.tpl
@@ -427,6 +427,8 @@
     {{- include "lib.sh" . }}
 
     mkdir -p /data/conf/
+    # A pod start, not a container restart: see redis-start.sh.
+    rm -f /data/conf/.redis-started
 
     echo "Initializing config.."
     copy_config
@@ -469,6 +471,43 @@
     echo "$(date) Ready..."
 {{- end }}
 
+{{- define "redis-start.sh" }}
+    {{- include "vars.sh" . }}
+
+    set -eu
+
+    {{- include "lib.sh" . }}
+
+    # config-init removes the marker on every pod start, so finding it means
+    # the redis container restarted inside a running pod: a crash, an OOM
+    # kill or a failed liveness probe. If sentinel still names this node as
+    # master, its data is older than its replicas' data. Starting it as
+    # master would make every replica resync from that older data. Ask
+    # sentinel to promote a replica first, and start as its replica.
+    if [ -f /data/conf/.redis-started ]; then
+        identify_announce_ip
+        MASTER="$(sentinel_get_master)"
+        if [ -n "${ANNOUNCE_IP}" ] && [ "${MASTER}" = "${ANNOUNCE_IP}" ]; then
+            echo "$(date) Restarted while sentinel names this node as master; failing over first.."
+            echo "  sentinel replied: $(sentinel_failover)"
+            _wait=0
+            while [ "${_wait}" -lt 30 ] && [ "$(sentinel_get_master)" = "${ANNOUNCE_IP}" ]; do
+                sleep 1
+                _wait=$((_wait + 1))
+            done
+            MASTER="$(sentinel_get_master)"
+            if [ -n "${MASTER}" ] && [ "${MASTER}" != "${ANNOUNCE_IP}" ]; then
+                sed -i -e '/^slaveof /d' -e '/^replicaof /d' "${REDIS_CONF}"
+                redis_update "${MASTER}"
+            else
+                echo "  $(date) No other master; starting with the local data."
+            fi
+        fi
+    fi
+    touch /data/conf/.redis-started
+    exec redis-server "$@"
+{{- end }}
+
 {{- define "trigger-failover-if-master.sh" }}
     {{- if or (eq (int .Values.redis.port) 0) (eq (int .Values.sentinel.port) 0) }}
     TLS_CLIENT_OPTION="--tls --cacert /tls-certs/{{ .Values.tls.caCertFile }}{{ if ne (default "yes" .Values.sentinel.authClients) "no"}} --cert /tls-certs/{{ .Values.tls.certFile }} --key /tls-certs/{{ .Values.tls.keyFile }}{{end}}"
diff --git a/charts/redis-ha/templates/redis-ha-configmap.yaml b/charts/redis-ha/templates/redis-ha-configmap.yaml
index 48f2b03..1c11852 100644
--- a/charts/redis-ha/templates/redis-ha-configmap.yaml
+++ b/charts/redis-ha/templates/redis-ha-configmap.yaml
@@ -35,3 +35,5 @@ data:
 {{- include "config-haproxy_init.sh" . }}
   trigger-failover-if-master.sh: |
 {{- include "trigger-failover-if-master.sh" . }}
+  redis-start.sh: |
+{{- include "redis-start.sh" . }}
diff --git a/charts/redis-ha/templates/redis-ha-statefulset.yaml b/charts/redis-ha/templates/redis-ha-statefulset.yaml
index 58a86fe..9e41a3a 100644
--- a/charts/redis-ha/templates/redis-ha-statefulset.yaml
+++ b/charts/redis-ha/templates/redis-ha-statefulset.yaml
@@ -285,7 +285,8 @@ spec:
         {{- if .Values.redis.customCommand }}
 {{ toYaml .Values.redis.customCommand | indent 10 }}
         {{- else }}
-          - redis-server
+          - sh
+          - /readonly-config/redis-start.sh
         {{- end }}
         args:
         {{- if .Values.redis.customArgs }}

From 4de1538eec712c145dffc688dc248586c77a1739 Mon Sep 17 00:00:00 2001
From: Robert Moucha <robert.moucha@gooddata.com>
Date: Fri, 9 Oct 2026 19:25:01 +0200
Subject: [PATCH 7/8] fix(redis-ha): keep the master role off terminating
 replicas

When a replica pod and the master pod stop at the same time, sentinel
can promote the replica that is shutting down. The set then waits for
sentinel to notice the second loss and fail over again.

A terminating replica now sets replica-priority 0, then watches its role
for 10s. If a failover that started before sentinel saw the new priority
promotes it anyway, it hands the master role on with the same handoff
the master uses. The old master sets replica-priority 0 after its own
handoff. The sentinel preStop hook stays up for the replica's watch.

Signed-off-by: Robert Moucha <robert.moucha@gooddata.com>
Co-authored-by: Claude <noreply@anthropic.com>
---
 charts/redis-ha/templates/_configs.tpl | 27 +++++++++++++++++++++++---
 1 file changed, 24 insertions(+), 3 deletions(-)

diff --git a/charts/redis-ha/templates/_configs.tpl b/charts/redis-ha/templates/_configs.tpl
index 4b3bd17..56f9446 100644
--- a/charts/redis-ha/templates/_configs.tpl
+++ b/charts/redis-ha/templates/_configs.tpl
@@ -543,6 +543,18 @@
       is_master=$(redis_local info replication | grep -c 'role:master' || true)
     }
     get_redis_role
+    if [ "$is_master" -ne 1 ]; then
+      # A terminating replica must not be promoted. A failover that started
+      # before sentinel saw the new priority can still pick it, so watch for
+      # a few seconds and hand the role on if that happens.
+      redis_local config set replica-priority 0 >/dev/null 2>&1 || true
+      waited=0
+      while [ "$is_master" -ne 1 ] && [ "$waited" -lt 10 ]; do
+        sleep 1
+        get_redis_role
+        waited=$((waited + 1))
+      done
+    fi
     if [ "$is_master" -eq 1 ]; then
       echo "This node is currently master, we trigger a failover."
       # Hold writes during the handoff. A write this node accepts after the
@@ -584,6 +596,8 @@
         echo "Failover did not complete within 30s."
         exit 1
       fi
+      # This node is now a terminating replica: keep the next failover off it.
+      redis_local config set replica-priority 0 >/dev/null 2>&1 || true
       echo "Failover successful"
     fi
 {{- end }}
@@ -1142,13 +1156,20 @@
     }
     # The redis container's preStop hook asks this sentinel to fail over.
     # Kubernetes stops both containers at once, so keep this sentinel running
-    # until the local redis is no longer master.
+    # until the local redis is no longer master. A replica watches for 10s
+    # whether it gets promoted anyway, so stay up at least that long.
     waited=0
-    while [ "$waited" -lt 40 ] && [ "$(redis_role)" = "master" ]; do
+    was_master=0
+    while [ "$waited" -lt 40 ]; do
+      if [ "$(redis_role)" = "master" ]; then
+        was_master=1
+      elif [ "$waited" -ge 12 ]; then
+        break
+      fi
       sleep 1
       waited=$((waited + 1))
     done
-    if [ "$waited" -gt 0 ]; then
+    if [ "$was_master" -eq 1 ]; then
       # Give this sentinel time to publish the new configuration.
       sleep 5
     fi

From 377de48aed4ae840557a3b6fa0ae8627d707c91f Mon Sep 17 00:00:00 2001
From: Robert Moucha <robert.moucha@gooddata.com>
Date: Fri, 9 Oct 2026 22:29:20 +0200
Subject: [PATCH 8/8] chore(redis-ha): bump chart version to 4.40.0

Minor bump: the sentinel container gets a default preStop hook, and the redis container starts through redis-start.sh.

Signed-off-by: Robert Moucha <robert.moucha@gooddata.com>
Co-authored-by: Claude <noreply@anthropic.com>
---
 charts/redis-ha/Chart.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/charts/redis-ha/Chart.yaml b/charts/redis-ha/Chart.yaml
index d4e21a8..5c4e920 100644
--- a/charts/redis-ha/Chart.yaml
+++ b/charts/redis-ha/Chart.yaml
@@ -5,7 +5,7 @@ keywords:
 - redis
 - keyvalue
 - database
-version: 4.39.0
+version: 4.40.0
 appVersion: 8.8.0
 description: This Helm chart provides a highly available Redis implementation with a master/slave configuration and uses Sentinel sidecars for failover management
 icon: https://img.icons8.com/external-tal-revivo-shadow-tal-revivo/24/external-redis-an-in-memory-data-structure-project-implementing-a-distributed-logo-shadow-tal-revivo.png