[stable/redis-ha] Make master failover reliable when pods are disrupted

charts/redis-ha/Chart.yaml

@@ -5,7 +5,7 @@ keywords:
 - redis
 - keyvalue
 - database
-version: 4.39.0
+version: 4.40.0
 appVersion: 8.8.0
 description: This Helm chart provides a highly available Redis implementation with a master/slave configuration and uses Sentinel sidecars for failover management
 icon: https://img.icons8.com/external-tal-revivo-shadow-tal-revivo/24/external-redis-an-in-memory-data-structure-project-implementing-a-distributed-logo-shadow-tal-revivo.png

charts/redis-ha/README.md

@@ -208,7 +208,7 @@ The following table lists the configurable parameters of the Redis chart and the
 | `sentinel.customConfig` | Allows for custom sentinel.conf files to be applied. If this is used then `sentinel.config` is ignored | string | `""` |
 | `sentinel.existingSecret` | An existing secret containing a key defined by `sentinel.authKey` that configures `requirepass` in the conf parameters (Requires `sentinel.auth: enabled`, cannot be used in conjunction with `.Values.sentinel.password`) | string | `""` |
 | `sentinel.extraVolumeMounts` | additional volumeMounts for Sentinel container | list | `[]` |
-| `sentinel.lifecycle` | Container Lifecycle Hooks for sentinel container. Ref: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/ | object | `{}` |
+| `sentinel.lifecycle` | Container Lifecycle Hooks for sentinel container. The default preStop hook keeps sentinel running while the redis container's preStop hook fails the master over. Without it, sentinel stops first and the failover request fails. Ref: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/ | object | see values.yaml |
 | `sentinel.livenessProbe.enabled` |  | bool | `true` |
 | `sentinel.livenessProbe.failureThreshold` | Failure threshold for liveness probe | int | `5` |
 | `sentinel.livenessProbe.initialDelaySeconds` | Initial delay in seconds for liveness probe | int | `30` |

charts/redis-ha/templates/redis-ha-configmap.yaml

@@ -35,3 +35,5 @@ data:
 {{- include "config-haproxy_init.sh" . }}
   trigger-failover-if-master.sh: |
 {{- include "trigger-failover-if-master.sh" . }}
+  redis-start.sh: |
+{{- include "redis-start.sh" . }}

charts/redis-ha/templates/redis-ha-health-configmap.yaml

@@ -18,3 +18,5 @@ data:
 {{- include "redis_readiness.sh" . }}
   sentinel_liveness.sh: |
 {{- include "sentinel_liveness.sh" . }}
+  sentinel_prestop.sh: |
+{{- include "sentinel_prestop.sh" . }}

charts/redis-ha/templates/redis-ha-statefulset.yaml

@@ -285,7 +285,8 @@ spec:
         {{- if .Values.redis.customCommand }}
 {{ toYaml .Values.redis.customCommand | indent 10 }}
         {{- else }}
-          - redis-server
+          - sh
+          - /readonly-config/redis-start.sh
         {{- end }}
         args:
         {{- if .Values.redis.customArgs }}
@@ -294,8 +295,9 @@ spec:
           - /data/conf/redis.conf
         {{- end }}
         securityContext: {{- include "compatibility.renderSecurityContext" (dict "secContext" .Values.containerSecurityContext "context" $) | nindent 10 }}
-        {{- if .Values.auth }}
+        {{- if or .Values.auth .Values.sentinel.auth }}
         env:
+        {{- if .Values.auth }}
         - name: AUTH
           valueFrom:
             secretKeyRef:
@@ -306,6 +308,19 @@ spec:
             {{- end }}
               key: {{ .Values.authKey }}
         {{- end }}
+        {{- if .Values.sentinel.auth }}
+        # The preStop hook asks sentinel to fail over, so it needs sentinel's password.
+        - name: SENTINELAUTH
+          valueFrom:
+            secretKeyRef:
+            {{- if tpl (.Values.sentinel.existingSecret | default "" ) . }}
+              name: {{ tpl (.Values.sentinel.existingSecret | default "" ) . }}
+            {{- else }}
+              name: {{ template "redis-ha.fullname" . }}-sentinel
+            {{- end }}
+              key: {{ .Values.sentinel.authKey }}
+        {{- end }}
+        {{- end }}
         {{- if .Values.redis.envFrom }}
         envFrom:
 {{ toYaml .Values.redis.envFrom | indent 10 }}

charts/redis-ha/values.yaml

@@ -645,8 +645,15 @@ sentinel:
   #    memory: 200Mi
 
   # -- Container Lifecycle Hooks for sentinel container.
+  # The default preStop hook keeps sentinel running while the redis container's
+  # preStop hook fails the master over. Without it, sentinel stops first and the
+  # failover request fails.
   # Ref: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/
-  lifecycle: {}
+  # @default -- see values.yaml
+  lifecycle:
+    preStop:
+      exec:
+        command: ["/bin/sh", "/health/sentinel_prestop.sh"]
 
   # -- additional volumeMounts for Sentinel container
   extraVolumeMounts: []