diff --git a/.github/workflows/helm-install-test.yml b/.github/workflows/helm-install-test.yml index bc22a13..63c7646 100644 --- a/.github/workflows/helm-install-test.yml +++ b/.github/workflows/helm-install-test.yml @@ -71,58 +71,6 @@ jobs: EOF kubectl wait --for=condition=ready pod -l app=minio -n s3proxy --timeout=120s - - name: Deploy Redis with password - run: | - # Create Redis password secret - kubectl create secret generic redis-secret \ - --from-literal=redis-password=testredispassword123 \ - -n s3proxy - - cat < Multipart SigV4 - Redis HA + Distributed State Scaling

@@ -130,7 +130,7 @@ A request signed by an access key with no configured KEK is rejected. Via Helm: | `secrets.credentials` | `[]` | AWS credentials, each `{accessKey, secretKey, kek}` | | `secrets.existingSecrets.enabled` | `false` | Use existing K8s secret | | `admin.secret` | `change-me` | Secret signing admin session cookies (when admin UI on) | -| `redis-ha.enabled` | `true` | Deploy embedded Redis HA | +| `redis.enabled` | `true` | Deploy the bundled single Redis pod (transient upload state) | | `frontproxy.enabled` | `false` | Bundled HAProxy for even load distribution (no external dependency) | | `ingress.enabled` | `false` | Expose S3 outside the cluster via Ingress (requires `frontproxy.enabled`) | | `performance.memoryLimitMb` | `64` | Memory budget for streaming concurrency | @@ -153,7 +153,7 @@ Data written by that credential is unrecoverable. Each object records the access
What if Redis fails mid-upload? -Upload fails and must restart. Use redis-ha.enabled=true with persistence. +Only in-flight multipart uploads are affected — they fail and must restart. No committed object is ever at risk, since Redis holds only transient, TTL'd upload state. For extra resilience, point externalRedis.url at your own HA Redis and set redis.enabled=false.
diff --git a/chart/Chart.lock b/chart/Chart.lock deleted file mode 100644 index aef14ae..0000000 --- a/chart/Chart.lock +++ /dev/null @@ -1,6 +0,0 @@ -dependencies: -- name: redis-ha - repository: https://dandydeveloper.github.io/charts - version: 4.35.10 -digest: sha256:847c879db9c912fe3623ed305e15aaa6e3d3809d9810b9f730785f837e71a193 -generated: "2026-06-07T15:34:11.732857499Z" diff --git a/chart/Chart.yaml b/chart/Chart.yaml index 757c5f9..94d6fc5 100644 --- a/chart/Chart.yaml +++ b/chart/Chart.yaml @@ -4,9 +4,3 @@ description: Transparent S3 encryption proxy with AES-256-GCM type: application version: 2026.2.0 appVersion: "2026.2.0" - -dependencies: - - name: redis-ha - version: 4.35.10 - repository: "https://dandydeveloper.github.io/charts" - condition: redis-ha.enabled diff --git a/chart/README.md b/chart/README.md index 54d5802..6c3970a 100644 --- a/chart/README.md +++ b/chart/README.md @@ -41,13 +41,14 @@ helm install s3proxy oci://ghcr.io/serversidehannes/s3proxy-python/charts/s3prox | `admin.ingress.host` | `""` | Hostname for the dashboard (required when enabled) | | `admin.ingress.annotations` | `{}` | Annotations (e.g. IP allowlist) for the admin Ingress | | `admin.ingress.tls` | `[]` | TLS config for the admin Ingress | -| `redis-ha.enabled` | `true` | Deploy embedded Redis HA | -| `redis-ha.replicas` | `1` | Redis replicas | -| `redis-ha.auth` | `false` | Enable Redis auth | -| `redis-ha.haproxy.enabled` | `true` | Deploy HAProxy for Redis | -| `redis-ha.persistentVolume.enabled` | `true` | Persistent storage | -| `redis-ha.persistentVolume.size` | `10Gi` | Volume size | -| `externalRedis.url` | `""` | External Redis URL | +| `redis.enabled` | `true` | Deploy the bundled single Redis pod (transient upload state) | +| `redis.image.repository` | `redis` | Redis image (bump manually; not tracked by dependabot) | +| `redis.image.tag` | `7-alpine` | Redis image tag | +| `redis.auth.enabled` | `false` | Require a password on the bundled Redis | +| `redis.auth.password` | `""` | Password (when auth enabled and no existingSecret) | +| `redis.auth.existingSecret` | `""` | Use a pre-created Secret for the Redis password | +| `redis.auth.passwordKey` | `redis-password` | Key name in the Redis password Secret | +| `externalRedis.url` | `""` | External Redis URL (used when `redis.enabled=false`) | | `externalRedis.uploadTtlHours` | `24` | Upload state TTL | | `externalRedis.existingSecret` | `""` | K8s secret with Redis password | | `externalRedis.passwordKey` | `redis-password` | Key name in Redis secret | diff --git a/chart/templates/configmap.yaml b/chart/templates/configmap.yaml index 7d1d7a9..5a46ed7 100644 --- a/chart/templates/configmap.yaml +++ b/chart/templates/configmap.yaml @@ -11,8 +11,8 @@ data: S3PROXY_PORT: {{ .Values.server.port | quote }} S3PROXY_NO_TLS: {{ .Values.server.noTls | quote }} S3PROXY_MEMORY_LIMIT_MB: {{ .Values.performance.memoryLimitMb | quote }} - {{- if index .Values "redis-ha" "enabled" }} - S3PROXY_REDIS_URL: "redis://{{ .Release.Name }}-redis-ha-haproxy:6379/0" + {{- if .Values.redis.enabled }} + S3PROXY_REDIS_URL: "redis://{{ .Chart.Name }}-redis:{{ .Values.redis.port }}/0" {{- else }} S3PROXY_REDIS_URL: {{ .Values.externalRedis.url | quote }} {{- end }} diff --git a/chart/templates/deployment.yaml b/chart/templates/deployment.yaml index 5167198..cdb8d76 100644 --- a/chart/templates/deployment.yaml +++ b/chart/templates/deployment.yaml @@ -35,8 +35,12 @@ spec: is set, or the app uses an existing secret so envFrom never fires). */ -}} {{- $adminFromChartSecret := and .Values.admin.enabled (not .Values.admin.existingSecret.name) (not .Values.secrets.existingSecrets.enabled) }} {{- $adminNeedsEnv := and .Values.admin.enabled (not $adminFromChartSecret) }} + {{- /* Redis password is needed when the bundled Redis has auth on, or + external Redis is used with an existing secret. */ -}} + {{- $redisAuth := and .Values.redis.enabled .Values.redis.auth.enabled }} + {{- $extRedisAuth := and (not .Values.redis.enabled) .Values.externalRedis.existingSecret }} {{- /* Determine if we need env section */ -}} - {{- $needsEnv := or .Values.secrets.existingSecrets.enabled $adminNeedsEnv (and (index .Values "redis-ha" "enabled") (index .Values "redis-ha" "auth")) (and (not (index .Values "redis-ha" "enabled")) .Values.externalRedis.existingSecret) }} + {{- $needsEnv := or .Values.secrets.existingSecrets.enabled $adminNeedsEnv $redisAuth $extRedisAuth }} {{- if $needsEnv }} env: {{- /* App secrets from existing secret */ -}} @@ -66,19 +70,15 @@ spec: name: {{ $adminSecret }} key: {{ .Values.admin.existingSecret.secretKey }} {{- end }} - {{- /* Redis password from redis-ha secret */ -}} - {{- if and (index .Values "redis-ha" "enabled") (index .Values "redis-ha" "auth") }} + {{- /* Redis password from the bundled Redis secret */ -}} + {{- if $redisAuth }} - name: S3PROXY_REDIS_PASSWORD valueFrom: secretKeyRef: - {{- if index .Values "redis-ha" "existingSecret" }} - name: {{ index .Values "redis-ha" "existingSecret" }} - {{- else }} - name: {{ .Release.Name }}-redis-ha - {{- end }} - key: {{ index .Values "redis-ha" "authKey" | default "auth" }} + name: {{ .Values.redis.auth.existingSecret | default (printf "%s-redis" .Chart.Name) }} + key: {{ .Values.redis.auth.passwordKey }} {{- /* Redis password from external Redis secret */ -}} - {{- else if and (not (index .Values "redis-ha" "enabled")) .Values.externalRedis.existingSecret }} + {{- else if $extRedisAuth }} - name: S3PROXY_REDIS_PASSWORD valueFrom: secretKeyRef: diff --git a/chart/templates/frontproxy-configmap.yaml b/chart/templates/frontproxy-configmap.yaml index 55e2b6c..568536e 100644 --- a/chart/templates/frontproxy-configmap.yaml +++ b/chart/templates/frontproxy-configmap.yaml @@ -29,8 +29,10 @@ data: balance leastconn # Per-request balancing: don't pin a client to one server, and re-resolve the # headless Service via DNS so new/restarted pods are picked up automatically. + # Use the FQDN: HAProxy's resolver does NOT apply /etc/resolv.conf search + # domains, so a bare service name NXDOMAINs and leaves 0 backends (503 ). option redispatch - server-template pod 1-{{ int .Values.replicaCount }} {{ .Chart.Name }}-headless:{{ .Values.service.port }} check resolvers k8s init-addr none + server-template pod 1-{{ int .Values.replicaCount }} {{ .Chart.Name }}-headless.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.service.port }} check resolvers k8s init-addr none resolvers k8s parse-resolv-conf diff --git a/chart/templates/pdb.yaml b/chart/templates/pdb.yaml index 8743536..8c1308b 100644 --- a/chart/templates/pdb.yaml +++ b/chart/templates/pdb.yaml @@ -16,4 +16,5 @@ spec: selector: matchLabels: {{- include "s3proxy.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: server {{- end }} diff --git a/chart/templates/redis/deployment.yaml b/chart/templates/redis/deployment.yaml new file mode 100644 index 0000000..b34ae4b --- /dev/null +++ b/chart/templates/redis/deployment.yaml @@ -0,0 +1,81 @@ +{{- if .Values.redis.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Chart.Name }}-redis + labels: + {{- include "s3proxy.labels" . | nindent 4 }} + app.kubernetes.io/component: redis +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + {{- include "s3proxy.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: redis + template: + metadata: + labels: + {{- include "s3proxy.labels" . | nindent 8 }} + app.kubernetes.io/component: redis + spec: + containers: + - name: redis + image: "{{ .Values.redis.image.repository }}:{{ .Values.redis.image.tag }}" + imagePullPolicy: {{ .Values.redis.image.pullPolicy }} + args: + - "--maxmemory" + - {{ .Values.redis.maxmemory | quote }} + - "--maxmemory-policy" + - {{ .Values.redis.maxmemoryPolicy | quote }} + {{- if .Values.redis.auth.enabled }} + - "--requirepass" + - "$(REDIS_PASSWORD)" + {{- end }} + {{- if .Values.redis.auth.enabled }} + env: + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.redis.auth.existingSecret | default (printf "%s-redis" .Chart.Name) }} + key: {{ .Values.redis.auth.passwordKey }} + {{- end }} + ports: + - name: redis + containerPort: {{ .Values.redis.port }} + protocol: TCP + livenessProbe: + tcpSocket: + port: redis + initialDelaySeconds: 5 + periodSeconds: 10 + readinessProbe: + exec: + command: + - sh + - -c + {{- if .Values.redis.auth.enabled }} + - 'redis-cli -a "$REDIS_PASSWORD" ping | grep -q PONG' + {{- else }} + - 'redis-cli ping | grep -q PONG' + {{- end }} + initialDelaySeconds: 2 + periodSeconds: 5 + resources: + {{- toYaml .Values.redis.resources | nindent 12 }} + securityContext: + runAsNonRoot: true + runAsUser: 999 + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + volumeMounts: + - name: data + mountPath: /data + volumes: + - name: data + emptyDir: {} +{{- end }} diff --git a/chart/templates/redis/secret.yaml b/chart/templates/redis/secret.yaml new file mode 100644 index 0000000..3d1bd2f --- /dev/null +++ b/chart/templates/redis/secret.yaml @@ -0,0 +1,15 @@ +{{- if and .Values.redis.enabled .Values.redis.auth.enabled (not .Values.redis.auth.existingSecret) }} +{{- if not .Values.redis.auth.password }} +{{- fail "redis.auth.enabled is true but redis.auth.password is empty — set a password or point redis.auth.existingSecret at your own Secret" }} +{{- end }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Chart.Name }}-redis + labels: + {{- include "s3proxy.labels" . | nindent 4 }} + app.kubernetes.io/component: redis +type: Opaque +stringData: + {{ .Values.redis.auth.passwordKey }}: {{ .Values.redis.auth.password | quote }} +{{- end }} diff --git a/chart/templates/redis/service.yaml b/chart/templates/redis/service.yaml new file mode 100644 index 0000000..50e2584 --- /dev/null +++ b/chart/templates/redis/service.yaml @@ -0,0 +1,19 @@ +{{- if .Values.redis.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ .Chart.Name }}-redis + labels: + {{- include "s3proxy.labels" . | nindent 4 }} + app.kubernetes.io/component: redis +spec: + type: ClusterIP + selector: + {{- include "s3proxy.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: redis + ports: + - name: redis + port: {{ .Values.redis.port }} + targetPort: redis + protocol: TCP +{{- end }} diff --git a/chart/values.yaml b/chart/values.yaml index 8195e2f..e1c9726 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -18,62 +18,31 @@ server: performance: memoryLimitMb: 64 -externalRedis: - url: "" - uploadTtlHours: 24 - existingSecret: "" - passwordKey: "redis-password" - -redis-ha: +# Redis holds only transient multipart-upload state (TTL'd, deleted on +# completion). It is NOT a durable store — losing it only forces in-flight +# multipart uploads to restart, never any committed object. So a single +# bundled Redis pod is the sensible default; for HA or shared infra, disable +# this and point `externalRedis.url` at your own Redis. +redis: enabled: true - replicas: 1 - auth: false - redisPassword: "" - existingSecret: "" - authKey: "auth" - - persistentVolume: - enabled: true - size: 10Gi - storageClass: "" - - haproxy: - enabled: true - replicas: 2 - # CRITICAL: Health check and timeout settings for high load tolerance - # Redis can be slow during BGSAVE - don't mark as DOWN too quickly - checkInterval: 5s # Check every 5 seconds (default: 1s) - timeout: - check: "10s" # Wait 10s for health check response (default: 2s) - server: "60s" # Server response timeout (default: 30s) - client: "60s" # Client connection timeout (default: 30s) - resources: - requests: - cpu: "50m" - memory: "64Mi" - limits: - cpu: "200m" - memory: "128Mi" - - sentinel: - port: 26379 - quorum: 2 - config: - down-after-milliseconds: 10000 - failover-timeout: 180000 - parallel-syncs: 5 - - redis: - port: 6379 - config: - maxmemory: "200mb" - maxmemory-policy: noeviction - - min-replicas-to-write: 2 - min-replicas-max-lag: 10 - - hardAntiAffinity: true - + image: + # NOTE: not auto-bumped by dependabot (it doesn't track image tags in + # values.yaml). Bump this tag manually when upgrading Redis. + repository: redis + tag: "7-alpine" + pullPolicy: IfNotPresent + port: 6379 + # maxmemory-policy noeviction so the proxy gets an error rather than silent + # eviction of in-flight upload state. + maxmemory: "200mb" + maxmemoryPolicy: "noeviction" + # Optional auth. Off by default (in-cluster ClusterIP, transient data). + # Set `password`, or point `existingSecret`/`passwordKey` at your own Secret. + auth: + enabled: false + password: "" + existingSecret: "" + passwordKey: "redis-password" resources: requests: cpu: "100m" @@ -82,6 +51,14 @@ redis-ha: cpu: "500m" memory: "256Mi" +# External Redis — used when `redis.enabled: false`. Point at a Redis you run +# yourself (managed service, shared cluster, your own HA setup). +externalRedis: + url: "" + uploadTtlHours: 24 + existingSecret: "" + passwordKey: "redis-password" + secrets: existingSecrets: enabled: false diff --git a/e2e/docker-compose.yml b/e2e/docker-compose.yml index 04f61c3..664bf2d 100644 --- a/e2e/docker-compose.yml +++ b/e2e/docker-compose.yml @@ -18,6 +18,11 @@ services: privileged: true depends_on: - registry + # Publish the s3proxy admin panel to the host. After the cluster is up, the + # entrypoint runs `kubectl port-forward` (on 0.0.0.0) onto container :14433, + # so the dashboard is reachable at http://localhost:14433/admin (admin/admin). + ports: + - "14433:14433" volumes: - /var/run/docker.sock:/var/run/docker.sock - ..:/repo @@ -204,23 +209,36 @@ services: # 1. ECK (Elasticsearch) operator ( + set -e + # retry wrapper: apply a remote manifest, failing hard only after N attempts + eck_apply() { + for i in 1 2 3 4 5; do + kubectl apply $$2 -f "$$1" && return 0 + echo "apply $$1 failed, retry $$i..."; sleep 10 + done + echo "ERROR: apply $$1 failed after 5 attempts"; return 1 + } + # wait for a CRD to exist, then to be Established (avoids the + # "no matching resources" race when wait runs before apply propagates) + eck_wait_crd() { + until kubectl get "crd/$$1" >/dev/null 2>&1; do sleep 2; done + kubectl wait --for=condition=Established "crd/$$1" --timeout=180s + } + echo "Installing ECK CRDs..." - for i in 1 2 3; do - kubectl apply -f https://download.elastic.co/downloads/eck/3.3.2/crds.yaml && break - echo "ECK CRDs apply failed, retry $$i..." - sleep 5 - done - # Wait for CRDs to be established before applying operator + eck_apply https://download.elastic.co/downloads/eck/3.3.2/crds.yaml echo "Waiting for ECK CRDs to be established..." - kubectl wait --for=condition=Established crd/elasticsearches.elasticsearch.k8s.elastic.co --timeout=60s - kubectl wait --for=condition=Established crd/kibanas.kibana.k8s.elastic.co --timeout=60s + eck_wait_crd elasticsearches.elasticsearch.k8s.elastic.co + eck_wait_crd kibanas.kibana.k8s.elastic.co + echo "Installing ECK operator..." + eck_apply https://download.elastic.co/downloads/eck/3.3.2/operator.yaml + # statefulset may not be registered the instant apply returns; wait for it + until kubectl get --namespace elastic-system statefulset/elastic-operator >/dev/null 2>&1; do sleep 2; done for i in 1 2 3; do - kubectl apply -f https://download.elastic.co/downloads/eck/3.3.2/operator.yaml && break - echo "ECK operator apply failed, retry $$i..." - sleep 5 + kubectl rollout status --namespace elastic-system statefulset/elastic-operator --timeout=300s && break + echo "elastic-operator rollout not ready, retry $$i..."; sleep 10 done - kubectl rollout status --namespace elastic-system statefulset/elastic-operator --timeout=300s echo "✓ ECK operator ready" ) & ECK_PID=$$! @@ -236,7 +254,14 @@ services: # 3. ClickHouse operator ( - kubectl apply -f https://raw.githubusercontent.com/Altinity/clickhouse-operator/master/deploy/operator/clickhouse-operator-install-bundle.yaml + set -e + for i in 1 2 3 4 5; do + kubectl apply -f https://raw.githubusercontent.com/Altinity/clickhouse-operator/master/deploy/operator/clickhouse-operator-install-bundle.yaml && break + echo "ClickHouse operator apply failed, retry $$i..."; sleep 10 + [ "$$i" = 5 ] && { echo "ERROR: ClickHouse operator apply failed after 5 attempts"; exit 1; } + done + # pod may not be scheduled the instant apply returns; wait for it to exist first + until kubectl get pods --namespace kube-system --selector=app=clickhouse-operator 2>/dev/null | grep -q .; do sleep 2; done kubectl wait --namespace kube-system --for=condition=ready pod --selector=app=clickhouse-operator --timeout=300s echo "✓ ClickHouse operator ready" ) & @@ -244,13 +269,28 @@ services: # 4. cert-manager + Scylla Operator + Scylla Manager (sequential chain in single background job) ( + set -e + # retry wrapper for remote manifests (network-flaky downloads) + remote_apply() { + for i in 1 2 3 4 5; do + kubectl apply $$2 -f "$$1" && return 0 + echo "apply $$1 failed, retry $$i..."; sleep 10 + done + echo "ERROR: apply $$1 failed after 5 attempts"; return 1 + } # cert-manager - kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.0/cert-manager.yaml - kubectl wait --namespace cert-manager --for=condition=ready pod --all --timeout=300s + remote_apply https://github.com/cert-manager/cert-manager/releases/download/v1.19.0/cert-manager.yaml + # wait for the 3 cert-manager deployments to exist before waiting on pods + # (--all matches zero pods and returns immediately if none scheduled yet) + for d in cert-manager cert-manager-cainjector cert-manager-webhook; do + until kubectl get --namespace cert-manager "deployment/$$d" >/dev/null 2>&1; do sleep 2; done + kubectl rollout status --namespace cert-manager "deployment/$$d" --timeout=300s + done echo "✓ cert-manager ready" # Scylla Operator (requires cert-manager) - kubectl apply --server-side -f https://raw.githubusercontent.com/scylladb/scylla-operator/v1.20.2/deploy/operator.yaml + remote_apply https://raw.githubusercontent.com/scylladb/scylla-operator/v1.20.2/deploy/operator.yaml --server-side + until kubectl get pods -l app.kubernetes.io/name=scylla-operator -n scylla-operator 2>/dev/null | grep -q .; do sleep 2; done kubectl wait --for=condition=ready pod -l app.kubernetes.io/name=scylla-operator -n scylla-operator --timeout=300s kubectl wait --for=condition=ready pod -l app.kubernetes.io/name=webhook-server -n scylla-operator --timeout=300s # Wait for webhook to be fully ready @@ -411,24 +451,9 @@ services: --set performance.memoryLimitMb=64 \ --set frontproxy.enabled=true \ --set admin.enabled=true \ - --set redis-ha.enabled=true \ - --set redis-ha.persistentVolume.enabled=true \ - --set redis-ha.persistentVolume.size=10Gi \ - --set redis-ha.hardAntiAffinity=false \ - --set redis-ha.haproxy.hardAntiAffinity=false \ - --set redis-ha.auth=true \ - --set redis-ha.replicas=1 \ - --set redis-ha.redisPassword=testredispassword \ - --set redis-ha.redis.resources.requests.memory=128Mi \ - --set redis-ha.redis.resources.limits.memory=256Mi \ - --set redis-ha.redis.config.maxmemory=200mb \ - --set redis-ha.redis.config.maxmemory-policy=noeviction \ - --set redis-ha.redis.config.min-replicas-to-write=0 \ - --set redis-ha.redis.config.min-replicas-max-lag=10 \ - --set redis-ha.haproxy.checkInterval=5s \ - --set redis-ha.haproxy.timeout.check=10s \ - --set redis-ha.haproxy.timeout.server=60s \ - --set redis-ha.haproxy.timeout.client=60s & + --set redis.enabled=true \ + --set redis.auth.enabled=true \ + --set redis.auth.password=testredispassword & S3PROXY_PID=$$! # 8. Wait for ALL parallel tasks @@ -450,9 +475,18 @@ services: kubectl get pods -n s3proxy kubectl get svc -n s3proxy + # Expose the s3proxy admin panel to the host. The container publishes + # :14433; this self-healing forward maps it to the in-cluster service so + # the dashboard is reachable at http://localhost:14433/admin on the host. + ( while true; do + kubectl port-forward -n s3proxy --address 0.0.0.0 svc/s3proxy-python 14433:4433 >/dev/null 2>&1 || true + sleep 2 + done ) & + echo "" echo "S3 Proxy endpoint for databases: http://s3proxy-python-frontproxy.s3proxy.svc.cluster.local:80" echo "Direct MinIO (unencrypted): http://minio.minio.svc.cluster.local:9000" + echo "Admin panel (host browser): http://localhost:14433/admin (login: admin / admin)" echo "" echo "==========================================" diff --git a/tests/unit/test_chart_frontproxy.py b/tests/unit/test_chart_frontproxy.py index 02b285c..98f346a 100644 --- a/tests/unit/test_chart_frontproxy.py +++ b/tests/unit/test_chart_frontproxy.py @@ -148,10 +148,22 @@ def test_haproxy_config_balances_and_scales_with_replicas(chart_dir): cfg = cm["data"]["haproxy.cfg"] assert "balance leastconn" in cfg # per-request balancing across all pods via the headless Service - assert "server-template pod 1-5 s3proxy-python-headless:4433" in cfg + assert "server-template pod 1-5 " in cfg assert "parse-resolv-conf" in cfg +def test_haproxy_backend_uses_fqdn(chart_dir): + """Regression: HAProxy's resolver ignores /etc/resolv.conf search domains, so + the server-template MUST target the FQDN. A bare name NXDOMAINs -> 0 backends + -> every request 503s with (which is exactly what broke postgres e2e).""" + docs = render(chart_dir, "frontproxy.enabled=true") + cfg = by_kind_name(docs, "ConfigMap", "s3proxy-python-frontproxy")["data"]["haproxy.cfg"] + # default release namespace in `helm template` is "default" + assert "s3proxy-python-headless.default.svc.cluster.local:4433" in cfg + # and never the bare service name (would resolve to nothing in HAProxy) + assert "headless:4433" not in cfg + + def test_frontproxy_replicas_and_pdb(chart_dir): docs = render(chart_dir, "frontproxy.enabled=true") dep = by_kind_name(docs, "Deployment", "s3proxy-python-frontproxy") diff --git a/tests/unit/test_chart_redis.py b/tests/unit/test_chart_redis.py new file mode 100644 index 0000000..e25de54 --- /dev/null +++ b/tests/unit/test_chart_redis.py @@ -0,0 +1,159 @@ +"""Helm chart tests for the bundled single Redis. + +Renders the chart with `helm template` and asserts on the output. The chart has +no external dependencies (redis-ha was removed), so these run on a clean checkout +with no `helm dependency build`. Skipped when the `helm` binary is unavailable. +""" + +from __future__ import annotations + +import shutil +import subprocess +from pathlib import Path + +import pytest +import yaml + +CHART_DIR = Path(__file__).resolve().parents[2] / "chart" + +# Minimal valid credentials so the chart renders. +CREDS = ( + "secrets.credentials[0].accessKey=a", + "secrets.credentials[0].secretKey=b", + "secrets.credentials[0].kek=c", +) + +pytestmark = pytest.mark.skipif(shutil.which("helm") is None, reason="helm binary not available") + + +def _run(set_args: tuple[str, ...]) -> subprocess.CompletedProcess: + cmd = ["helm", "template", "t", str(CHART_DIR)] + for s in CREDS + set_args: + cmd += ["--set", s] + return subprocess.run(cmd, capture_output=True, text=True) + + +def render(*set_args: str) -> list[dict]: + out = _run(set_args) + if out.returncode != 0: + raise AssertionError(f"helm template failed:\n{out.stderr}") + return [d for d in yaml.safe_load_all(out.stdout) if d] + + +def render_expect_error(*set_args: str) -> str: + out = _run(set_args) + assert out.returncode != 0, "expected helm template to fail but it succeeded" + return out.stderr + + +def by_kind_name(docs: list[dict], kind: str, name: str) -> dict | None: + for d in docs: + if d.get("kind") == kind and d["metadata"]["name"] == name: + return d + return None + + +def configmap_data(docs: list[dict]) -> dict: + cm = by_kind_name(docs, "ConfigMap", "s3proxy-python-config") + return cm["data"] + + +# -------------------------------------------------------------------------- +# Bundled Redis is on by default +# -------------------------------------------------------------------------- + + +def test_no_redis_ha_dependency(): + """redis-ha is fully removed — Chart.yaml must declare no dependencies.""" + meta = yaml.safe_load((CHART_DIR / "Chart.yaml").read_text()) + assert "dependencies" not in meta or not meta["dependencies"] + assert not (CHART_DIR / "Chart.lock").exists() + + +def test_redis_rendered_by_default(): + docs = render() + assert by_kind_name(docs, "Deployment", "s3proxy-python-redis") + assert by_kind_name(docs, "Service", "s3proxy-python-redis") + + +def test_proxy_points_at_bundled_redis(): + docs = render() + assert configmap_data(docs)["S3PROXY_REDIS_URL"] == "redis://s3proxy-python-redis:6379/0" + + +def test_redis_uses_emptydir_not_pvc(): + docs = render() + dep = by_kind_name(docs, "Deployment", "s3proxy-python-redis") + vols = dep["spec"]["template"]["spec"]["volumes"] + assert any("emptyDir" in v for v in vols) + assert not any(d.get("kind") == "PersistentVolumeClaim" for d in docs) + + +def test_redis_image_pinned_from_values(): + docs = render("redis.image.tag=7.4-alpine") + dep = by_kind_name(docs, "Deployment", "s3proxy-python-redis") + img = dep["spec"]["template"]["spec"]["containers"][0]["image"] + assert img == "redis:7.4-alpine" + + +def test_service_selectors_do_not_overlap(): + """Main Service must select only server pods, Redis Service only redis pods.""" + docs = render() + main = by_kind_name(docs, "Service", "s3proxy-python") + redis = by_kind_name(docs, "Service", "s3proxy-python-redis") + assert main["spec"]["selector"]["app.kubernetes.io/component"] == "server" + assert redis["spec"]["selector"]["app.kubernetes.io/component"] == "redis" + + server_pod = by_kind_name(docs, "Deployment", "s3proxy-python") + redis_pod = by_kind_name(docs, "Deployment", "s3proxy-python-redis") + assert ( + server_pod["spec"]["template"]["metadata"]["labels"]["app.kubernetes.io/component"] + == "server" + ) + assert ( + redis_pod["spec"]["template"]["metadata"]["labels"]["app.kubernetes.io/component"] + == "redis" + ) + + +# -------------------------------------------------------------------------- +# Auth +# -------------------------------------------------------------------------- + + +def test_no_auth_by_default(): + docs = render() + dep = by_kind_name(docs, "Deployment", "s3proxy-python-redis") + args = dep["spec"]["template"]["spec"]["containers"][0].get("args", []) + assert "--requirepass" not in args + assert not by_kind_name(docs, "Secret", "s3proxy-python-redis") + + +def test_auth_wires_secret_and_requirepass_and_proxy_env(): + docs = render("redis.auth.enabled=true", "redis.auth.password=secret123") + secret = by_kind_name(docs, "Secret", "s3proxy-python-redis") + assert secret["stringData"]["redis-password"] == "secret123" + + redis = by_kind_name(docs, "Deployment", "s3proxy-python-redis") + assert "--requirepass" in redis["spec"]["template"]["spec"]["containers"][0]["args"] + + proxy = by_kind_name(docs, "Deployment", "s3proxy-python") + env = proxy["spec"]["template"]["spec"]["containers"][0].get("env", []) + assert any(e["name"] == "S3PROXY_REDIS_PASSWORD" for e in env) + + +def test_auth_without_password_fails(): + err = render_expect_error("redis.auth.enabled=true") + assert "redis.auth.password is empty" in err + + +# -------------------------------------------------------------------------- +# External Redis (redis.enabled=false) +# -------------------------------------------------------------------------- + + +def test_external_redis_no_bundled_pod(): + docs = render("redis.enabled=false", "externalRedis.url=redis://ext:6379/0") + assert not by_kind_name(docs, "Deployment", "s3proxy-python-redis") + assert not by_kind_name(docs, "Service", "s3proxy-python-redis") + assert configmap_data(docs)["S3PROXY_REDIS_URL"] == "redis://ext:6379/0"