diff --git a/.github/workflows/helm-install-test.yml b/.github/workflows/helm-install-test.yml index b0dba80..bc22a13 100644 --- a/.github/workflows/helm-install-test.yml +++ b/.github/workflows/helm-install-test.yml @@ -156,6 +156,7 @@ jobs: --set externalRedis.passwordKey=redis-password \ --set admin.enabled=true \ --set admin.secret=test-admin-secret-for-ci \ + --set frontproxy.enabled=true \ --set replicaCount=3 \ --set resources.limits.cpu=100m \ --set resources.requests.cpu=50m \ @@ -173,6 +174,35 @@ jobs: fi echo "✓ All 3 s3proxy pods running" + - name: Verify front proxy + run: | + set -euo pipefail + echo "=== Front proxy pods ready ===" + kubectl wait --for=condition=ready pod \ + -l app.kubernetes.io/component=frontproxy -n s3proxy --timeout=120s + kubectl get pods -n s3proxy -l app.kubernetes.io/component=frontproxy + + echo "=== S3 round-trip THROUGH the front proxy ===" + kubectl run frontproxy-test -n s3proxy --rm -i --restart=Never \ + --image=amazon/aws-cli:latest \ + --env="AWS_ACCESS_KEY_ID=minioadmin" \ + --env="AWS_SECRET_ACCESS_KEY=minioadmin" \ + --env="AWS_DEFAULT_REGION=us-east-1" \ + --command -- /bin/sh -c ' + set -e + ENDPOINT="http://s3proxy-python-frontproxy:80" + echo "via front proxy - $(date)" > /tmp/fp.txt + ORIG=$(md5sum /tmp/fp.txt | cut -c1-32) + aws --endpoint-url $ENDPOINT s3 mb s3://frontproxy-bucket + aws --endpoint-url $ENDPOINT s3 cp /tmp/fp.txt s3://frontproxy-bucket/fp.txt + aws --endpoint-url $ENDPOINT s3 cp s3://frontproxy-bucket/fp.txt /tmp/down.txt + DOWN=$(md5sum /tmp/down.txt | cut -c1-32) + [ "$ORIG" = "$DOWN" ] || { echo "✗ round-trip via front proxy failed"; exit 1; } + aws --endpoint-url $ENDPOINT s3 rm s3://frontproxy-bucket/fp.txt + aws --endpoint-url $ENDPOINT s3 rb s3://frontproxy-bucket + echo "✓ S3 round-trip through front proxy succeeded" + ' + - name: Check health endpoint run: | kubectl port-forward svc/s3proxy-python 4433:4433 -n s3proxy & @@ -305,6 +335,9 @@ jobs: echo "=== S3Proxy Logs ===" kubectl logs -l app.kubernetes.io/name=s3proxy-python -n s3proxy --tail=100 echo "" + echo "=== Front Proxy Logs ===" + kubectl logs -l app.kubernetes.io/component=frontproxy -n s3proxy --tail=100 + echo "" echo "=== MinIO Logs ===" kubectl logs -l app=minio -n s3proxy --tail=50 echo "" diff --git a/README.md b/README.md index 5d13d25..1d38755 100644 --- a/README.md +++ b/README.md @@ -75,7 +75,7 @@ aws s3 --endpoint-url http://s3proxy-python:4433 cp file.txt s3://bucket/ Use the **same credentials** you configured above. That's it. -> **Endpoints** — In-cluster: `http://s3proxy-python.:4433` · Gateway: `http://s3-gateway.` · Ingress: `https://s3proxy.example.com` +> **Endpoints** — In-cluster: `http://s3proxy-python.:4433` · Front proxy (even load distribution, `frontproxy.enabled=true`): `http://s3proxy-python-frontproxy.` · Outside the cluster: enable `ingress` in front of the front proxy > > **Health** — `GET /healthz` · `GET /readyz` · **Metrics** — `GET /metrics` @@ -131,8 +131,8 @@ A request signed by an access key with no configured KEK is rejected. Via Helm: | `secrets.existingSecrets.enabled` | `false` | Use existing K8s secret | | `admin.secret` | `change-me` | Secret signing admin session cookies (when admin UI on) | | `redis-ha.enabled` | `true` | Deploy embedded Redis HA | -| `gateway.enabled` | `false` | Create gateway service | -| `ingress.enabled` | `false` | Enable ingress | +| `frontproxy.enabled` | `false` | Bundled HAProxy for even load distribution (no external dependency) | +| `ingress.enabled` | `false` | Expose S3 outside the cluster via Ingress (requires `frontproxy.enabled`) | | `performance.memoryLimitMb` | `64` | Memory budget for streaming concurrency | See [chart/README.md](chart/README.md) for all options. diff --git a/chart/README.md b/chart/README.md index 1c78c4f..54d5802 100644 --- a/chart/README.md +++ b/chart/README.md @@ -37,7 +37,7 @@ helm install s3proxy oci://ghcr.io/serversidehannes/s3proxy-python/charts/s3prox | `admin.existingSecret.passwordKey` | `S3PROXY_ADMIN_PASSWORD` | Password key in the existing secret | | `admin.existingSecret.secretKey` | `S3PROXY_ADMIN_SECRET` | Session-secret key in the existing secret | | `admin.ingress.enabled` | `false` | Dedicated Ingress for the dashboard (keep off unless intentionally exposing it) | -| `admin.ingress.className` | `nginx` | Ingress class for the admin Ingress | +| `admin.ingress.className` | `""` | Ingress class for the admin Ingress (set to your cluster's controller) | | `admin.ingress.host` | `""` | Hostname for the dashboard (required when enabled) | | `admin.ingress.annotations` | `{}` | Annotations (e.g. IP allowlist) for the admin Ingress | | `admin.ingress.tls` | `[]` | TLS config for the admin Ingress | @@ -53,14 +53,22 @@ helm install s3proxy oci://ghcr.io/serversidehannes/s3proxy-python/charts/s3prox | `externalRedis.passwordKey` | `redis-password` | Key name in Redis secret | | `service.type` | `ClusterIP` | Service type | | `service.port` | `4433` | Service port | -| `ingress.enabled` | `false` | Enable ingress | -| `ingress.className` | `nginx` | Ingress class | -| `ingress.annotations` | nginx streaming defaults | Ingress annotations | -| `ingress.hosts` | `[]` | Ingress hostnames | +| `frontproxy.enabled` | `false` | Deploy a bundled HAProxy front proxy for even load distribution (see below) | +| `frontproxy.replicaCount` | `2` | Front proxy replicas | +| `frontproxy.image.repository` | `haproxy` | Front proxy image | +| `frontproxy.image.tag` | `3.0-alpine` | Front proxy image tag | +| `frontproxy.service.type` | `ClusterIP` | Front proxy service type | +| `frontproxy.service.port` | `80` | Front proxy service port (clients connect here) | +| `frontproxy.timeouts.client` | `1h` | Client-side timeout (tolerate large transfers) | +| `frontproxy.timeouts.server` | `1h` | Backend timeout (tolerate large transfers) | +| `frontproxy.timeouts.connect` | `10s` | Backend connect timeout | +| `frontproxy.podDisruptionBudget.enabled` | `true` | Enable front proxy PDB | +| `frontproxy.podDisruptionBudget.minAvailable` | `1` | Min available front proxy pods | +| `ingress.enabled` | `false` | Expose S3 outside the cluster via Ingress (requires `frontproxy.enabled`) | +| `ingress.className` | `""` | Ingress class (set to your cluster's controller) | +| `ingress.annotations` | `{}` | Ingress annotations (controller-specific tuning) | +| `ingress.hosts` | `[]` | Ingress host/path rules | | `ingress.tls` | `[]` | Ingress TLS config | -| `gateway.enabled` | `false` | ExternalName gateway service | -| `gateway.serviceName` | `s3-gateway` | Gateway service name | -| `gateway.ingressService` | `ingress-nginx-controller...` | Target ingress service | | `resources.requests.cpu` | `100m` | CPU request | | `resources.requests.memory` | `512Mi` | Memory request | | `resources.limits.cpu` | `500m` | CPU limit | @@ -71,3 +79,41 @@ helm install s3proxy oci://ghcr.io/serversidehannes/s3proxy-python/charts/s3prox | `topologySpreadConstraints` | `[]` | Topology spread | | `podDisruptionBudget.enabled` | `true` | Enable PDB | | `podDisruptionBudget.minAvailable` | `1` | Min available pods | + +## Even load distribution (front proxy) + +A plain `ClusterIP` Service balances per **connection** (L4, via kube-proxy). S3 +clients (boto3, aws-cli) hold long-lived keep-alive connections, so each client +tends to pin to a single pod — leaving load uneven across replicas. + +Setting `frontproxy.enabled=true` deploys a small bundled HAProxy in front of the +s3proxy pods that balances per **request** (L7), spreading even a single client's +stream across every pod. It is fully self-contained — it requires **no** ingress +controller, service mesh, or anything else in the cluster. + +```bash +helm install s3proxy ... --set frontproxy.enabled=true +``` + +Clients then connect to the front proxy Service (`s3proxy-python-frontproxy`) +instead of the s3proxy Service. The front proxy runs `replicaCount: 2` with a PDB so +it is not a single point of failure. + +### Exposing S3 outside the cluster + +For S3 operations from **outside** the cluster, enable an Ingress in front of the +front proxy. It is controller-agnostic — set `ingress.className` to whatever ingress +controller you run and add any controller-specific tuning via `ingress.annotations`. + +```bash +helm install s3proxy ... \ + --set frontproxy.enabled=true \ + --set ingress.enabled=true \ + --set ingress.className=nginx \ + --set 'ingress.hosts[0].host=s3.example.com' \ + --set 'ingress.hosts[0].paths[0].path=/' \ + --set 'ingress.hosts[0].paths[0].pathType=Prefix' +``` + +The Ingress routes to the front proxy, so external clients also get even per-request +distribution. `ingress.enabled` therefore requires `frontproxy.enabled=true`. diff --git a/chart/templates/deployment.yaml b/chart/templates/deployment.yaml index 7a381ad..5167198 100644 --- a/chart/templates/deployment.yaml +++ b/chart/templates/deployment.yaml @@ -13,6 +13,7 @@ spec: metadata: labels: {{- include "s3proxy.labels" . | nindent 8 }} + app.kubernetes.io/component: server spec: containers: - name: {{ .Chart.Name }} diff --git a/chart/templates/frontproxy-configmap.yaml b/chart/templates/frontproxy-configmap.yaml new file mode 100644 index 0000000..55e2b6c --- /dev/null +++ b/chart/templates/frontproxy-configmap.yaml @@ -0,0 +1,39 @@ +{{- if .Values.frontproxy.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Chart.Name }}-frontproxy + labels: + {{- include "s3proxy.labels" . | nindent 4 }} + app.kubernetes.io/component: frontproxy +data: + haproxy.cfg: | + global + log stdout format raw local0 + maxconn 4096 + + defaults + mode http + log global + option httplog + option dontlognull + timeout connect {{ .Values.frontproxy.timeouts.connect }} + timeout client {{ .Values.frontproxy.timeouts.client }} + timeout server {{ .Values.frontproxy.timeouts.server }} + + frontend http_in + bind *:{{ .Values.frontproxy.service.port }} + default_backend s3proxy_pods + + backend s3proxy_pods + balance leastconn + # Per-request balancing: don't pin a client to one server, and re-resolve the + # headless Service via DNS so new/restarted pods are picked up automatically. + option redispatch + server-template pod 1-{{ int .Values.replicaCount }} {{ .Chart.Name }}-headless:{{ .Values.service.port }} check resolvers k8s init-addr none + + resolvers k8s + parse-resolv-conf + accepted_payload_size 8192 + hold valid 5s +{{- end }} diff --git a/chart/templates/frontproxy-deployment.yaml b/chart/templates/frontproxy-deployment.yaml new file mode 100644 index 0000000..615b605 --- /dev/null +++ b/chart/templates/frontproxy-deployment.yaml @@ -0,0 +1,68 @@ +{{- if .Values.frontproxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Chart.Name }}-frontproxy + labels: + {{- include "s3proxy.labels" . | nindent 4 }} + app.kubernetes.io/component: frontproxy +spec: + replicas: {{ .Values.frontproxy.replicaCount }} + selector: + matchLabels: + {{- include "s3proxy.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: frontproxy + template: + metadata: + labels: + {{- include "s3proxy.labels" . | nindent 8 }} + app.kubernetes.io/component: frontproxy + annotations: + checksum/config: {{ include (print $.Template.BasePath "/frontproxy-configmap.yaml") . | sha256sum }} + spec: + containers: + - name: haproxy + image: "{{ .Values.frontproxy.image.repository }}:{{ .Values.frontproxy.image.tag }}" + imagePullPolicy: {{ .Values.frontproxy.image.pullPolicy }} + ports: + - name: http + containerPort: {{ .Values.frontproxy.service.port }} + protocol: TCP + volumeMounts: + - name: config + mountPath: /usr/local/etc/haproxy/haproxy.cfg + subPath: haproxy.cfg + readOnly: true + livenessProbe: + tcpSocket: + port: http + initialDelaySeconds: 5 + periodSeconds: 10 + readinessProbe: + tcpSocket: + port: http + initialDelaySeconds: 2 + periodSeconds: 5 + resources: + {{- toYaml .Values.frontproxy.resources | nindent 12 }} + securityContext: + runAsNonRoot: true + runAsUser: 99 + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + volumes: + - name: config + configMap: + name: {{ .Chart.Name }}-frontproxy + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + {{- include "s3proxy.selectorLabels" . | nindent 14 }} + app.kubernetes.io/component: frontproxy +{{- end }} diff --git a/chart/templates/frontproxy-pdb.yaml b/chart/templates/frontproxy-pdb.yaml new file mode 100644 index 0000000..45d7e99 --- /dev/null +++ b/chart/templates/frontproxy-pdb.yaml @@ -0,0 +1,15 @@ +{{- if and .Values.frontproxy.enabled .Values.frontproxy.podDisruptionBudget.enabled }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ .Chart.Name }}-frontproxy + labels: + {{- include "s3proxy.labels" . | nindent 4 }} + app.kubernetes.io/component: frontproxy +spec: + minAvailable: {{ .Values.frontproxy.podDisruptionBudget.minAvailable }} + selector: + matchLabels: + {{- include "s3proxy.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: frontproxy +{{- end }} diff --git a/chart/templates/frontproxy-service.yaml b/chart/templates/frontproxy-service.yaml new file mode 100644 index 0000000..30afe47 --- /dev/null +++ b/chart/templates/frontproxy-service.yaml @@ -0,0 +1,19 @@ +{{- if .Values.frontproxy.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ .Chart.Name }}-frontproxy + labels: + {{- include "s3proxy.labels" . | nindent 4 }} + app.kubernetes.io/component: frontproxy +spec: + type: {{ .Values.frontproxy.service.type }} + selector: + {{- include "s3proxy.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: frontproxy + ports: + - name: http + port: {{ .Values.frontproxy.service.port }} + targetPort: http + protocol: TCP +{{- end }} diff --git a/chart/templates/gateway-service.yaml b/chart/templates/gateway-service.yaml deleted file mode 100644 index be7b373..0000000 --- a/chart/templates/gateway-service.yaml +++ /dev/null @@ -1,13 +0,0 @@ -{{- if .Values.gateway.enabled }} -apiVersion: v1 -kind: Service -metadata: - name: {{ .Values.gateway.serviceName }} - namespace: {{ .Release.Namespace }} - labels: - {{- include "s3proxy.labels" . | nindent 4 }} - app.kubernetes.io/component: gateway -spec: - type: ExternalName - externalName: {{ .Values.gateway.ingressService }} -{{- end }} diff --git a/chart/templates/ingress.yaml b/chart/templates/ingress.yaml index 360db57..76a09f4 100644 --- a/chart/templates/ingress.yaml +++ b/chart/templates/ingress.yaml @@ -1,4 +1,10 @@ {{- if .Values.ingress.enabled -}} +{{- if not .Values.frontproxy.enabled }} +{{- fail "ingress.enabled requires frontproxy.enabled=true — the ingress routes to the front proxy" }} +{{- end }} +{{- if not .Values.ingress.hosts }} +{{- fail "ingress.enabled is true but ingress.hosts is empty — set at least one host" }} +{{- end }} apiVersion: networking.k8s.io/v1 kind: Ingress metadata: @@ -10,10 +16,12 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} spec: - ingressClassName: {{ .Values.ingress.className }} - {{- if .Values.ingress.tls }} + {{- with .Values.ingress.className }} + ingressClassName: {{ . }} + {{- end }} + {{- with .Values.ingress.tls }} tls: - {{- range .Values.ingress.tls }} + {{- range . }} - hosts: {{- range .hosts }} - {{ . | quote }} @@ -22,19 +30,6 @@ spec: {{- end }} {{- end }} rules: - {{- if and .Values.gateway.enabled (not .Values.ingress.hosts) }} - # Auto-generate host from gateway service name + namespace - - host: {{ printf "%s.%s" .Values.gateway.serviceName .Release.Namespace | quote }} - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: {{ .Chart.Name }} - port: - number: {{ .Values.service.port }} - {{- else }} {{- range .Values.ingress.hosts }} - host: {{ .host | quote }} http: @@ -44,10 +39,9 @@ spec: pathType: {{ .pathType }} backend: service: - name: {{ $.Chart.Name }} + name: {{ $.Chart.Name }}-frontproxy port: - number: {{ $.Values.service.port }} + number: {{ $.Values.frontproxy.service.port }} {{- end }} {{- end }} - {{- end }} {{- end }} diff --git a/chart/templates/service.yaml b/chart/templates/service.yaml index b83df52..45023c5 100644 --- a/chart/templates/service.yaml +++ b/chart/templates/service.yaml @@ -8,8 +8,31 @@ spec: type: {{ .Values.service.type }} selector: {{- include "s3proxy.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: server ports: - name: http port: {{ .Values.service.port }} targetPort: http protocol: TCP +{{- if .Values.frontproxy.enabled }} +--- +# Headless Service: resolves to individual pod IPs so the front proxy can +# enumerate and balance per-request across every s3proxy pod (not a single VIP). +apiVersion: v1 +kind: Service +metadata: + name: {{ .Chart.Name }}-headless + labels: + {{- include "s3proxy.labels" . | nindent 4 }} +spec: + clusterIP: None + publishNotReadyAddresses: false + selector: + {{- include "s3proxy.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: server + ports: + - name: http + port: {{ .Values.service.port }} + targetPort: http + protocol: TCP +{{- end }} diff --git a/chart/values.yaml b/chart/values.yaml index 7f7be6e..8195e2f 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -117,13 +117,13 @@ admin: passwordKey: "S3PROXY_ADMIN_PASSWORD" secretKey: "S3PROXY_ADMIN_SECRET" - # Optional dedicated Ingress for the dashboard, separate from the S3 data-plane - # ingress above. OFF by default — the admin UI should not be internet-facing - # unless you explicitly expose it (ideally behind an internal ingress class, - # an IP allowlist, or VPN). Routes `admin.path` to the proxy service. + # Optional dedicated Ingress for the dashboard. OFF by default — the admin UI + # should not be internet-facing unless you explicitly expose it (ideally behind + # an internal ingress class, an IP allowlist, or VPN). Routes `admin.path` to the + # proxy service. Set `className` to whatever ingress controller your cluster runs. ingress: enabled: false - className: "nginx" + className: "" host: "" annotations: {} tls: [] # e.g. [{ secretName: admin-tls, hosts: [admin.example.com] }] @@ -140,10 +140,55 @@ service: type: ClusterIP port: 4433 -gateway: +# Bundled front proxy (HAProxy) for even load distribution across s3proxy pods. +# A plain ClusterIP balances per-connection (L4), so S3 clients holding long-lived +# keep-alive connections pin to a single pod. The front proxy balances per-request +# (L7) across all pods, fanning out even a single client's stream. Self-contained: +# requires nothing else in the cluster. OFF by default — clients then talk to the +# front proxy Service (`s3proxy-python-frontproxy`) instead of the s3proxy Service. +frontproxy: enabled: false - serviceName: s3-gateway - ingressService: ingress-nginx-controller.ingress-nginx.svc.cluster.local + replicaCount: 2 + image: + repository: haproxy + tag: "3.0-alpine" + pullPolicy: IfNotPresent + service: + type: ClusterIP + port: 80 + # Per-request timeouts must tolerate large S3 transfers. + timeouts: + client: "1h" + server: "1h" + connect: "10s" + resources: + requests: + cpu: "50m" + memory: "64Mi" + limits: + cpu: "500m" + memory: "128Mi" + podDisruptionBudget: + enabled: true + minAvailable: 1 + +# Optional Ingress to expose S3 operations OUTSIDE the cluster. Routes to the +# front proxy, so external clients also get even per-request distribution — +# therefore requires `frontproxy.enabled: true`. Controller-agnostic: set +# `className` to whatever ingress controller your cluster runs, and add any +# controller-specific tuning (buffering, body size, timeouts) via `annotations`. +ingress: + enabled: false + className: "" + annotations: {} + hosts: [] + # - host: s3.example.com + # paths: + # - path: / + # pathType: Prefix + tls: [] + # - secretName: s3-tls + # hosts: [s3.example.com] nodeSelector: {} @@ -157,15 +202,4 @@ topologySpreadConstraints: [] podDisruptionBudget: enabled: true minAvailable: 1 - # maxUnavailable: 1 # Alternative to minAvailable - -ingress: - enabled: false - className: "nginx" - annotations: - nginx.ingress.kubernetes.io/proxy-buffering: "off" - nginx.ingress.kubernetes.io/proxy-request-buffering: "off" - nginx.ingress.kubernetes.io/proxy-send-timeout: "3600" - nginx.ingress.kubernetes.io/proxy-read-timeout: "3600" - tls: [] - hosts: [] \ No newline at end of file + # maxUnavailable: 1 # Alternative to minAvailable \ No newline at end of file diff --git a/e2e/clickhouse/templates/backup-config.yaml b/e2e/clickhouse/templates/backup-config.yaml index 9784617..d16f22a 100644 --- a/e2e/clickhouse/templates/backup-config.yaml +++ b/e2e/clickhouse/templates/backup-config.yaml @@ -15,7 +15,7 @@ data: username: default password: "" s3: - endpoint: http://s3-gateway.s3proxy:80 + endpoint: http://s3proxy-python-frontproxy.s3proxy:80 bucket: clickhouse-backups path: backups access_key: minioadmin diff --git a/e2e/cluster.sh b/e2e/cluster.sh index 3633964..5ee91e1 100755 --- a/e2e/cluster.sh +++ b/e2e/cluster.sh @@ -89,7 +89,7 @@ case "${1:-help}" in --env="AWS_SECRET_ACCESS_KEY=minioadmin" \ --env="AWS_DEFAULT_REGION=us-east-1" \ --command -- /bin/sh -c " - aws --endpoint-url http://s3-gateway.s3proxy s3 mb s3://load-test-bucket 2>/dev/null || true + aws --endpoint-url http://s3proxy-python-frontproxy.s3proxy s3 mb s3://load-test-bucket 2>/dev/null || true echo \"Generating 10MB test files...\" mkdir -p /tmp/testfiles @@ -102,19 +102,19 @@ case "${1:-help}" in echo \"=== Starting concurrent uploads ===\" START=\$(date +%s) for i in 1 2 3; do - aws --endpoint-url http://s3-gateway.s3proxy s3 cp /tmp/testfiles/file-\$i.bin s3://load-test-bucket/file-\$i.bin & + aws --endpoint-url http://s3proxy-python-frontproxy.s3proxy s3 cp /tmp/testfiles/file-\$i.bin s3://load-test-bucket/file-\$i.bin & done wait END=\$(date +%s) echo \"Upload complete in \$((END - START))s\" echo \"=== Verifying uploads ===\" - aws --endpoint-url http://s3-gateway.s3proxy s3 ls s3://load-test-bucket/ + aws --endpoint-url http://s3proxy-python-frontproxy.s3proxy s3 ls s3://load-test-bucket/ echo \"=== Downloading and verifying ===\" mkdir -p /tmp/downloads for i in 1 2 3; do - aws --endpoint-url http://s3-gateway.s3proxy s3 cp s3://load-test-bucket/file-\$i.bin /tmp/downloads/file-\$i.bin & + aws --endpoint-url http://s3proxy-python-frontproxy.s3proxy s3 cp s3://load-test-bucket/file-\$i.bin /tmp/downloads/file-\$i.bin & done wait diff --git a/e2e/docker-compose.yml b/e2e/docker-compose.yml index deb95f1..04f61c3 100644 --- a/e2e/docker-compose.yml +++ b/e2e/docker-compose.yml @@ -82,7 +82,6 @@ services: # 2. Helm repo setup + dependency build (background) ( - helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx helm repo add dandydeveloper https://dandydeveloper.github.io/charts helm repo update helm dependency build /repo/chart @@ -359,15 +358,6 @@ services: port: 9001 MINIOEOF - # 2. Start ingress-nginx install (background) - echo " - Ingress Controller starting..." - helm upgrade --install ingress-nginx ingress-nginx/ingress-nginx \ - --namespace ingress-nginx --create-namespace \ - --set controller.service.type=ClusterIP \ - --set controller.admissionWebhooks.enabled=false \ - --wait --timeout 300s & - INGRESS_PID=$$! - # 3. Wait for docker builds (already running in background) echo "=== Waiting for docker builds ===" wait $$BUILD_PID || { echo "s3proxy build failed"; exit 1; } @@ -406,7 +396,7 @@ services: " & BUCKETS_PID=$$! - # 7. Start s3proxy install (background) - MinIO is ready, ingress not needed yet + # 7. Start s3proxy install (background) - MinIO is ready echo "=== Installing s3proxy (background) ===" helm upgrade --install s3proxy /repo/chart \ -n s3proxy --wait --timeout 600s \ @@ -419,10 +409,8 @@ services: --set admin.secret="$$ENCRYPT_KEY" \ --set logLevel="DEBUG" \ --set performance.memoryLimitMb=64 \ - --set gateway.enabled=true \ - --set ingress.enabled=true \ + --set frontproxy.enabled=true \ --set admin.enabled=true \ - --set 'ingress.annotations.nginx\.ingress\.kubernetes\.io/proxy-body-size=256m' \ --set redis-ha.enabled=true \ --set redis-ha.persistentVolume.enabled=true \ --set redis-ha.persistentVolume.size=10Gi \ @@ -448,7 +436,6 @@ services: # Infrastructure wait $$BUCKETS_PID || { echo "Bucket creation failed"; exit 1; }; echo "✓ Buckets" - wait $$INGRESS_PID || { echo "Ingress failed"; exit 1; }; echo "✓ Ingress" wait $$S3PROXY_PID || { echo "S3proxy failed"; exit 1; }; echo "✓ S3Proxy" # Operators @@ -464,7 +451,7 @@ services: kubectl get svc -n s3proxy echo "" - echo "S3 Proxy endpoint for databases: http://s3-gateway.s3proxy.svc.cluster.local:80" + echo "S3 Proxy endpoint for databases: http://s3proxy-python-frontproxy.s3proxy.svc.cluster.local:80" echo "Direct MinIO (unencrypted): http://minio.minio.svc.cluster.local:9000" echo "" diff --git a/e2e/elasticsearch/templates/elasticsearch-cluster-restore.yaml b/e2e/elasticsearch/templates/elasticsearch-cluster-restore.yaml index 99b7fd4..88c069c 100644 --- a/e2e/elasticsearch/templates/elasticsearch-cluster-restore.yaml +++ b/e2e/elasticsearch/templates/elasticsearch-cluster-restore.yaml @@ -9,7 +9,7 @@ spec: count: 3 config: node.store.allow_mmap: false - s3.client.default.endpoint: "s3-gateway.s3proxy:80" + s3.client.default.endpoint: "s3proxy-python-frontproxy.s3proxy:80" s3.client.default.protocol: "http" s3.client.default.path_style_access: true podTemplate: diff --git a/e2e/elasticsearch/templates/elasticsearch-cluster.yaml b/e2e/elasticsearch/templates/elasticsearch-cluster.yaml index acca31e..9480fbb 100644 --- a/e2e/elasticsearch/templates/elasticsearch-cluster.yaml +++ b/e2e/elasticsearch/templates/elasticsearch-cluster.yaml @@ -10,7 +10,7 @@ spec: config: node.store.allow_mmap: false # S3 repository settings - s3.client.default.endpoint: "s3-gateway.s3proxy:80" + s3.client.default.endpoint: "s3proxy-python-frontproxy.s3proxy:80" s3.client.default.protocol: "http" s3.client.default.path_style_access: true podTemplate: diff --git a/e2e/elasticsearch/test.sh b/e2e/elasticsearch/test.sh index 3253f90..0c17a64 100755 --- a/e2e/elasticsearch/test.sh +++ b/e2e/elasticsearch/test.sh @@ -122,7 +122,7 @@ kubectl exec -n "$NAMESPACE" "${ES_POD}" -- \ "type": "s3", "settings": { "bucket": "elasticsearch-backups", - "endpoint": "s3-gateway.s3proxy:80", + "endpoint": "s3proxy-python-frontproxy.s3proxy:80", "protocol": "http", "path_style_access": true } @@ -237,7 +237,7 @@ kubectl exec -n "$NAMESPACE" "${RESTORED_POD}" -- \ "type": "s3", "settings": { "bucket": "elasticsearch-backups", - "endpoint": "s3-gateway.s3proxy:80", + "endpoint": "s3proxy-python-frontproxy.s3proxy:80", "protocol": "http", "path_style_access": true } diff --git a/e2e/postgres/templates/postgres-cluster-restore.yaml b/e2e/postgres/templates/postgres-cluster-restore.yaml index 97eeadf..a5a8937 100644 --- a/e2e/postgres/templates/postgres-cluster-restore.yaml +++ b/e2e/postgres/templates/postgres-cluster-restore.yaml @@ -25,7 +25,7 @@ spec: - name: ${CLUSTER_NAME} barmanObjectStore: destinationPath: "s3://postgres-backups/" - endpointURL: "http://s3-gateway.s3proxy:80" + endpointURL: "http://s3proxy-python-frontproxy.s3proxy:80" s3Credentials: accessKeyId: name: s3-credentials diff --git a/e2e/postgres/templates/postgres-cluster.yaml b/e2e/postgres/templates/postgres-cluster.yaml index 8fa8f53..8434ce6 100644 --- a/e2e/postgres/templates/postgres-cluster.yaml +++ b/e2e/postgres/templates/postgres-cluster.yaml @@ -25,7 +25,7 @@ spec: backup: barmanObjectStore: destinationPath: "s3://postgres-backups/" - endpointURL: "http://s3-gateway.s3proxy:80" + endpointURL: "http://s3proxy-python-frontproxy.s3proxy:80" s3Credentials: accessKeyId: name: s3-credentials diff --git a/e2e/s3-compatibility/templates/s3-tests-config.yaml b/e2e/s3-compatibility/templates/s3-tests-config.yaml index 0653159..43cfe17 100644 --- a/e2e/s3-compatibility/templates/s3-tests-config.yaml +++ b/e2e/s3-compatibility/templates/s3-tests-config.yaml @@ -5,8 +5,8 @@ metadata: data: s3tests.conf: | [DEFAULT] - # Use gateway hostname matching ingress rule - host = s3-gateway.s3proxy + # Front proxy service (even load distribution across s3proxy pods) + host = s3proxy-python-frontproxy.s3proxy port = 80 is_secure = False ssl_verify = False diff --git a/e2e/scylla/templates/agent-config-secret.yaml b/e2e/scylla/templates/agent-config-secret.yaml index 305219f..01dd138 100644 --- a/e2e/scylla/templates/agent-config-secret.yaml +++ b/e2e/scylla/templates/agent-config-secret.yaml @@ -9,4 +9,4 @@ stringData: access_key_id: minioadmin secret_access_key: minioadmin provider: Minio - endpoint: http://s3-gateway.s3proxy:80 + endpoint: http://s3proxy-python-frontproxy.s3proxy:80 diff --git a/s3proxy/config.py b/s3proxy/config.py index 98a8e8f..0d9e24e 100644 --- a/s3proxy/config.py +++ b/s3proxy/config.py @@ -52,7 +52,7 @@ class Settings(BaseSettings): # Memory settings # This is the ONLY setting needed for OOM protection. - # Use nginx proxy-body-size at ingress to reject oversized requests before they reach Python. + # Reject oversized requests at the front proxy to stop them before they reach Python. memory_limit_mb: int = Field( default=64, description="Memory budget for concurrent requests in MB. 0=unlimited. " diff --git a/tests/unit/test_chart_frontproxy.py b/tests/unit/test_chart_frontproxy.py new file mode 100644 index 0000000..02b285c --- /dev/null +++ b/tests/unit/test_chart_frontproxy.py @@ -0,0 +1,214 @@ +"""Helm chart tests for the bundled front proxy and the optional ingress. + +These render the chart with `helm template` and assert on the output, so they +cover the template logic (guards, selectors, backend wiring) without a cluster. +Skipped automatically when the `helm` binary is unavailable. +""" + +from __future__ import annotations + +import shutil +import subprocess +from pathlib import Path + +import pytest +import yaml + +CHART_DIR = Path(__file__).resolve().parents[2] / "chart" + +pytestmark = pytest.mark.skipif(shutil.which("helm") is None, reason="helm binary not available") + + +@pytest.fixture(scope="module") +def chart_dir(tmp_path_factory) -> Path: + """A copy of the chart with the redis-ha dependency stripped from Chart.yaml. + + `helm template` checks declared dependencies before evaluating their + `condition`, so it fails on a clean checkout where charts/ is empty — even + with redis-ha.enabled=false. These tests don't exercise redis-ha, so we + render a dependency-free copy and stay fully hermetic (no network, no + `helm dependency build`).""" + dest = tmp_path_factory.mktemp("chart") / "chart" + shutil.copytree(CHART_DIR, dest) + shutil.rmtree(dest / "charts", ignore_errors=True) + chart_yaml = dest / "Chart.yaml" + meta = yaml.safe_load(chart_yaml.read_text()) + meta.pop("dependencies", None) + chart_yaml.write_text(yaml.safe_dump(meta, sort_keys=False)) + return dest + + +def _render(chart_dir: Path, set_args: tuple[str, ...]) -> subprocess.CompletedProcess: + cmd = [ + "helm", + "template", + "t", + str(chart_dir), + "--set", + "secrets.encryptKey=x", + "--set", + "secrets.awsAccessKeyId=a", + "--set", + "secrets.awsSecretAccessKey=b", + ] + for s in set_args: + cmd += ["--set", s] + return subprocess.run(cmd, capture_output=True, text=True) + + +def render(chart_dir: Path, *set_args: str) -> list[dict]: + """Render the chart and return parsed manifests.""" + out = _render(chart_dir, set_args) + if out.returncode != 0: + raise AssertionError(f"helm template failed:\n{out.stderr}") + return [d for d in yaml.safe_load_all(out.stdout) if d] + + +def render_expect_error(chart_dir: Path, *set_args: str) -> str: + """Render expecting failure; return stderr for assertion.""" + out = _render(chart_dir, set_args) + assert out.returncode != 0, "expected helm template to fail but it succeeded" + return out.stderr + + +def by_kind_name(docs: list[dict], kind: str, name: str) -> dict | None: + for d in docs: + if d.get("kind") == kind and d["metadata"]["name"] == name: + return d + return None + + +# -------------------------------------------------------------------------- +# Default: front proxy and ingress are off +# -------------------------------------------------------------------------- + + +def test_disabled_by_default(chart_dir): + docs = render(chart_dir) + names = {(d.get("kind"), d["metadata"]["name"]) for d in docs} + assert ("Deployment", "s3proxy-python-frontproxy") not in names + assert ("Service", "s3proxy-python-frontproxy") not in names + assert ("Service", "s3proxy-python-headless") not in names + assert not any(d.get("kind") == "Ingress" for d in docs) + + +def test_no_nginx_or_gateway_anywhere(chart_dir): + """The whole point of the change: no nginx/gateway artifacts, on or off.""" + docs = render(chart_dir, "frontproxy.enabled=true") + blob = yaml.safe_dump_all(docs).lower() + assert "nginx" not in blob + assert "gateway" not in blob + assert "externalname" not in blob + + +# -------------------------------------------------------------------------- +# Front proxy +# -------------------------------------------------------------------------- + + +def test_frontproxy_resources_rendered(chart_dir): + docs = render(chart_dir, "frontproxy.enabled=true") + for kind, name in [ + ("Deployment", "s3proxy-python-frontproxy"), + ("Service", "s3proxy-python-frontproxy"), + ("ConfigMap", "s3proxy-python-frontproxy"), + ("PodDisruptionBudget", "s3proxy-python-frontproxy"), + ("Service", "s3proxy-python-headless"), + ]: + assert by_kind_name(docs, kind, name), f"missing {kind}/{name}" + + +def test_headless_service_is_headless(chart_dir): + docs = render(chart_dir, "frontproxy.enabled=true") + hs = by_kind_name(docs, "Service", "s3proxy-python-headless") + assert hs["spec"]["clusterIP"] == "None" + assert hs["spec"]["selector"]["app.kubernetes.io/component"] == "server" + + +def test_service_selectors_do_not_overlap(chart_dir): + """The main/headless Services must select only s3proxy pods, the frontproxy + Service only HAProxy pods — otherwise the front proxy round-robins to itself.""" + docs = render(chart_dir, "frontproxy.enabled=true") + main = by_kind_name(docs, "Service", "s3proxy-python") + fp = by_kind_name(docs, "Service", "s3proxy-python-frontproxy") + assert main["spec"]["selector"]["app.kubernetes.io/component"] == "server" + assert fp["spec"]["selector"]["app.kubernetes.io/component"] == "frontproxy" + + server_pod = by_kind_name(docs, "Deployment", "s3proxy-python") + fp_pod = by_kind_name(docs, "Deployment", "s3proxy-python-frontproxy") + server_labels = server_pod["spec"]["template"]["metadata"]["labels"] + fp_labels = fp_pod["spec"]["template"]["metadata"]["labels"] + assert server_labels["app.kubernetes.io/component"] == "server" + assert fp_labels["app.kubernetes.io/component"] == "frontproxy" + + +def test_haproxy_config_balances_and_scales_with_replicas(chart_dir): + docs = render(chart_dir, "frontproxy.enabled=true", "replicaCount=5") + cm = by_kind_name(docs, "ConfigMap", "s3proxy-python-frontproxy") + cfg = cm["data"]["haproxy.cfg"] + assert "balance leastconn" in cfg + # per-request balancing across all pods via the headless Service + assert "server-template pod 1-5 s3proxy-python-headless:4433" in cfg + assert "parse-resolv-conf" in cfg + + +def test_frontproxy_replicas_and_pdb(chart_dir): + docs = render(chart_dir, "frontproxy.enabled=true") + dep = by_kind_name(docs, "Deployment", "s3proxy-python-frontproxy") + assert dep["spec"]["replicas"] == 2 + pdb = by_kind_name(docs, "PodDisruptionBudget", "s3proxy-python-frontproxy") + assert pdb["spec"]["minAvailable"] == 1 + + +# -------------------------------------------------------------------------- +# Ingress (external access) — routes to the front proxy +# -------------------------------------------------------------------------- + + +def _ingress_args() -> list[str]: + return [ + "frontproxy.enabled=true", + "ingress.enabled=true", + "ingress.className=nginx", + "ingress.hosts[0].host=s3.example.com", + "ingress.hosts[0].paths[0].path=/", + "ingress.hosts[0].paths[0].pathType=Prefix", + ] + + +def test_ingress_routes_to_frontproxy(chart_dir): + docs = render(chart_dir, *_ingress_args()) + ing = by_kind_name(docs, "Ingress", "s3proxy-python") + assert ing is not None + assert ing["spec"]["ingressClassName"] == "nginx" + backend = ing["spec"]["rules"][0]["http"]["paths"][0]["backend"]["service"] + assert backend["name"] == "s3proxy-python-frontproxy" + assert backend["port"]["number"] == 80 + + +def test_ingress_requires_frontproxy(chart_dir): + err = render_expect_error( + chart_dir, + "ingress.enabled=true", + "ingress.hosts[0].host=s3.example.com", + "ingress.hosts[0].paths[0].path=/", + "ingress.hosts[0].paths[0].pathType=Prefix", + ) + assert "frontproxy.enabled=true" in err + + +def test_ingress_requires_hosts(chart_dir): + err = render_expect_error(chart_dir, "frontproxy.enabled=true", "ingress.enabled=true") + assert "ingress.hosts is empty" in err + + +def test_ingress_tls_rendered(chart_dir): + docs = render( + chart_dir, + *_ingress_args(), + "ingress.tls[0].secretName=s3-tls", + "ingress.tls[0].hosts[0]=s3.example.com", + ) + ing = by_kind_name(docs, "Ingress", "s3proxy-python") + assert ing["spec"]["tls"][0]["secretName"] == "s3-tls" + assert "s3.example.com" in ing["spec"]["tls"][0]["hosts"]