From 9c3b1ea07d726d537e85c5aeb04e494d9fd2b1e3 Mon Sep 17 00:00:00 2001 From: Armando Ruocco Date: Mon, 21 Sep 2026 12:09:25 +0200 Subject: [PATCH 01/11] test: replace MinIO with RustFS as the S3-compatible object store The MinIO images are no longer maintained. Replace them with RustFS in the e2e harness, in hack/ and in the docs, as cloudnative-pg did in #10865, and name the resources in provider-neutral terms (object-store, s3). The e2e Deployment mirrors core: an init container makes the data and log subdirectories writable for the non-root RustFS user, no fsGroup, glibc image, probes on /health. Credentials are longer because RustFS requires at least 3 characters for the access key and 8 for the secret. hack/minio becomes hack/object-store, keeping the cert-manager TLS setup, and the mc pod becomes an AWS CLI pod. The docs replace the MinIO operator section with a RustFS one; versioned docs are untouched. Verified with task e2e-ephemeral and task e2e-external-kind (all S3 specs green, the skipped ones are the pre-existing GCS specs), and by applying hack/object-store with cert-manager on kind: a plugin backup completed over TLS and the cleanup script emptied the bucket. Closes #624 Signed-off-by: Armando Ruocco --- .wordlist.txt | 1 + hack/examples/cluster-example-legacy.yaml | 8 +- hack/examples/cluster-example.yaml | 2 +- .../cluster-replica-log-shipping.yaml | 2 +- hack/examples/cluster-replica-streaming.yaml | 2 +- hack/examples/cluster-restore-archive.yaml | 4 +- hack/examples/cluster-restore.yaml | 2 +- .../{minio-store.yaml => object-store.yaml} | 10 +- hack/minio/kustomization.yaml | 8 -- hack/minio/minio-client.yaml | 20 --- hack/minio/minio-deployment.yaml | 56 -------- hack/object-store/kustomization.yaml | 8 ++ .../object-store-certificate.yaml} | 10 +- .../object-store/object-store-deployment.yaml | 101 ++++++++++++++ .../object-store-pvc.yaml} | 2 +- .../object-store-secret.yaml} | 2 +- .../object-store-service.yaml} | 8 +- hack/object-store/s3-client.yaml | 50 +++++++ .../selfsigned-issuer.yaml | 0 scripts/cleanup.sh | 2 +- scripts/minio-delete.sh | 19 --- .../object-store-delete.sh | 2 +- .../internal/objectstore/{minio.go => s3.go} | 132 ++++++++++++++---- test/e2e/internal/tests/backup/fixtures.go | 26 ++-- .../credentialrotation/credential_rotation.go | 12 +- .../internal/tests/replicacluster/fixtures.go | 12 +- .../tests/replicacluster/replica_cluster.go | 2 +- .../e2e/internal/tests/walrestore/fixtures.go | 20 +-- .../internal/tests/walrestore/walrestore.go | 2 +- web/docs/intro.md | 2 +- web/docs/migration.md | 32 ++--- web/docs/object_stores.md | 75 ++++------ web/docs/usage.md | 28 ++-- 33 files changed, 389 insertions(+), 273 deletions(-) rename hack/examples/{minio-store.yaml => object-store.yaml} (82%) delete mode 100644 hack/minio/kustomization.yaml delete mode 100644 hack/minio/minio-client.yaml delete mode 100644 hack/minio/minio-deployment.yaml create mode 100644 hack/object-store/kustomization.yaml rename hack/{minio/minio-certificate.yaml => object-store/object-store-certificate.yaml} (61%) create mode 100644 hack/object-store/object-store-deployment.yaml rename hack/{minio/minio-pvc.yaml => object-store/object-store-pvc.yaml} (88%) rename hack/{minio/minio-secret.yaml => object-store/object-store-secret.yaml} (89%) rename hack/{minio/minio-service.yaml => object-store/object-store-service.yaml} (56%) create mode 100644 hack/object-store/s3-client.yaml rename hack/{minio => object-store}/selfsigned-issuer.yaml (100%) delete mode 100755 scripts/minio-delete.sh rename hack/minio/minio-delete.sh => scripts/object-store-delete.sh (91%) mode change 100644 => 100755 rename test/e2e/internal/objectstore/{minio.go => s3.go} (51%) diff --git a/.wordlist.txt b/.wordlist.txt index 3df20daf1..ca97df685 100644 --- a/.wordlist.txt +++ b/.wordlist.txt @@ -48,6 +48,7 @@ RTO RecoveryWindow ResourceRequirements RetentionPolicy +RustFS SAS SDK SFO diff --git a/hack/examples/cluster-example-legacy.yaml b/hack/examples/cluster-example-legacy.yaml index 4deb9ffd8..17fb80b67 100644 --- a/hack/examples/cluster-example-legacy.yaml +++ b/hack/examples/cluster-example-legacy.yaml @@ -8,16 +8,16 @@ spec: backup: barmanObjectStore: endpointCA: - name: minio-server-tls + name: object-store-tls key: tls.crt destinationPath: s3://backups/ - endpointURL: https://minio:9000 + endpointURL: https://object-store:9000 s3Credentials: accessKeyId: - name: minio + name: object-store key: ACCESS_KEY_ID secretAccessKey: - name: minio + name: object-store key: ACCESS_SECRET_KEY wal: compression: gzip diff --git a/hack/examples/cluster-example.yaml b/hack/examples/cluster-example.yaml index 1d240b3df..241d4f967 100644 --- a/hack/examples/cluster-example.yaml +++ b/hack/examples/cluster-example.yaml @@ -9,7 +9,7 @@ spec: - name: barman-cloud.cloudnative-pg.io isWALArchiver: true parameters: - barmanObjectName: minio-store + barmanObjectName: object-store storage: size: 1Gi diff --git a/hack/examples/cluster-replica-log-shipping.yaml b/hack/examples/cluster-replica-log-shipping.yaml index 5973b2b6c..f02970c3b 100644 --- a/hack/examples/cluster-replica-log-shipping.yaml +++ b/hack/examples/cluster-replica-log-shipping.yaml @@ -15,7 +15,7 @@ spec: plugin: name: barman-cloud.cloudnative-pg.io parameters: - barmanObjectName: minio-store + barmanObjectName: object-store serverName: cluster-example storage: size: 1Gi diff --git a/hack/examples/cluster-replica-streaming.yaml b/hack/examples/cluster-replica-streaming.yaml index 1fba84cfe..3a76bab74 100644 --- a/hack/examples/cluster-replica-streaming.yaml +++ b/hack/examples/cluster-replica-streaming.yaml @@ -29,7 +29,7 @@ spec: plugin: name: barman-cloud.cloudnative-pg.io parameters: - barmanObjectName: minio-store + barmanObjectName: object-store serverName: cluster-example storage: size: 1Gi diff --git a/hack/examples/cluster-restore-archive.yaml b/hack/examples/cluster-restore-archive.yaml index 0505eeda0..3f8a6047a 100644 --- a/hack/examples/cluster-restore-archive.yaml +++ b/hack/examples/cluster-restore-archive.yaml @@ -14,14 +14,14 @@ spec: - name: barman-cloud.cloudnative-pg.io isWALArchiver: true parameters: - barmanObjectName: minio-store-bis + barmanObjectName: object-store-bis externalClusters: - name: source plugin: name: barman-cloud.cloudnative-pg.io parameters: - barmanObjectName: minio-store + barmanObjectName: object-store serverName: cluster-example storage: diff --git a/hack/examples/cluster-restore.yaml b/hack/examples/cluster-restore.yaml index 8b226acfa..b04be182e 100644 --- a/hack/examples/cluster-restore.yaml +++ b/hack/examples/cluster-restore.yaml @@ -15,7 +15,7 @@ spec: plugin: name: barman-cloud.cloudnative-pg.io parameters: - barmanObjectName: minio-store + barmanObjectName: object-store serverName: cluster-example storage: diff --git a/hack/examples/minio-store.yaml b/hack/examples/object-store.yaml similarity index 82% rename from hack/examples/minio-store.yaml rename to hack/examples/object-store.yaml index cb9eb6556..765f2c556 100644 --- a/hack/examples/minio-store.yaml +++ b/hack/examples/object-store.yaml @@ -1,7 +1,7 @@ apiVersion: barmancloud.cnpg.io/v1 kind: ObjectStore metadata: - name: minio-store + name: object-store spec: retentionPolicy: "1m" instanceSidecarConfiguration: @@ -16,16 +16,16 @@ spec: cpu: "500m" configuration: endpointCA: - name: minio-server-tls + name: object-store-tls key: tls.crt destinationPath: s3://backups/ - endpointURL: https://minio:9000 + endpointURL: https://object-store:9000 s3Credentials: accessKeyId: - name: minio + name: object-store key: ACCESS_KEY_ID secretAccessKey: - name: minio + name: object-store key: ACCESS_SECRET_KEY wal: compression: gzip diff --git a/hack/minio/kustomization.yaml b/hack/minio/kustomization.yaml deleted file mode 100644 index de537e5df..000000000 --- a/hack/minio/kustomization.yaml +++ /dev/null @@ -1,8 +0,0 @@ -resources: -- minio-deployment.yaml -- minio-pvc.yaml -- minio-secret.yaml -- minio-service.yaml -- minio-certificate.yaml -- selfsigned-issuer.yaml - diff --git a/hack/minio/minio-client.yaml b/hack/minio/minio-client.yaml deleted file mode 100644 index 4d8ebb82d..000000000 --- a/hack/minio/minio-client.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - labels: - run: mc - name: mc -spec: - containers: - - env: - - name: MC_HOST_minio - value: http://chooJeiroroo2noquomei2uuceisheth:ongeiqueitohL0queeLohkiur2quaing@minio:9000 - image: minio/mc - name: mc - resources: {} - # Keep the pod up to exec stuff on it - command: - - sleep - - "3600" - dnsPolicy: ClusterFirst - restartPolicy: Always diff --git a/hack/minio/minio-deployment.yaml b/hack/minio/minio-deployment.yaml deleted file mode 100644 index 4bd3d0777..000000000 --- a/hack/minio/minio-deployment.yaml +++ /dev/null @@ -1,56 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: minio - labels: - app: minio -spec: - replicas: 1 - selector: - matchLabels: - app: minio - template: - metadata: - labels: - app: minio - spec: - containers: - - name: minio - image: minio/minio - ports: - - containerPort: 9000 - volumeMounts: - - mountPath: /data - name: data - - mountPath: /opt/minio/certs - name: certs - args: - - server - - --certs-dir - - /opt/minio/certs - - /data - env: - - name: MINIO_ROOT_USER - valueFrom: - secretKeyRef: - name: minio - key: ACCESS_KEY_ID - - name: MINIO_ROOT_PASSWORD - valueFrom: - secretKeyRef: - name: minio - key: ACCESS_SECRET_KEY - volumes: - - name: data - persistentVolumeClaim: - claimName: minio - - name: certs - projected: - sources: - - secret: - name: minio-server-tls - items: - - key: tls.crt - path: public.crt - - key: tls.key - path: private.key diff --git a/hack/object-store/kustomization.yaml b/hack/object-store/kustomization.yaml new file mode 100644 index 000000000..d2f16340a --- /dev/null +++ b/hack/object-store/kustomization.yaml @@ -0,0 +1,8 @@ +resources: +- object-store-deployment.yaml +- object-store-pvc.yaml +- object-store-secret.yaml +- object-store-service.yaml +- object-store-certificate.yaml +- selfsigned-issuer.yaml +- s3-client.yaml diff --git a/hack/minio/minio-certificate.yaml b/hack/object-store/object-store-certificate.yaml similarity index 61% rename from hack/minio/minio-certificate.yaml rename to hack/object-store/object-store-certificate.yaml index 845fc7d17..bc3ca63de 100644 --- a/hack/minio/minio-certificate.yaml +++ b/hack/object-store/object-store-certificate.yaml @@ -1,12 +1,14 @@ apiVersion: cert-manager.io/v1 kind: Certificate metadata: - name: minio-server + name: object-store-server spec: - secretName: minio-server-tls - commonName: minio + secretName: object-store-tls + commonName: object-store dnsNames: - - minio + - object-store + - object-store.default + - object-store.default.svc duration: 2160h # 90d renewBefore: 360h # 15d diff --git a/hack/object-store/object-store-deployment.yaml b/hack/object-store/object-store-deployment.yaml new file mode 100644 index 000000000..d737105df --- /dev/null +++ b/hack/object-store/object-store-deployment.yaml @@ -0,0 +1,101 @@ +# RustFS, an S3-compatible object store, served over TLS with a certificate +# issued by cert-manager (see object-store-certificate.yaml). +apiVersion: apps/v1 +kind: Deployment +metadata: + name: object-store + labels: + app: object-store +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: object-store + template: + metadata: + labels: + app: object-store + spec: + # RustFS runs as a non-root user but the PVC is root-owned: the init + # container creates writable subdirectories for data and logs. + initContainers: + - name: init-permissions + image: docker.io/library/busybox:1.38.0 + command: + - sh + - -c + - mkdir -p /data/rustfs /logs/rustfs && chmod 0777 /data/rustfs /logs/rustfs + volumeMounts: + - mountPath: /data + name: data + - mountPath: /logs + name: logs + containers: + - name: object-store + image: docker.io/rustfs/rustfs:1.0.0-glibc + command: + - /usr/bin/rustfs + ports: + - containerPort: 9000 + name: api + env: + - name: RUSTFS_ADDRESS + value: ":9000" + - name: RUSTFS_VOLUMES + value: /data/rustfs + - name: RUSTFS_REGION + value: us-east-1 + - name: RUSTFS_CONSOLE_ENABLE + value: "false" + - name: RUSTFS_OBS_LOG_DIRECTORY + value: /logs/rustfs + # RustFS enables TLS when it finds rustfs_cert.pem and rustfs_key.pem + # in this directory. + - name: RUSTFS_TLS_PATH + value: /certs + - name: RUSTFS_ACCESS_KEY + valueFrom: + secretKeyRef: + name: object-store + key: ACCESS_KEY_ID + - name: RUSTFS_SECRET_KEY + valueFrom: + secretKeyRef: + name: object-store + key: ACCESS_SECRET_KEY + livenessProbe: + httpGet: + path: /health + port: 9000 + scheme: HTTPS + initialDelaySeconds: 30 + readinessProbe: + httpGet: + path: /health + port: 9000 + scheme: HTTPS + initialDelaySeconds: 30 + volumeMounts: + - mountPath: /data + name: data + - mountPath: /logs + name: logs + - mountPath: /certs + name: certs + readOnly: true + volumes: + - name: data + persistentVolumeClaim: + claimName: object-store + - name: logs + emptyDir: {} + - name: certs + secret: + secretName: object-store-tls + items: + - key: tls.crt + path: rustfs_cert.pem + - key: tls.key + path: rustfs_key.pem diff --git a/hack/minio/minio-pvc.yaml b/hack/object-store/object-store-pvc.yaml similarity index 88% rename from hack/minio/minio-pvc.yaml rename to hack/object-store/object-store-pvc.yaml index 6a402cfb1..25ad007c5 100644 --- a/hack/minio/minio-pvc.yaml +++ b/hack/object-store/object-store-pvc.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: - name: minio + name: object-store spec: accessModes: - ReadWriteOnce diff --git a/hack/minio/minio-secret.yaml b/hack/object-store/object-store-secret.yaml similarity index 89% rename from hack/minio/minio-secret.yaml rename to hack/object-store/object-store-secret.yaml index 0189d9a34..7ad8bbe5d 100644 --- a/hack/minio/minio-secret.yaml +++ b/hack/object-store/object-store-secret.yaml @@ -4,4 +4,4 @@ data: ACCESS_SECRET_KEY: b25nZWlxdWVpdG9oTDBxdWVlTG9oa2l1cjJxdWFpbmc= kind: Secret metadata: - name: minio + name: object-store diff --git a/hack/minio/minio-service.yaml b/hack/object-store/object-store-service.yaml similarity index 56% rename from hack/minio/minio-service.yaml rename to hack/object-store/object-store-service.yaml index 401c745cb..6446c0c22 100644 --- a/hack/minio/minio-service.yaml +++ b/hack/object-store/object-store-service.yaml @@ -1,16 +1,12 @@ apiVersion: v1 kind: Service metadata: - name: minio + name: object-store spec: selector: - app: minio + app: object-store ports: - protocol: TCP port: 9000 targetPort: 9000 name: api - - protocol: TCP - port: 36261 - targetPort: 36261 - name: webui diff --git a/hack/object-store/s3-client.yaml b/hack/object-store/s3-client.yaml new file mode 100644 index 000000000..4dcffcc31 --- /dev/null +++ b/hack/object-store/s3-client.yaml @@ -0,0 +1,50 @@ +# AWS CLI pod kept alive to inspect the object store, e.g. +# kubectl exec -ti s3-client -- aws s3 ls s3://backups/ +apiVersion: v1 +kind: Pod +metadata: + name: s3-client + labels: + app: s3-client +spec: + containers: + - name: s3-client + image: docker.io/amazon/aws-cli:2.36.49 + command: + - sleep + - infinity + env: + - name: AWS_ENDPOINT_URL + value: https://object-store:9000 + - name: AWS_DEFAULT_REGION + value: us-east-1 + - name: AWS_CA_BUNDLE + value: /certs/ca.crt + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: object-store + key: ACCESS_KEY_ID + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: object-store + key: ACCESS_SECRET_KEY + # The CRC-based default checksums introduced in AWS CLI 2.23 are not + # supported by every S3-compatible object store. + - name: AWS_REQUEST_CHECKSUM_CALCULATION + value: when_required + - name: AWS_RESPONSE_CHECKSUM_VALIDATION + value: when_required + volumeMounts: + - name: certs + mountPath: /certs/ca.crt + subPath: ca.crt + readOnly: true + volumes: + - name: certs + secret: + secretName: object-store-tls + items: + - key: tls.crt + path: ca.crt diff --git a/hack/minio/selfsigned-issuer.yaml b/hack/object-store/selfsigned-issuer.yaml similarity index 100% rename from hack/minio/selfsigned-issuer.yaml rename to hack/object-store/selfsigned-issuer.yaml diff --git a/scripts/cleanup.sh b/scripts/cleanup.sh index b885f285a..6474a0db0 100755 --- a/scripts/cleanup.sh +++ b/scripts/cleanup.sh @@ -24,4 +24,4 @@ cd "$(dirname "$0")/.." || exit kubectl delete clusters --all kubectl delete backups --all -kubectl exec -ti mc -- mc rm -r --force minio/backups \ No newline at end of file +kubectl exec -ti s3-client -- aws s3 rm --recursive s3://backups \ No newline at end of file diff --git a/scripts/minio-delete.sh b/scripts/minio-delete.sh deleted file mode 100755 index 479cdb78a..000000000 --- a/scripts/minio-delete.sh +++ /dev/null @@ -1,19 +0,0 @@ -## -## Copyright © contributors to CloudNativePG, established as -## CloudNativePG a Series of LF Projects, LLC. -## -## Licensed under the Apache License, Version 2.0 (the "License"); -## you may not use this file except in compliance with the License. -## You may obtain a copy of the License at -## -## http://www.apache.org/licenses/LICENSE-2.0 -## -## Unless required by applicable law or agreed to in writing, software -## distributed under the License is distributed on an "AS IS" BASIS, -## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -## See the License for the specific language governing permissions and -## limitations under the License. -## -## SPDX-License-Identifier: Apache-2.0 -## -kubectl exec -ti mc -- mc rm -r --force minio/backups diff --git a/hack/minio/minio-delete.sh b/scripts/object-store-delete.sh old mode 100644 new mode 100755 similarity index 91% rename from hack/minio/minio-delete.sh rename to scripts/object-store-delete.sh index 479cdb78a..f8c68ca69 --- a/hack/minio/minio-delete.sh +++ b/scripts/object-store-delete.sh @@ -16,4 +16,4 @@ ## ## SPDX-License-Identifier: Apache-2.0 ## -kubectl exec -ti mc -- mc rm -r --force minio/backups +kubectl exec -ti s3-client -- aws s3 rm --recursive s3://backups diff --git a/test/e2e/internal/objectstore/minio.go b/test/e2e/internal/objectstore/s3.go similarity index 51% rename from test/e2e/internal/objectstore/minio.go rename to test/e2e/internal/objectstore/s3.go index 654103b51..31a93425a 100644 --- a/test/e2e/internal/objectstore/minio.go +++ b/test/e2e/internal/objectstore/s3.go @@ -20,6 +20,7 @@ SPDX-License-Identifier: Apache-2.0 package objectstore import ( + "fmt" "net" barmanapi "github.com/cloudnative-pg/barman-cloud/pkg/api" @@ -34,17 +35,30 @@ import ( pluginBarmanCloudV1 "github.com/cloudnative-pg/plugin-barman-cloud/api/v1" ) -// NewMinioObjectStoreResources creates the resources required to create a Minio object store. -func NewMinioObjectStoreResources(namespace, name string) *Resources { +// The S3-compatible object store used by the e2e tests is RustFS. It runs as +// a non-root user and writes its data and logs to these subdirectories, which +// an init container creates and makes writable. +const ( + s3DataDir = "/data/rustfs" + s3LogDir = "/logs/rustfs" +) + +// NewS3ObjectStoreResources creates the resources required to run an +// S3-compatible object store. +func NewS3ObjectStoreResources(namespace, name string) *Resources { return &Resources{ - Deployment: newMinioDeployment(namespace, name), - Service: newMinioService(namespace, name), - PVC: newMinioPVC(namespace, name), - Secret: newMinioSecret(namespace, name), + Deployment: newS3Deployment(namespace, name), + Service: newS3Service(namespace, name), + PVC: newS3PVC(namespace, name), + Secret: newS3Secret(namespace, name), } } -func newMinioDeployment(namespace, name string) *appsv1.Deployment { +func newS3Deployment(namespace, name string) *appsv1.Deployment { + seccompProfile := &corev1.SeccompProfile{ + Type: corev1.SeccompProfileTypeRuntimeDefault, + } + return &appsv1.Deployment{ TypeMeta: metav1.TypeMeta{ Kind: "Deployment", @@ -68,13 +82,40 @@ func newMinioDeployment(namespace, name string) *appsv1.Deployment { }, }, Spec: corev1.PodSpec{ + // RustFS runs as a non-root user but the PVC is root-owned, and + // a non-root init container cannot chown it on OpenShift. + // Instead the init creates a subdirectory it owns and makes it + // world-writable, which works as root (kind, cloud) or as the + // SCC-assigned UID (OpenShift). + InitContainers: []corev1.Container{ + { + Name: "init-permissions", + // renovate: datasource=docker depName=busybox versioning=docker + // Version: 1.38.0 + Image: "docker.io/library/busybox@sha256:dc2d74b28e4cf8984fa52af1f39bc7c3d9c73760b41a74d629f5d11b1ab28616", + Command: []string{ + "sh", "-c", + fmt.Sprintf("mkdir -p %[1]s %[2]s && chmod 0777 %[1]s %[2]s", s3DataDir, s3LogDir), + }, + VolumeMounts: []corev1.VolumeMount{ + {Name: "data", MountPath: "/data"}, + {Name: "logs", MountPath: "/logs"}, + }, + SecurityContext: &corev1.SecurityContext{ + AllowPrivilegeEscalation: ptr.To(false), + SeccompProfile: seccompProfile, + }, + }, + }, Containers: []corev1.Container{ { Name: name, - // renovate: datasource=docker depName=minio/minio versioning=docker - // Version: RELEASE.2025-09-07T16-13-09Z - Image: "minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e", - Args: []string{"server", "/data"}, + // The glibc build is used because the musl default is + // noticeably slower (rustfs/rustfs#1662). + // renovate: datasource=docker depName=rustfs/rustfs versioning=docker + // Version: 1.0.0-glibc + Image: "docker.io/rustfs/rustfs@sha256:bffcab0c9d647aab0055d1c69d340b202d0909966b385932d4ead1aeb7602858", + Command: []string{"/usr/bin/rustfs"}, Ports: []corev1.ContainerPort{ { ContainerPort: 9000, @@ -82,8 +123,13 @@ func newMinioDeployment(namespace, name string) *appsv1.Deployment { }, }, Env: []corev1.EnvVar{ + {Name: "RUSTFS_ADDRESS", Value: ":9000"}, + {Name: "RUSTFS_VOLUMES", Value: s3DataDir}, + {Name: "RUSTFS_REGION", Value: "us-east-1"}, + {Name: "RUSTFS_CONSOLE_ENABLE", Value: "false"}, + {Name: "RUSTFS_OBS_LOG_DIRECTORY", Value: s3LogDir}, { - Name: "MINIO_ACCESS_KEY", + Name: "RUSTFS_ACCESS_KEY", ValueFrom: &corev1.EnvVarSource{ SecretKeyRef: &corev1.SecretKeySelector{ LocalObjectReference: corev1.LocalObjectReference{ @@ -94,7 +140,7 @@ func newMinioDeployment(namespace, name string) *appsv1.Deployment { }, }, { - Name: "MINIO_SECRET_KEY", + Name: "RUSTFS_SECRET_KEY", ValueFrom: &corev1.EnvVarSource{ SecretKeyRef: &corev1.SecretKeySelector{ LocalObjectReference: corev1.LocalObjectReference{ @@ -106,10 +152,16 @@ func newMinioDeployment(namespace, name string) *appsv1.Deployment { }, }, VolumeMounts: []corev1.VolumeMount{ - { - Name: "data", - MountPath: "/data", - }, + {Name: "data", MountPath: "/data"}, + {Name: "logs", MountPath: "/logs"}, + }, + LivenessProbe: newHealthProbe(30, 10), + // RustFS is up within a few seconds; poll early so + // each spec does not wait the liveness grace period. + ReadinessProbe: newHealthProbe(5, 5), + SecurityContext: &corev1.SecurityContext{ + AllowPrivilegeEscalation: ptr.To(false), + SeccompProfile: seccompProfile, }, }, }, @@ -122,6 +174,17 @@ func newMinioDeployment(namespace, name string) *appsv1.Deployment { }, }, }, + { + Name: "logs", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + }, + // No fsGroup/runAsUser: let OpenShift's restricted SCC assign + // the UID; elsewhere the server runs as its image default user. + SecurityContext: &corev1.PodSecurityContext{ + SeccompProfile: seccompProfile, }, }, }, @@ -129,7 +192,21 @@ func newMinioDeployment(namespace, name string) *appsv1.Deployment { } } -func newMinioService(namespace, name string) *corev1.Service { +// newHealthProbe returns a probe hitting the RustFS health endpoint. +func newHealthProbe(initialDelaySeconds, periodSeconds int32) *corev1.Probe { + return &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/health", + Port: intstr.FromInt32(9000), + }, + }, + InitialDelaySeconds: initialDelaySeconds, + PeriodSeconds: periodSeconds, + } +} + +func newS3Service(namespace, name string) *corev1.Service { return &corev1.Service{ TypeMeta: metav1.TypeMeta{ Kind: "Service", @@ -154,7 +231,7 @@ func newMinioService(namespace, name string) *corev1.Service { } } -func newMinioSecret(namespace, name string) *corev1.Secret { +func newS3Secret(namespace, name string) *corev1.Secret { return &corev1.Secret{ TypeMeta: metav1.TypeMeta{ Kind: "Secret", @@ -165,13 +242,13 @@ func newMinioSecret(namespace, name string) *corev1.Secret { Namespace: namespace, }, Data: map[string][]byte{ - "ACCESS_KEY_ID": []byte("minio"), - "ACCESS_SECRET_KEY": []byte("minio123"), + "ACCESS_KEY_ID": []byte("s3accesskey"), + "ACCESS_SECRET_KEY": []byte("s3secretkey123"), }, } } -func newMinioPVC(namespace, name string) *corev1.PersistentVolumeClaim { +func newS3PVC(namespace, name string) *corev1.PersistentVolumeClaim { return &corev1.PersistentVolumeClaim{ TypeMeta: metav1.TypeMeta{ Kind: "PersistentVolumeClaim", @@ -194,8 +271,9 @@ func newMinioPVC(namespace, name string) *corev1.PersistentVolumeClaim { } } -// NewMinioObjectStore creates a new Minio object store. -func NewMinioObjectStore(namespace, name, minioOSName string) *pluginBarmanCloudV1.ObjectStore { +// NewS3ObjectStore creates a new ObjectStore pointing at the S3-compatible +// object store created by NewS3ObjectStoreResources with the given name. +func NewS3ObjectStore(namespace, name, s3Name string) *pluginBarmanCloudV1.ObjectStore { return &pluginBarmanCloudV1.ObjectStore{ TypeMeta: metav1.TypeMeta{ Kind: "ObjectStore", @@ -211,19 +289,19 @@ func NewMinioObjectStore(namespace, name, minioOSName string) *pluginBarmanCloud AWS: &barmanapi.S3Credentials{ AccessKeyIDReference: &api.SecretKeySelector{ LocalObjectReference: api.LocalObjectReference{ - Name: minioOSName, + Name: s3Name, }, Key: "ACCESS_KEY_ID", }, SecretAccessKeyReference: &api.SecretKeySelector{ LocalObjectReference: api.LocalObjectReference{ - Name: minioOSName, + Name: s3Name, }, Key: "ACCESS_SECRET_KEY", }, }, }, - EndpointURL: "http://" + net.JoinHostPort(minioOSName, "9000"), + EndpointURL: "http://" + net.JoinHostPort(s3Name, "9000"), DestinationPath: "s3://backups/", }, }, diff --git a/test/e2e/internal/tests/backup/fixtures.go b/test/e2e/internal/tests/backup/fixtures.go index 9537921ff..559096071 100644 --- a/test/e2e/internal/tests/backup/fixtures.go +++ b/test/e2e/internal/tests/backup/fixtures.go @@ -35,7 +35,7 @@ import ( ) const ( - minio = "minio" + s3 = "s3" azurite = "azurite" gcs = "gcs" // Size of the PVCs for the object stores and the cluster instances. @@ -67,8 +67,8 @@ func (s s3BackupPluginBackupPluginRestore) createBackupRestoreTestResources( ) backupRestoreTestResources { result := backupRestoreTestResources{} - result.ObjectStoreResources = objectstore.NewMinioObjectStoreResources(namespace, minio) - result.ObjectStore = objectstore.NewMinioObjectStore(namespace, objectStoreName, minio) + result.ObjectStoreResources = objectstore.NewS3ObjectStoreResources(namespace, s3) + result.ObjectStore = objectstore.NewS3ObjectStore(namespace, objectStoreName, s3) result.SrcCluster = newSrcClusterWithPlugin(namespace) result.SrcBackup = newSrcPluginBackup(namespace) result.DstCluster = newDstClusterWithPlugin(namespace) @@ -84,8 +84,8 @@ func (s s3BackupPluginBackupInTreeRestore) createBackupRestoreTestResources( ) backupRestoreTestResources { result := backupRestoreTestResources{} - result.ObjectStoreResources = objectstore.NewMinioObjectStoreResources(namespace, minio) - result.ObjectStore = objectstore.NewMinioObjectStore(namespace, objectStoreName, minio) + result.ObjectStoreResources = objectstore.NewS3ObjectStoreResources(namespace, s3) + result.ObjectStore = objectstore.NewS3ObjectStore(namespace, objectStoreName, s3) result.SrcCluster = newSrcClusterWithPlugin(namespace) result.SrcBackup = newSrcPluginBackup(namespace) result.DstCluster = newDstClusterInTreeS3(namespace) @@ -101,8 +101,8 @@ func (s s3BackupPluginInTreeBackupPluginRestore) createBackupRestoreTestResource ) backupRestoreTestResources { result := backupRestoreTestResources{} - result.ObjectStoreResources = objectstore.NewMinioObjectStoreResources(namespace, minio) - result.ObjectStore = objectstore.NewMinioObjectStore(namespace, objectStoreName, minio) + result.ObjectStoreResources = objectstore.NewS3ObjectStoreResources(namespace, s3) + result.ObjectStore = objectstore.NewS3ObjectStore(namespace, objectStoreName, s3) result.SrcCluster = newSrcClusterInTreeS3(namespace) result.SrcBackup = newSrcInTreeBackup(namespace) result.DstCluster = newDstClusterWithPlugin(namespace) @@ -392,19 +392,19 @@ func newSrcClusterInTreeS3(namespace string) *cloudnativepgv1.Cluster { AWS: &barmanapi.S3Credentials{ AccessKeyIDReference: &api.SecretKeySelector{ LocalObjectReference: api.LocalObjectReference{ - Name: minio, + Name: s3, }, Key: "ACCESS_KEY_ID", }, SecretAccessKeyReference: &api.SecretKeySelector{ LocalObjectReference: api.LocalObjectReference{ - Name: minio, + Name: s3, }, Key: "ACCESS_SECRET_KEY", }, }, }, - EndpointURL: "http://" + net.JoinHostPort(minio, "9000"), + EndpointURL: "http://" + net.JoinHostPort(s3, "9000"), DestinationPath: "s3://backups/", }, }, @@ -454,19 +454,19 @@ func newDstClusterInTreeS3(namespace string) *cloudnativepgv1.Cluster { AWS: &barmanapi.S3Credentials{ AccessKeyIDReference: &api.SecretKeySelector{ LocalObjectReference: api.LocalObjectReference{ - Name: minio, + Name: s3, }, Key: "ACCESS_KEY_ID", }, SecretAccessKeyReference: &api.SecretKeySelector{ LocalObjectReference: api.LocalObjectReference{ - Name: minio, + Name: s3, }, Key: "ACCESS_SECRET_KEY", }, }, }, - EndpointURL: "http://" + net.JoinHostPort(minio, "9000"), + EndpointURL: "http://" + net.JoinHostPort(s3, "9000"), DestinationPath: "s3://backups/", }, }, diff --git a/test/e2e/internal/tests/credentialrotation/credential_rotation.go b/test/e2e/internal/tests/credentialrotation/credential_rotation.go index f5302591d..a91a576b9 100644 --- a/test/e2e/internal/tests/credentialrotation/credential_rotation.go +++ b/test/e2e/internal/tests/credentialrotation/credential_rotation.go @@ -44,8 +44,8 @@ import ( const ( clusterName = "source" objectStoreName = "source" - oldSecretName = "minio" - newSecretName = "minio-rotated" + oldSecretName = "s3" + newSecretName = "s3-rotated" ) var _ = Describe("Credential rotation", func() { @@ -66,11 +66,11 @@ var _ = Describe("Credential rotation", func() { It("should update the Role when the ObjectStore secret reference changes", func(ctx SpecContext) { By("starting the ObjectStore deployment") - resources := objectstore.NewMinioObjectStoreResources(namespace.Name, oldSecretName) + resources := objectstore.NewS3ObjectStoreResources(namespace.Name, oldSecretName) Expect(resources.Create(ctx, cl)).To(Succeed()) By("creating the ObjectStore") - store := objectstore.NewMinioObjectStore(namespace.Name, objectStoreName, oldSecretName) + store := objectstore.NewS3ObjectStore(namespace.Name, objectStoreName, oldSecretName) Expect(cl.Create(ctx, store)).To(Succeed()) By("creating the Cluster") @@ -104,8 +104,8 @@ var _ = Describe("Credential rotation", func() { Namespace: namespace.Name, }, Data: map[string][]byte{ - "ACCESS_KEY_ID": []byte("minio"), - "ACCESS_SECRET_KEY": []byte("minio123"), + "ACCESS_KEY_ID": []byte("s3accesskey"), + "ACCESS_SECRET_KEY": []byte("s3secretkey123"), }, } Expect(cl.Create(ctx, newSecret)).To(Succeed()) diff --git a/test/e2e/internal/tests/replicacluster/fixtures.go b/test/e2e/internal/tests/replicacluster/fixtures.go index b373aedb5..2f0ef4858 100644 --- a/test/e2e/internal/tests/replicacluster/fixtures.go +++ b/test/e2e/internal/tests/replicacluster/fixtures.go @@ -42,8 +42,8 @@ const ( replicaObjectStoreName = "replica" replicaClusterName = "replica" replicaBackupName = "replica" - minioSrc = "minio-src" - minioReplica = "minio-replica" + s3Src = "s3-src" + s3Replica = "s3-replica" gcsSrc = "fakegcs-src" azuriteSrc = "azurite-src" azuriteReplica = "azurite-replica" @@ -65,12 +65,12 @@ type s3ReplicaClusterFactory struct{} func (f s3ReplicaClusterFactory) createReplicaClusterTestResources(namespace string) replicaClusterTestResources { result := replicaClusterTestResources{} - result.SrcObjectStoreResources = objectstore.NewMinioObjectStoreResources(namespace, minioSrc) - result.SrcObjectStore = objectstore.NewMinioObjectStore(namespace, srcObjectStoreName, minioSrc) + result.SrcObjectStoreResources = objectstore.NewS3ObjectStoreResources(namespace, s3Src) + result.SrcObjectStore = objectstore.NewS3ObjectStore(namespace, srcObjectStoreName, s3Src) result.SrcCluster = newSrcCluster(namespace) result.SrcBackup = newSrcBackup(namespace) - result.ReplicaObjectStoreResources = objectstore.NewMinioObjectStoreResources(namespace, minioReplica) - result.ReplicaObjectStore = objectstore.NewMinioObjectStore(namespace, replicaObjectStoreName, minioReplica) + result.ReplicaObjectStoreResources = objectstore.NewS3ObjectStoreResources(namespace, s3Replica) + result.ReplicaObjectStore = objectstore.NewS3ObjectStore(namespace, replicaObjectStoreName, s3Replica) result.ReplicaCluster = newReplicaCluster(namespace) result.ReplicaBackup = newReplicaBackup(namespace) diff --git a/test/e2e/internal/tests/replicacluster/replica_cluster.go b/test/e2e/internal/tests/replicacluster/replica_cluster.go index e4cf5985e..ee7934a23 100644 --- a/test/e2e/internal/tests/replicacluster/replica_cluster.go +++ b/test/e2e/internal/tests/replicacluster/replica_cluster.go @@ -266,7 +266,7 @@ var _ = Describe("Replica cluster", func() { }).Within(2 * time.Minute).WithPolling(5 * time.Second).Should(Succeed()) }, Entry( - "with MinIO", + "with S3", s3ReplicaClusterFactory{}, ), Entry( diff --git a/test/e2e/internal/tests/walrestore/fixtures.go b/test/e2e/internal/tests/walrestore/fixtures.go index c0634ad8b..990718492 100644 --- a/test/e2e/internal/tests/walrestore/fixtures.go +++ b/test/e2e/internal/tests/walrestore/fixtures.go @@ -32,7 +32,7 @@ import ( ) const ( - minioName = "minio" + s3Name = "s3" objectStoreName = "source" clusterName = "source" s3ClientName = "s3-client" @@ -43,16 +43,16 @@ const ( walMaxParallel = 3 ) -// newObjectStoreResources returns the minio server Deployment/Service/Secret/PVC. +// newObjectStoreResources returns the S3 object store Deployment/Service/Secret/PVC. func newObjectStoreResources(namespace string) *objectstore.Resources { - return objectstore.NewMinioObjectStoreResources(namespace, minioName) + return objectstore.NewS3ObjectStoreResources(namespace, s3Name) } -// newObjectStore returns a minio-backed ObjectStore configured with the WAL +// newObjectStore returns an S3-backed ObjectStore configured with the WAL // prefetch parallelism (maxParallel) under test. Archiving with gzip makes the // archived segments carry the ".gz" suffix that forged segments are copied from. func newObjectStore(namespace string) *pluginBarmanCloudV1.ObjectStore { - store := objectstore.NewMinioObjectStore(namespace, objectStoreName, minioName) + store := objectstore.NewS3ObjectStore(namespace, objectStoreName, s3Name) store.Spec.Configuration.Wal = &barmanapi.WalBackupConfiguration{ MaxParallel: walMaxParallel, Compression: barmanapi.CompressionTypeGzip, @@ -98,7 +98,7 @@ func newCluster(namespace string) *cloudnativepgv1.Cluster { } // newS3ClientDeployment returns a deployment running the AWS CLI configured to -// talk to the in-namespace minio service. The test execs `aws s3` commands in +// talk to the in-namespace S3 service. The test execs `aws s3` commands in // it to forge WAL segments on the object store and to check their presence. func newS3ClientDeployment(namespace string) *appsv1.Deployment { labels := map[string]string{"app": s3ClientName} @@ -127,13 +127,13 @@ func newS3ClientDeployment(namespace string) *appsv1.Deployment { Env: []corev1.EnvVar{ { Name: "AWS_ENDPOINT_URL", - Value: "http://" + minioName + ":9000", + Value: "http://" + s3Name + ":9000", }, { Name: "AWS_ACCESS_KEY_ID", ValueFrom: &corev1.EnvVarSource{ SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{Name: minioName}, + LocalObjectReference: corev1.LocalObjectReference{Name: s3Name}, Key: "ACCESS_KEY_ID", }, }, @@ -142,7 +142,7 @@ func newS3ClientDeployment(namespace string) *appsv1.Deployment { Name: "AWS_SECRET_ACCESS_KEY", ValueFrom: &corev1.EnvVarSource{ SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{Name: minioName}, + LocalObjectReference: corev1.LocalObjectReference{Name: s3Name}, Key: "ACCESS_SECRET_KEY", }, }, @@ -153,7 +153,7 @@ func newS3ClientDeployment(namespace string) *appsv1.Deployment { }, // The CRC-based default checksums introduced in AWS // CLI 2.23 are not supported by every S3-compatible - // object store, minio included. + // object store. { Name: "AWS_REQUEST_CHECKSUM_CALCULATION", Value: "when_required", diff --git a/test/e2e/internal/tests/walrestore/walrestore.go b/test/e2e/internal/tests/walrestore/walrestore.go index 287748ff0..bc055f2e3 100644 --- a/test/e2e/internal/tests/walrestore/walrestore.go +++ b/test/e2e/internal/tests/walrestore/walrestore.go @@ -64,7 +64,7 @@ const ( // walLogDir is the WALs subdirectory (timeline + log id) the forged segments // live under; a freshly bootstrapped, idle cluster stays within it. walLogDir = "0000000100000000" - // bucket is the destination bucket of the minio ObjectStore. + // bucket is the destination bucket of the S3 ObjectStore. bucket = "backups" ) diff --git a/web/docs/intro.md b/web/docs/intro.md index 9781d0d66..81250c6c7 100644 --- a/web/docs/intro.md +++ b/web/docs/intro.md @@ -77,7 +77,7 @@ The plugin works with all storage backends supported by `barman-cloud`, includin In addition, the following S3-compatible and simulator solutions have been tested and verified: -- [MinIO](https://min.io/) – An S3-compatible storage solution +- [RustFS](https://rustfs.com/) – An S3-compatible storage solution - [Azurite](https://github.com/Azure/Azurite) – A simulator for Azure Blob Storage - [fake-gcs-server](https://github.com/fsouza/fake-gcs-server) – A simulator for Google Cloud Storage diff --git a/web/docs/migration.md b/web/docs/migration.md index 2c99ada6d..de6b71e8a 100644 --- a/web/docs/migration.md +++ b/web/docs/migration.md @@ -64,13 +64,13 @@ spec: backup: barmanObjectStore: destinationPath: s3://backups/ - endpointURL: http://minio-eu:9000 + endpointURL: http://object-store-eu:9000 s3Credentials: accessKeyId: - name: minio-eu + name: s3-eu key: ACCESS_KEY_ID secretAccessKey: - name: minio-eu + name: s3-eu key: ACCESS_SECRET_KEY wal: compression: gzip @@ -83,17 +83,17 @@ plugin: apiVersion: barmancloud.cnpg.io/v1 kind: ObjectStore metadata: - name: minio-eu + name: s3-eu spec: configuration: destinationPath: s3://backups/ - endpointURL: http://minio-eu:9000 + endpointURL: http://object-store-eu:9000 s3Credentials: accessKeyId: - name: minio-eu + name: s3-eu key: ACCESS_KEY_ID secretAccessKey: - name: minio-eu + name: s3-eu key: ACCESS_SECRET_KEY wal: compression: gzip @@ -128,7 +128,7 @@ previous `backup` section: - name: barman-cloud.cloudnative-pg.io isWALArchiver: true parameters: - barmanObjectName: minio-eu + barmanObjectName: s3-eu ``` --- @@ -205,14 +205,14 @@ spec: - name: pg-eu barmanObjectStore: destinationPath: s3://backups/ - endpointURL: http://minio-eu:9000 + endpointURL: http://object-store-eu:9000 serverName: pg-eu s3Credentials: accessKeyId: - name: minio-eu + name: s3-eu key: ACCESS_KEY_ID secretAccessKey: - name: minio-eu + name: s3-eu key: ACCESS_SECRET_KEY wal: compression: gzip @@ -224,17 +224,17 @@ Create the `ObjectStore` resource for the external cluster: apiVersion: barmancloud.cnpg.io/v1 kind: ObjectStore metadata: - name: minio-eu + name: s3-eu spec: configuration: destinationPath: s3://backups/ - endpointURL: http://minio-eu:9000 + endpointURL: http://object-store-eu:9000 s3Credentials: accessKeyId: - name: minio-eu + name: s3-eu key: ACCESS_KEY_ID secretAccessKey: - name: minio-eu + name: s3-eu key: ACCESS_SECRET_KEY wal: compression: gzip @@ -254,7 +254,7 @@ spec: plugin: name: barman-cloud.cloudnative-pg.io parameters: - barmanObjectName: minio-eu + barmanObjectName: s3-eu serverName: pg-eu ``` diff --git a/web/docs/object_stores.md b/web/docs/object_stores.md index 11b1ff8cd..2c0277bb1 100644 --- a/web/docs/object_stores.md +++ b/web/docs/object_stores.md @@ -131,7 +131,7 @@ overhead. ### S3-Compatible Storage Providers -You can use S3-compatible services like **MinIO**, **Linode (Akamai) Object Storage**, +You can use S3-compatible services like **RustFS**, **Linode (Akamai) Object Storage**, or **DigitalOcean Spaces** by specifying a custom `endpointURL`. Example with Linode (Akamai) Object Storage (`us-east1`): @@ -196,7 +196,7 @@ spec: ### Using Object Storage with a Private CA -For object storage services (e.g., MinIO) that use HTTPS with certificates +For object storage services (e.g., RustFS) that use HTTPS with certificates signed by a private CA, set the `endpointCA` field in the `ObjectStore` definition. Unless you already have it, create a Kubernetes `Secret` with the CA bundle: @@ -211,7 +211,7 @@ Then reference it: apiVersion: barmancloud.cnpg.io/v1 kind: ObjectStore metadata: - name: minio-store + name: s3-store spec: configuration: endpointURL: @@ -427,71 +427,54 @@ write permissions to the bucket. --- -## MinIO Object Store +## RustFS Object Store -In order to use the Tenant resource you first need to deploy the -[MinIO operator](https://docs.min.io/community/minio-object-store/operations/deployments/installation.html). -For the latest documentation of MinIO, please refer to the -[MinIO official documentation](https://docs.min.io/community/minio-object-store/). +[RustFS](https://rustfs.com/) is an open source, S3-compatible object store +that can run inside your Kubernetes cluster. Please refer to the +[RustFS documentation](https://docs.rustfs.com/) for deployment options; a +minimal `Deployment` with a `PersistentVolumeClaim` and a `Service` exposing +port 9000 is enough for testing purposes. RustFS serves plain HTTP unless it +finds a certificate and key under the directory pointed to by +`RUSTFS_TLS_PATH`, in which case it serves HTTPS and you must reference the +CA through `endpointCA` (see +[Using Object Storage with a Private CA](#using-object-storage-with-a-private-ca)). -MinIO Object Store's API is compatible with S3, and the default configuration of the Tenant -will create these services: -- `-console` on port 9090 (with autocert) or 9443 (without autocert) -- `-hl` on port 9000 -Where `` is the `metadata.name` you assigned to your Tenant resource. - -:::note -The `-console` service will only be available if you have enabled the -[MinIO Console](https://docs.min.io/community/minio-object-store/administration/minio-console.html). - -For example, the following Tenant: -```yml -apiVersion: minio.min.io/v2 -kind: Tenant -metadata: - name: cnpg-backups -spec: - [...] -``` -would have services called `cnpg-backups-console` and `cnpg-backups-hl` respectively. - -The `console` service is for managing the tenant, while the `hl` service exposes the S3 -compatible API. If your tenant is configured with `requestAutoCert` you will communicate -to these services over HTTPS, if not you will use HTTP. - -For authentication you can use your username and password, or create an access key. -Whichever method you choose, it has to be stored as a secret. +RustFS reads its root credentials from the `RUSTFS_ACCESS_KEY` and +`RUSTFS_SECRET_KEY` environment variables. Store the same values in a +`Secret` for the plugin: ```sh -kubectl create secret generic minio-creds \ - --from-literal=MINIO_ACCESS_KEY= \ - --from-literal=MINIO_SECRET_KEY= +kubectl create secret generic s3-creds \ + --from-literal=ACCESS_KEY_ID= \ + --from-literal=ACCESS_SECRET_KEY= ``` -Finally, create the Barman ObjectStore: +Finally, create the Barman `ObjectStore`, pointing `endpointURL` at the +RustFS `Service`: ```yaml apiVersion: barmancloud.cnpg.io/v1 kind: ObjectStore metadata: - name: minio-store + name: s3-store spec: configuration: destinationPath: s3://BUCKET_NAME/ - endpointURL: http://-hl:9000 + endpointURL: http://:9000 s3Credentials: accessKeyId: - name: minio-creds - key: MINIO_ACCESS_KEY + name: s3-creds + key: ACCESS_KEY_ID secretAccessKey: - name: minio-creds - key: MINIO_SECRET_KEY + name: s3-creds + key: ACCESS_SECRET_KEY [...] ``` +The bucket is created on first use if it does not exist. + :::important Verify on `s3://BUCKET_NAME/` the presence of archived WAL files before proceeding with a backup. ::: - --- diff --git a/web/docs/usage.md b/web/docs/usage.md index 6406d38de..a29197c6a 100644 --- a/web/docs/usage.md +++ b/web/docs/usage.md @@ -17,28 +17,28 @@ Cloud Plugin involves just a few steps: From that moment, you’ll be able to issue on-demand backups or define a backup schedule, as well as rely on the object store for recovery operations. -The rest of this page details each step, using MinIO as object store provider. +The rest of this page details each step, using RustFS as object store provider. ## Defining the `ObjectStore` An `ObjectStore` resource must be created for each object store used in your -PostgreSQL architecture. Here's an example configuration using MinIO: +PostgreSQL architecture. Here's an example configuration using RustFS: ```yaml apiVersion: barmancloud.cnpg.io/v1 kind: ObjectStore metadata: - name: minio-store + name: s3-store spec: configuration: destinationPath: s3://backups/ - endpointURL: http://minio:9000 + endpointURL: http://object-store:9000 s3Credentials: accessKeyId: - name: minio + name: object-store key: ACCESS_KEY_ID secretAccessKey: - name: minio + name: object-store key: ACCESS_SECRET_KEY wal: compression: gzip @@ -72,7 +72,7 @@ spec: - name: barman-cloud.cloudnative-pg.io isWALArchiver: true parameters: - barmanObjectName: minio-store + barmanObjectName: s3-store storage: size: 1Gi ``` @@ -147,7 +147,7 @@ spec: plugin: name: barman-cloud.cloudnative-pg.io parameters: - barmanObjectName: minio-store + barmanObjectName: s3-store serverName: cluster-example storage: size: 1Gi @@ -176,14 +176,14 @@ spec: isWALArchiver: true parameters: # Backup Object Store (push, read-write) - barmanObjectName: minio-store-bis + barmanObjectName: s3-store-bis externalClusters: - name: source plugin: name: barman-cloud.cloudnative-pg.io parameters: # Recovery Object Store (pull, read-only) - barmanObjectName: minio-store + barmanObjectName: s3-store serverName: cluster-example storage: size: 1Gi @@ -215,7 +215,7 @@ spec: - name: barman-cloud.cloudnative-pg.io isWALArchiver: true parameters: - barmanObjectName: minio-store-a + barmanObjectName: s3-store-a replica: self: cluster-dc-a @@ -227,13 +227,13 @@ spec: plugin: name: barman-cloud.cloudnative-pg.io parameters: - barmanObjectName: minio-store-a + barmanObjectName: s3-store-a - name: cluster-dc-b plugin: name: barman-cloud.cloudnative-pg.io parameters: - barmanObjectName: minio-store-b + barmanObjectName: s3-store-b ``` ## Configuring the plugin instance sidecar @@ -262,7 +262,7 @@ and could generate a rollout of the `Cluster`. apiVersion: barmancloud.cnpg.io/v1 kind: ObjectStore metadata: - name: minio-store + name: s3-store spec: configuration: # [...] From 89ab62dae1e85dc26c07ca50b631ab7d2c4c7db1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Fei?= Date: Tue, 22 Sep 2026 17:20:44 +0200 Subject: [PATCH 02/11] test: use provider-neutral terms in unit-tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Niccolò Fei --- internal/cnpgi/operator/config/config_test.go | 6 +++--- internal/cnpgi/operator/lifecycle_test.go | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/internal/cnpgi/operator/config/config_test.go b/internal/cnpgi/operator/config/config_test.go index 3697875cb..aaec9fa0b 100644 --- a/internal/cnpgi/operator/config/config_test.go +++ b/internal/cnpgi/operator/config/config_test.go @@ -71,7 +71,7 @@ var _ = Describe("NewFromCluster", func() { PluginConfiguration: &cnpgv1.PluginConfiguration{ Name: metadata.PluginName, Parameters: map[string]string{ - "barmanObjectName": "minio-store", + "barmanObjectName": "object-store", "serverName": "cluster-example", }, }, @@ -86,7 +86,7 @@ var _ = Describe("NewFromCluster", func() { // while the backup/archive and recovery object stores remain empty: this is the // distinguishing trait of a pg_basebackup replica cluster (a recovery-bootstrapped // replica would also populate RecoveryBarmanObjectName). - Expect(cfg.ReplicaSourceBarmanObjectName).To(Equal("minio-store")) + Expect(cfg.ReplicaSourceBarmanObjectName).To(Equal("object-store")) Expect(cfg.ReplicaSourceServerName).To(Equal("cluster-example")) Expect(cfg.BarmanObjectName).To(BeEmpty()) Expect(cfg.RecoveryBarmanObjectName).To(BeEmpty()) @@ -111,7 +111,7 @@ var _ = Describe("NewFromCluster", func() { Name: "source", PluginConfiguration: &cnpgv1.PluginConfiguration{ Name: "some-other-plugin.cloudnative-pg.io", - Parameters: map[string]string{"barmanObjectName": "minio-store"}, + Parameters: map[string]string{"barmanObjectName": "object-store"}, }, }, }, diff --git a/internal/cnpgi/operator/lifecycle_test.go b/internal/cnpgi/operator/lifecycle_test.go index 5b3264ee3..76d658ad5 100644 --- a/internal/cnpgi/operator/lifecycle_test.go +++ b/internal/cnpgi/operator/lifecycle_test.go @@ -75,7 +75,7 @@ var _ = Describe("LifecycleImplementation", func() { BeforeEach(func() { pluginConfiguration = &config.PluginConfiguration{ - BarmanObjectName: "minio-store-dest", + BarmanObjectName: "object-store-dest", } cluster = &cnpgv1.Cluster{ Spec: cnpgv1.ClusterSpec{ @@ -90,7 +90,7 @@ var _ = Describe("LifecycleImplementation", func() { PluginConfiguration: &cnpgv1.PluginConfiguration{ Name: "barman-cloud.cloudnative-pg.io", Parameters: map[string]string{ - "barmanObjectName": "minio-store-source", + "barmanObjectName": "object-store-source", }, }, }, @@ -99,7 +99,7 @@ var _ = Describe("LifecycleImplementation", func() { { Name: "barman-cloud.cloudnative-pg.io", Parameters: map[string]string{ - "barmanObjectName": "minio-store-dest", + "barmanObjectName": "object-store-dest", }, }, }, @@ -244,7 +244,7 @@ var _ = Describe("LifecycleImplementation", func() { It("injects the sidecar for a recovery-only cluster", func(ctx SpecContext) { recoveryOnlyConfig := &config.PluginConfiguration{ - RecoveryBarmanObjectName: "minio-store-recovery", + RecoveryBarmanObjectName: "object-store-recovery", } pod := &corev1.Pod{ TypeMeta: podTypeMeta, @@ -268,7 +268,7 @@ var _ = Describe("LifecycleImplementation", func() { It("does not inject the sidecar for a recovery-only cluster that has "+ "already completed its initial bootstrap", func(ctx SpecContext) { recoveryOnlyConfig := &config.PluginConfiguration{ - RecoveryBarmanObjectName: "minio-store-recovery", + RecoveryBarmanObjectName: "object-store-recovery", } cluster.Status.CurrentPrimary = "test-pod" pod := &corev1.Pod{ From ccfb565766167feb95728bd7239225cbfc292219 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Fei?= Date: Wed, 23 Sep 2026 15:38:10 +0200 Subject: [PATCH 03/11] fix: add missing securityContext for object-store deployement MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Niccolò Fei --- hack/object-store/object-store-deployment.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/hack/object-store/object-store-deployment.yaml b/hack/object-store/object-store-deployment.yaml index d737105df..1c14512bb 100644 --- a/hack/object-store/object-store-deployment.yaml +++ b/hack/object-store/object-store-deployment.yaml @@ -32,6 +32,10 @@ spec: name: data - mountPath: /logs name: logs + securityContext: + allowPrivilegeEscalation: false + seccompProfile: + type: RuntimeDefault containers: - name: object-store image: docker.io/rustfs/rustfs:1.0.0-glibc @@ -65,6 +69,10 @@ spec: secretKeyRef: name: object-store key: ACCESS_SECRET_KEY + securityContext: + allowPrivilegeEscalation: false + seccompProfile: + type: RuntimeDefault livenessProbe: httpGet: path: /health @@ -85,6 +93,9 @@ spec: - mountPath: /certs name: certs readOnly: true + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - name: data persistentVolumeClaim: From f44e92e839464deb0043abf23982956164bbf354 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Fei?= Date: Wed, 23 Sep 2026 15:38:57 +0200 Subject: [PATCH 04/11] fix: turn s3-client into a deployment and add missing securityContext MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Niccolò Fei --- hack/object-store/s3-client.yaml | 106 ++++++++++++++++++------------- scripts/cleanup.sh | 2 +- scripts/object-store-delete.sh | 2 +- 3 files changed, 63 insertions(+), 47 deletions(-) diff --git a/hack/object-store/s3-client.yaml b/hack/object-store/s3-client.yaml index 4dcffcc31..cecfdb382 100644 --- a/hack/object-store/s3-client.yaml +++ b/hack/object-store/s3-client.yaml @@ -1,50 +1,66 @@ -# AWS CLI pod kept alive to inspect the object store, e.g. -# kubectl exec -ti s3-client -- aws s3 ls s3://backups/ -apiVersion: v1 -kind: Pod +# AWS CLI client kept alive to inspect the object store, e.g. +# kubectl exec -ti deploy/s3-client -- aws s3 ls s3://backups/ +apiVersion: apps/v1 +kind: Deployment metadata: name: s3-client labels: app: s3-client spec: - containers: - - name: s3-client - image: docker.io/amazon/aws-cli:2.36.49 - command: - - sleep - - infinity - env: - - name: AWS_ENDPOINT_URL - value: https://object-store:9000 - - name: AWS_DEFAULT_REGION - value: us-east-1 - - name: AWS_CA_BUNDLE - value: /certs/ca.crt - - name: AWS_ACCESS_KEY_ID - valueFrom: - secretKeyRef: - name: object-store - key: ACCESS_KEY_ID - - name: AWS_SECRET_ACCESS_KEY - valueFrom: - secretKeyRef: - name: object-store - key: ACCESS_SECRET_KEY - # The CRC-based default checksums introduced in AWS CLI 2.23 are not - # supported by every S3-compatible object store. - - name: AWS_REQUEST_CHECKSUM_CALCULATION - value: when_required - - name: AWS_RESPONSE_CHECKSUM_VALIDATION - value: when_required - volumeMounts: - - name: certs - mountPath: /certs/ca.crt - subPath: ca.crt - readOnly: true - volumes: - - name: certs - secret: - secretName: object-store-tls - items: - - key: tls.crt - path: ca.crt + replicas: 1 + selector: + matchLabels: + app: s3-client + template: + metadata: + labels: + app: s3-client + spec: + containers: + - name: s3-client + image: docker.io/amazon/aws-cli:2.36.49 + command: + - sleep + - infinity + env: + - name: AWS_ENDPOINT_URL + value: https://object-store:9000 + - name: AWS_DEFAULT_REGION + value: us-east-1 + - name: AWS_CA_BUNDLE + value: /certs/ca.crt + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: object-store + key: ACCESS_KEY_ID + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: object-store + key: ACCESS_SECRET_KEY + # The CRC-based default checksums introduced in AWS CLI 2.23 are not + # supported by every S3-compatible object store. + - name: AWS_REQUEST_CHECKSUM_CALCULATION + value: when_required + - name: AWS_RESPONSE_CHECKSUM_VALIDATION + value: when_required + volumeMounts: + - name: certs + mountPath: /certs/ca.crt + subPath: ca.crt + readOnly: true + securityContext: + allowPrivilegeEscalation: false + seccompProfile: + type: RuntimeDefault + securityContext: + seccompProfile: + type: RuntimeDefault + volumes: + - name: certs + secret: + secretName: object-store-tls + items: + - key: tls.crt + path: ca.crt diff --git a/scripts/cleanup.sh b/scripts/cleanup.sh index 6474a0db0..73cdf9215 100755 --- a/scripts/cleanup.sh +++ b/scripts/cleanup.sh @@ -24,4 +24,4 @@ cd "$(dirname "$0")/.." || exit kubectl delete clusters --all kubectl delete backups --all -kubectl exec -ti s3-client -- aws s3 rm --recursive s3://backups \ No newline at end of file +kubectl exec -ti deploy/s3-client -- aws s3 rm --recursive s3://backups diff --git a/scripts/object-store-delete.sh b/scripts/object-store-delete.sh index f8c68ca69..a92e0a9f1 100755 --- a/scripts/object-store-delete.sh +++ b/scripts/object-store-delete.sh @@ -16,4 +16,4 @@ ## ## SPDX-License-Identifier: Apache-2.0 ## -kubectl exec -ti s3-client -- aws s3 rm --recursive s3://backups +kubectl exec -ti deploy/s3-client -- aws s3 rm --recursive s3://backups From 0f18ca6e2357640b7cca5051137053ae0066bfa9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Fei?= Date: Wed, 23 Sep 2026 15:40:12 +0200 Subject: [PATCH 05/11] fix: add missing shebang in object-store-delete.sh script MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Niccolò Fei --- scripts/object-store-delete.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/object-store-delete.sh b/scripts/object-store-delete.sh index a92e0a9f1..9415b3e80 100755 --- a/scripts/object-store-delete.sh +++ b/scripts/object-store-delete.sh @@ -1,3 +1,4 @@ +#!/usr/bin/env bash ## ## Copyright © contributors to CloudNativePG, established as ## CloudNativePG a Series of LF Projects, LLC. From 25f4211eda6bd0ab8a60945c0708827c965f24b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Fei?= Date: Wed, 23 Sep 2026 15:55:03 +0200 Subject: [PATCH 06/11] fix: set strategy.type to Recreate in newS3Deployment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mimic the definition of object-store-deployment.yaml. The PVC is ReadWriteOnce: a rolling update would try to schedule the new pod while the old one still holds it. Signed-off-by: Niccolò Fei --- test/e2e/internal/objectstore/s3.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/e2e/internal/objectstore/s3.go b/test/e2e/internal/objectstore/s3.go index 31a93425a..571160cf2 100644 --- a/test/e2e/internal/objectstore/s3.go +++ b/test/e2e/internal/objectstore/s3.go @@ -70,6 +70,9 @@ func newS3Deployment(namespace, name string) *appsv1.Deployment { }, Spec: appsv1.DeploymentSpec{ Replicas: ptr.To(int32(1)), + Strategy: appsv1.DeploymentStrategy{ + Type: appsv1.RecreateDeploymentStrategyType, + }, Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "app": name, From 6d5b2e470984c3c2be1b484a974f76cad83344c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Fei?= Date: Wed, 23 Sep 2026 15:56:37 +0200 Subject: [PATCH 07/11] fix: add missing SecurityContext at pod level inside newS3ClientDeployment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Niccolò Fei --- test/e2e/internal/tests/walrestore/fixtures.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/test/e2e/internal/tests/walrestore/fixtures.go b/test/e2e/internal/tests/walrestore/fixtures.go index 990718492..db06ed3ae 100644 --- a/test/e2e/internal/tests/walrestore/fixtures.go +++ b/test/e2e/internal/tests/walrestore/fixtures.go @@ -101,6 +101,9 @@ func newCluster(namespace string) *cloudnativepgv1.Cluster { // talk to the in-namespace S3 service. The test execs `aws s3` commands in // it to forge WAL segments on the object store and to check their presence. func newS3ClientDeployment(namespace string) *appsv1.Deployment { + seccompProfile := &corev1.SeccompProfile{ + Type: corev1.SeccompProfileTypeRuntimeDefault, + } labels := map[string]string{"app": s3ClientName} return &appsv1.Deployment{ TypeMeta: metav1.TypeMeta{ @@ -165,12 +168,13 @@ func newS3ClientDeployment(namespace string) *appsv1.Deployment { }, SecurityContext: &corev1.SecurityContext{ AllowPrivilegeEscalation: ptr.To(false), - SeccompProfile: &corev1.SeccompProfile{ - Type: corev1.SeccompProfileTypeRuntimeDefault, - }, + SeccompProfile: seccompProfile, }, }, }, + SecurityContext: &corev1.PodSecurityContext{ + SeccompProfile: seccompProfile, + }, }, }, }, From 79d3133186b943e3dd5d8a6ca0a91ca7576e4bea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Fei?= Date: Wed, 23 Sep 2026 16:59:52 +0200 Subject: [PATCH 08/11] chore: add renovate for hack YAMLs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Niccolò Fei --- .../object-store/object-store-deployment.yaml | 8 +++++-- hack/object-store/s3-client.yaml | 4 +++- renovate.json5 | 23 +++++++++++++++++++ test/e2e/internal/objectstore/s3.go | 2 +- 4 files changed, 33 insertions(+), 4 deletions(-) diff --git a/hack/object-store/object-store-deployment.yaml b/hack/object-store/object-store-deployment.yaml index 1c14512bb..a165d7c20 100644 --- a/hack/object-store/object-store-deployment.yaml +++ b/hack/object-store/object-store-deployment.yaml @@ -22,7 +22,9 @@ spec: # container creates writable subdirectories for data and logs. initContainers: - name: init-permissions - image: docker.io/library/busybox:1.38.0 + # renovate: datasource=docker depName=busybox versioning=docker + # Version: 1.38.0 + image: docker.io/library/busybox@sha256:fd7dc98638c8e305f4dc34e979f1c0fdfdcaeb0fbf8fcff77ae834b6da3d7e6e command: - sh - -c @@ -38,7 +40,9 @@ spec: type: RuntimeDefault containers: - name: object-store - image: docker.io/rustfs/rustfs:1.0.0-glibc + # renovate: datasource=docker depName=rustfs/rustfs versioning=docker + # Version: 1.0.0-glibc + image: docker.io/rustfs/rustfs@sha256:bffcab0c9d647aab0055d1c69d340b202d0909966b385932d4ead1aeb7602858 command: - /usr/bin/rustfs ports: diff --git a/hack/object-store/s3-client.yaml b/hack/object-store/s3-client.yaml index cecfdb382..55474b9a0 100644 --- a/hack/object-store/s3-client.yaml +++ b/hack/object-store/s3-client.yaml @@ -18,7 +18,9 @@ spec: spec: containers: - name: s3-client - image: docker.io/amazon/aws-cli:2.36.49 + # renovate: datasource=docker depName=amazon/aws-cli versioning=docker + # Version: 2.37.0 + image: docker.io/amazon/aws-cli@sha256:337494c2047176fe9abcf45a5d1eaf1c2c62cae40953284fb1143b5c6170f065 command: - sleep - infinity diff --git a/renovate.json5 b/renovate.json5 index 132ff192a..7d58f7cf5 100644 --- a/renovate.json5 +++ b/renovate.json5 @@ -67,6 +67,15 @@ versioningTemplate: 'regex:(?\\d+)\\.(?\\d+)', extractVersionTemplate: '^(?v\\d+\\.\\d+)\\.\\d+$', }, + { + customType: 'regex', + managerFilePatterns: [ + '/(^|/)hack/.+\\.yaml$/', + ], + matchStrings: [ + '#\\s*renovate:\\s*datasource=(?[a-z-.]+?)\\s+depName=(?[^\\s]+?)(?:\\s+versioning=(?[^\\s]+?))?\\s*\\n\\s*#\\s*Version:\\s*(?[^\\s]+?)\\s*\\n\\s*image:\\s*[^@\\s]+@(?sha256:[a-f0-9]+)', + ], + }, ], 'pip-compile': { managerFilePatterns: [ @@ -80,6 +89,20 @@ enabled: false, }, packageRules: [ + { + // RustFS's default image is Alpine/musl, slower than glibc + // (rustfs/rustfs#1662). A bare `-glibc` suffix isn't a safe pin: + // semver compares prerelease identifiers lexically, and "rc" > + // "glibc", so a stale 1.0.0-rc.6-glibc still outranks the final + // 1.0.0-glibc. Restrict to final MAJOR.MINOR.PATCH-glibc tags so + // RC/beta builds never enter the comparison, and a missing glibc + // tag just stalls the update instead of falling back to musl or an + // old RC. + matchPackageNames: [ + 'docker.io/rustfs/rustfs', + ], + allowedVersions: '/^\\d+\\.\\d+\\.\\d+-glibc$/', + }, { matchDatasources: [ 'go', diff --git a/test/e2e/internal/objectstore/s3.go b/test/e2e/internal/objectstore/s3.go index 571160cf2..768d70f6a 100644 --- a/test/e2e/internal/objectstore/s3.go +++ b/test/e2e/internal/objectstore/s3.go @@ -95,7 +95,7 @@ func newS3Deployment(namespace, name string) *appsv1.Deployment { Name: "init-permissions", // renovate: datasource=docker depName=busybox versioning=docker // Version: 1.38.0 - Image: "docker.io/library/busybox@sha256:dc2d74b28e4cf8984fa52af1f39bc7c3d9c73760b41a74d629f5d11b1ab28616", + Image: "docker.io/library/busybox@sha256:fd7dc98638c8e305f4dc34e979f1c0fdfdcaeb0fbf8fcff77ae834b6da3d7e6e", Command: []string{ "sh", "-c", fmt.Sprintf("mkdir -p %[1]s %[2]s && chmod 0777 %[1]s %[2]s", s3DataDir, s3LogDir), From 082b29e1d7e555973fcd0bc28b26fc44ebdf9ba6 Mon Sep 17 00:00:00 2001 From: Marco Nenciarini Date: Thu, 24 Sep 2026 09:51:29 +0200 Subject: [PATCH 09/11] fix: apply review feedback on RustFS migration Apply Tao's review suggestions. Co-authored-by: Tao Li Signed-off-by: Marco Nenciarini --- hack/object-store/object-store-deployment.yaml | 3 ++- renovate.json5 | 15 ++++++--------- test/e2e/internal/objectstore/s3.go | 10 ++++------ test/e2e/internal/tests/walrestore/fixtures.go | 4 ++-- web/docs/migration.md | 4 ++-- web/docs/object_stores.md | 2 +- 6 files changed, 17 insertions(+), 21 deletions(-) diff --git a/hack/object-store/object-store-deployment.yaml b/hack/object-store/object-store-deployment.yaml index a165d7c20..c85bfb5d7 100644 --- a/hack/object-store/object-store-deployment.yaml +++ b/hack/object-store/object-store-deployment.yaml @@ -88,7 +88,8 @@ spec: path: /health port: 9000 scheme: HTTPS - initialDelaySeconds: 30 + initialDelaySeconds: 5 + periodSeconds: 5 volumeMounts: - mountPath: /data name: data diff --git a/renovate.json5 b/renovate.json5 index 7d58f7cf5..5d9a4cd51 100644 --- a/renovate.json5 +++ b/renovate.json5 @@ -90,16 +90,13 @@ }, packageRules: [ { - // RustFS's default image is Alpine/musl, slower than glibc - // (rustfs/rustfs#1662). A bare `-glibc` suffix isn't a safe pin: - // semver compares prerelease identifiers lexically, and "rc" > - // "glibc", so a stale 1.0.0-rc.6-glibc still outranks the final - // 1.0.0-glibc. Restrict to final MAJOR.MINOR.PATCH-glibc tags so - // RC/beta builds never enter the comparison, and a missing glibc - // tag just stalls the update instead of falling back to musl or an - // old RC. + // Musl (the default image) is slower than glibc (rustfs/rustfs#1662). + // Only accept final x.y.z-glibc tags: semver ranks "rc" above "glibc", + // so an rc build would look newer than a real release and win by + // mistake. If no glibc tag matches, the update waits instead of + // picking musl or an rc. matchPackageNames: [ - 'docker.io/rustfs/rustfs', + 'rustfs/rustfs', ], allowedVersions: '/^\\d+\\.\\d+\\.\\d+-glibc$/', }, diff --git a/test/e2e/internal/objectstore/s3.go b/test/e2e/internal/objectstore/s3.go index 768d70f6a..0712d9ee6 100644 --- a/test/e2e/internal/objectstore/s3.go +++ b/test/e2e/internal/objectstore/s3.go @@ -85,11 +85,10 @@ func newS3Deployment(namespace, name string) *appsv1.Deployment { }, }, Spec: corev1.PodSpec{ - // RustFS runs as a non-root user but the PVC is root-owned, and - // a non-root init container cannot chown it on OpenShift. - // Instead the init creates a subdirectory it owns and makes it - // world-writable, which works as root (kind, cloud) or as the - // SCC-assigned UID (OpenShift). + // The PVC is root-owned and a non-root init container can't chown it on + // OpenShift. So it owns a subdirectory it creates and makes world-writable + // — that works whether it runs as root (kind, cloud) or the SCC UID + // (OpenShift). InitContainers: []corev1.Container{ { Name: "init-permissions", @@ -195,7 +194,6 @@ func newS3Deployment(namespace, name string) *appsv1.Deployment { } } -// newHealthProbe returns a probe hitting the RustFS health endpoint. func newHealthProbe(initialDelaySeconds, periodSeconds int32) *corev1.Probe { return &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ diff --git a/test/e2e/internal/tests/walrestore/fixtures.go b/test/e2e/internal/tests/walrestore/fixtures.go index db06ed3ae..17d805cd7 100644 --- a/test/e2e/internal/tests/walrestore/fixtures.go +++ b/test/e2e/internal/tests/walrestore/fixtures.go @@ -124,8 +124,8 @@ func newS3ClientDeployment(namespace string) *appsv1.Deployment { { Name: s3ClientName, // renovate: datasource=docker depName=amazon/aws-cli versioning=docker - // Version: 2.36.32 - Image: "docker.io/amazon/aws-cli@sha256:f630107e3eadb6479fa441631bbf50d15cf354a6ace85b6028bf6b3e5c69c605", + // Version: 2.37.0 + Image: "docker.io/amazon/aws-cli@sha256:337494c2047176fe9abcf45a5d1eaf1c2c62cae40953284fb1143b5c6170f065", Command: []string{"sleep", "infinity"}, Env: []corev1.EnvVar{ { diff --git a/web/docs/migration.md b/web/docs/migration.md index de6b71e8a..49519934f 100644 --- a/web/docs/migration.md +++ b/web/docs/migration.md @@ -230,10 +230,10 @@ spec: destinationPath: s3://backups/ endpointURL: http://object-store-eu:9000 s3Credentials: - accessKeyId: + accessKeyId: name: s3-eu key: ACCESS_KEY_ID - secretAccessKey: + secretAccessKey: name: s3-eu key: ACCESS_SECRET_KEY wal: diff --git a/web/docs/object_stores.md b/web/docs/object_stores.md index 2c0277bb1..b5178f728 100644 --- a/web/docs/object_stores.md +++ b/web/docs/object_stores.md @@ -430,7 +430,7 @@ write permissions to the bucket. ## RustFS Object Store [RustFS](https://rustfs.com/) is an open source, S3-compatible object store -that can run inside your Kubernetes cluster. Please refer to the +that can run inside your Kubernetes cluster. See the [RustFS documentation](https://docs.rustfs.com/) for deployment options; a minimal `Deployment` with a `PersistentVolumeClaim` and a `Service` exposing port 9000 is enough for testing purposes. RustFS serves plain HTTP unless it From ef9e90068a1339379b3b6fb47882191a1a6fcf6c Mon Sep 17 00:00:00 2001 From: Marco Nenciarini Date: Thu, 24 Sep 2026 09:33:52 +0200 Subject: [PATCH 10/11] fix: harden securityContext on hack object-store containers Drop all Linux capabilities on the init, object-store and s3-client containers, none of them need any. Add runAsNonRoot to object-store and s3-client; the init container still needs root to chmod the PVC-owned data and log directories. This is a dev/testing deployment, but anyone copying it as a starting point should get a secure default, not a baseline one. Signed-off-by: Marco Nenciarini --- hack/object-store/object-store-deployment.yaml | 7 +++++++ hack/object-store/s3-client.yaml | 4 ++++ 2 files changed, 11 insertions(+) diff --git a/hack/object-store/object-store-deployment.yaml b/hack/object-store/object-store-deployment.yaml index c85bfb5d7..76a365859 100644 --- a/hack/object-store/object-store-deployment.yaml +++ b/hack/object-store/object-store-deployment.yaml @@ -36,6 +36,9 @@ spec: name: logs securityContext: allowPrivilegeEscalation: false + capabilities: + drop: + - ALL seccompProfile: type: RuntimeDefault containers: @@ -75,6 +78,10 @@ spec: key: ACCESS_SECRET_KEY securityContext: allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true seccompProfile: type: RuntimeDefault livenessProbe: diff --git a/hack/object-store/s3-client.yaml b/hack/object-store/s3-client.yaml index 55474b9a0..24e011961 100644 --- a/hack/object-store/s3-client.yaml +++ b/hack/object-store/s3-client.yaml @@ -54,6 +54,10 @@ spec: readOnly: true securityContext: allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true seccompProfile: type: RuntimeDefault securityContext: From 2e7e4740b6ab9d0508242a5a070148a6f394d87e Mon Sep 17 00:00:00 2001 From: Marco Nenciarini Date: Thu, 24 Sep 2026 09:41:18 +0200 Subject: [PATCH 11/11] fix: mount the CA cert as a directory in s3-client, not via subPath A subPath mount doesn't pick up an updated Secret when cert-manager renews the certificate, so the cached CA would go stale until the pod restarts. The secret's items already remap tls.crt to ca.crt, so mounting the whole volume at /certs keeps AWS_CA_BUNDLE pointing at the same file while staying live-updated. Signed-off-by: Marco Nenciarini --- hack/object-store/s3-client.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hack/object-store/s3-client.yaml b/hack/object-store/s3-client.yaml index 24e011961..b70fb54a7 100644 --- a/hack/object-store/s3-client.yaml +++ b/hack/object-store/s3-client.yaml @@ -49,8 +49,7 @@ spec: value: when_required volumeMounts: - name: certs - mountPath: /certs/ca.crt - subPath: ca.crt + mountPath: /certs readOnly: true securityContext: allowPrivilegeEscalation: false