diff --git a/.github/workflows/installer.yml b/.github/workflows/installer.yml index 5e632e1..86662bf 100644 --- a/.github/workflows/installer.yml +++ b/.github/workflows/installer.yml @@ -5,11 +5,13 @@ on: branches: [main] paths: - 'installer/**' + - 'charts/**' - 'tests/fixtures/governance/STUDIO_TAG' - '.github/workflows/installer.yml' pull_request: paths: - 'installer/**' + - 'charts/**' - 'tests/fixtures/governance/STUDIO_TAG' - '.github/workflows/installer.yml' @@ -40,6 +42,18 @@ jobs: run: bun run embed env: LIBREDB_STUDIO_REPO: ${{ github.workspace }}/studio-src + # The Helm chart's tests (test/unit/kubernetes/chart.test.ts) render the chart with a pinned Helm; its Studio + # dependency is the chart that embed just downloaded and checked against the release manifest's sha256. + - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 + with: + version: v3.21.3 + - name: Lint the Helm chart + working-directory: . + run: | + mkdir -p charts/libredb-platform/charts + cp installer/generated/libredb-studio-*.tgz charts/libredb-platform/charts/ + helm lint charts/libredb-platform --strict -f charts/libredb-platform/tests/e2e/values-kind.yaml \ + --set platform.license.value=ldl1.0123456789abcdef.eyJpZCI6IkxJTlQifQ.c2lnbmF0dXJl - run: bun run typecheck - run: bun run coverage:check - run: bun run build diff --git a/.gitignore b/.gitignore index 591d2a8..00a3c07 100644 --- a/.gitignore +++ b/.gitignore @@ -66,3 +66,6 @@ next-env.d.ts /installer/keycloak-export/ /installer/ca.crt *.license + +# Helm chart dependencies, built with helm dependency build from Chart.lock +charts/*/charts/*.tgz diff --git a/README.md b/README.md index 39b77f9..c92e2c4 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ Write to [info@libredb.org](mailto:info@libredb.org) or join the waiting list at Platform 1.0.1 is the current release, published as the image `ghcr.io/libredb/libredb-platform`, and pairs with LibreDB Studio 1.0.0. It installs with Docker on one x86_64 Linux host: the installer deploys PostgreSQL, Keycloak, Platform, Studio and a Caddy edge, and can use an existing PostgreSQL server or Keycloak instead. -Kubernetes support follows in the next release. +On Kubernetes, the Helm chart [`charts/libredb-platform`](charts/libredb-platform) installs the same, as the [Kubernetes install guide](docs/deploy/install-kubernetes.md) describes. Cloud mode serves development and the hosted service of a later release, and it is not a supported way to install LibreDB. Download the installer from this repository's [GitHub releases](https://github.com/libredb/libredb-platform/releases) and follow the [Docker install guide](docs/deploy/install-docker.md). Every install needs a license key from LibreDB, and Platform does not start in self-hosted mode without one. diff --git a/ROADMAP.md b/ROADMAP.md index 12f4736..7f9b515 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -6,7 +6,7 @@ - [ ] Studio users can connect to the platform with limited permissions(studio, account, support/tickets) - [ ] All libredb-studio database connections support -- [ ] Kubernetes +- [x] Kubernetes (Helm chart `charts/libredb-platform`) - [ ] GitOps --- diff --git a/charts/libredb-platform/.helmignore b/charts/libredb-platform/.helmignore new file mode 100644 index 0000000..2a43beb --- /dev/null +++ b/charts/libredb-platform/.helmignore @@ -0,0 +1,7 @@ +.DS_Store +.git/ +*.swp +*.bak +*.tmp +tests/ +README.md.gotmpl diff --git a/charts/libredb-platform/Chart.lock b/charts/libredb-platform/Chart.lock new file mode 100644 index 0000000..abaf0d1 --- /dev/null +++ b/charts/libredb-platform/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: libredb-studio + repository: https://libredb.org/libredb-studio/ + version: 0.1.100 +digest: sha256:a56f4096de2bd4d02e73a5b984ba8040490565a767be05a123ffb2abb88a92d6 +generated: "2026-10-20T02:08:21.66270654+03:00" diff --git a/charts/libredb-platform/Chart.yaml b/charts/libredb-platform/Chart.yaml new file mode 100644 index 0000000..f7257d1 --- /dev/null +++ b/charts/libredb-platform/Chart.yaml @@ -0,0 +1,31 @@ +apiVersion: v2 +name: libredb-platform +description: >- + LibreDB Platform, database access governance, with LibreDB Studio, on Kubernetes: Platform, its migrate job, + Studio under governance, and optionally a managed PostgreSQL and Keycloak. Source-available under the + Elastic License 2.0; every install needs a license from LibreDB. +type: application +# The chart version; appVersion is the Platform release whose images and Studio chart it pins +# (installer/release/release.config.json, which installer/test/unit/kubernetes/chart.test.ts compares). +version: 0.1.0 +appVersion: "1.0.1" +kubeVersion: ">=1.28.0-0" +home: https://libredb.org/platform/ +sources: + - https://github.com/libredb/libredb-platform +license: Elastic-2.0 +keywords: + - database + - governance + - audit + - keycloak + - libredb +maintainers: + - name: LibreDB + url: https://libredb.org +dependencies: + - name: libredb-studio + alias: studio + version: 0.1.100 + repository: https://libredb.org/libredb-studio/ + condition: studio.enabled diff --git a/charts/libredb-platform/README.md b/charts/libredb-platform/README.md new file mode 100644 index 0000000..929bdcf --- /dev/null +++ b/charts/libredb-platform/README.md @@ -0,0 +1,86 @@ +# libredb-platform Helm chart + +LibreDB Platform, database access governance, with LibreDB Studio, on Kubernetes. +The guide is [docs/deploy/install-kubernetes.md](../../docs/deploy/install-kubernetes.md); this file lists the values. +LibreDB Platform is source-available under the Elastic License 2.0, and every install needs a license from LibreDB. + +The chart is the installer's Kubernetes mode driven by Helm. +It renders the values as an installation configuration of `mode: kubernetes`, which its jobs validate with the installer's schema and turn into Secrets, certificates, environments, a PostgreSQL provisioning script and a Keycloak realm with the installer's own code, bundled in `files/libredb-jobs.cjs.gz` (`bun run build:helm-jobs` in `installer/` rebuilds it; a unit test fails while it is stale). + +## Install + +```sh +helm repo add libredb-studio https://libredb.org/libredb-studio/ +helm dependency build charts/libredb-platform +helm install libredb charts/libredb-platform -n libredb --create-namespace \ + -f libredb-values.yaml --set-file platform.license.value=acme.license \ + --wait --wait-for-jobs --timeout 10m +``` + +## What it deploys + +| Object | When | +| --- | --- | +| Job `libredb-bootstrap` (pre-install, pre-upgrade and pre-rollback hook) with its ServiceAccount and Role | always | +| StatefulSet and Service `libredb-postgres`, provisioning Job | `postgresql.mode: managed` | +| Deployment and Services `keycloak` and `keycloak-edge`, Ingress `keycloak` | `keycloak.mode: managed` | +| Job `libredb-keycloak-configure-` | a managed Keycloak, or `keycloak.external.provisioning: installer` | +| Job `libredb-platform-migrate-` | always | +| Job `libredb-restart` (post-install, post-upgrade and post-rollback hook) with its ServiceAccount and Role | always | +| Deployment, Service, Ingress and PVC `libredb-platform` | always | +| The Studio chart (pinned by the release manifest) under governance | `studio.enabled` | +| NetworkPolicies `libredb-postgres` and `keycloak` | `networkPolicy.enabled` | + +The bootstrap job writes `libredb-secrets` (generated secrets and seal keys, never replaced), `libredb-tls` (internal CA and leaves), `libredb-platform-env`, `libredb-platform-migrate-env`, `libredb-studio-env`, `libredb-keycloak-config`, `libredb-ca` and the objects of the managed PostgreSQL and Keycloak. +They carry no Helm ownership and outlive `helm uninstall`; back up `libredb-secrets` and `libredb-tls`. +When one of them changes, the restart job restarts the workloads that read it, once Helm applied the release. + +## Values + +| Value | Default | Meaning | +| --- | --- | --- | +| `installation.name` | `libredb` | The installation's name, which names the internal CA | +| `hosts.platform`, `hosts.studio`, `hosts.auth` | | Required: the public hosts, resolvable from browsers and from the cluster's pods | +| `platform.organizationName` | | Required: the name of the installation's one tenant | +| `platform.queryLogRetentionDays` | `null` (90) | Days Platform keeps query events | +| `platform.license.value` | | The license token; pass it with `--set-file` | +| `platform.license.existingSecret`, `platform.license.key` | `""`, `license` | Or a Secret that holds the token | +| `platform.image` | the release's, by digest | Platform's image | +| `platform.resources`, `nodeSelector`, `tolerations`, `affinity`, `podAnnotations`, `podLabels` | | Platform's pod; `nodeSelector` and `tolerations` also place the jobs that run the Platform image, and `nodeSelector` holds `kubernetes.io/arch: amd64` unless it names another arch | +| `platform.persistence.enabled`, `size`, `storageClass`, `accessModes` | `true`, `5Gi` | Support attachments under `/app/uploads` | +| `tls.mode` | `generated` | `generated`: the internal CA issues the ingress certificates; `secrets`: your TLS Secrets | +| `tls.secrets.platform`, `studio`, `auth` | | With `tls.mode: secrets`, the TLS Secret of each host (`auth` only with a managed Keycloak); repeat `studio` as `studio.ingress.tls[0].secretName` | +| `tls.caBundle` | | With `tls.mode: secrets`, the PEM of the CA that issued them (required) | +| `ingress.enabled`, `className`, `annotations` | `true`, `""`, `nginx.ingress.kubernetes.io/proxy-buffer-size: "16k"` | The Ingress of Platform and of a managed Keycloak; the annotation gives ingress-nginx room for the response headers of Platform's sign-in callback, and other controllers ignore it | +| `postgresql.mode` | `managed` | `managed` or `external` | +| `postgresql.managed.image`, `persistence.size`, `persistence.storageClass`, `resources`, `nodeSelector`, `tolerations`, `affinity` | the release's image, `10Gi` | The managed server | +| `postgresql.external.host`, `port`, `caBundle` | `""`, `5432` | The external server, provisioned once by a DBA with Secret `libredb-postgres-dba-script` | +| `keycloak.mode` | `managed` | `managed` or `external` | +| `keycloak.realm` | `libredb` | The realm | +| `keycloak.firstAdmin.enabled`, `username` | `null`, `libredb-admin` | The first admin account; enabled by default with a managed Keycloak | +| `keycloak.userManager.enabled`, `username` | `null`, `libredb-user-manager` | The user manager account; enabled by default with a managed Keycloak | +| `keycloak.managed.image`, `resources`, `nodeSelector`, `tolerations`, `affinity` | the release's image | The managed Keycloak | +| `keycloak.managed.edge.image`, `resources` | the release's Caddy | The installer's edge in front of Keycloak's public paths | +| `keycloak.external.baseUrl`, `adminUrl` | | Your Keycloak; `hosts.auth` must be the host of `baseUrl` | +| `keycloak.external.provisioning` | `installer` | `installer`: the chart's job configures the realm; `manual`: your identity team does | +| `keycloak.external.adminRealm`, `adminSecret`, `adminClientId`, `adminUser` | `master` | With `installer`: the Secret with `admin-client-secret` (client) or `admin-password` (user) | +| `keycloak.external.clientSecretsSecret` | | With `manual`: the Secret with `studio-client-secret` and `platform-client-secret` | +| `keycloak.external.caBundle` | | The PEM of your Keycloak's CA, when a private CA issued its certificate | +| `network.httpsProxy` | | An http(s) proxy for the apps' outbound HTTPS, without credentials | +| `jobs.backoffLimit`, `activeDeadlineSeconds`, `resources` | `6`, `900` | The chart's jobs | +| `networkPolicy.enabled` | `true` | PostgreSQL and Keycloak's Admin API reachable only from this release's pods | +| `platform.sealKeys.issueNew`, `activeKid`, `retire` | `false`, `""`, `[]` | The seal key rotation, one upgrade per step; see the guide's "Rotate secrets and the seal key" | +| `platform.authSecretPrevious.existingSecret`, `key` | `""`, `auth-secret-previous` | During a rotation of Platform's `AUTH_SECRET`, a Secret with the previous value | +| `global.compatibility.openshift.adaptSecurityContext` | `auto` | `auto` leaves the pods' uids to OpenShift's SCC when the cluster is OpenShift, `force` always, `disabled` never; Studio reads it too | +| `imagePullSecrets` | `[]` | For every pod of the chart (Studio has its own under `studio:`) | +| `studio.*` | governance defaults | The Studio chart's values; repeat `hosts.studio` in `studio.ingress` and the issuer in `studio.config.oidcIssuer` | + +The chart refuses to render, naming the value, when a required value is missing or two copies of a host or the issuer disagree. + +## Tests + +- `installer/test/unit/kubernetes/chart.test.ts` renders the chart with `helm template` and holds it to the installer: render guards, the installation document against the schema, the release manifest's pins, Studio's and Keycloak's environments against the installer's renderers, the bootstrap Role against every object the job writes, and the pods' security. +- `tests/e2e/run-e2e.sh` installs on a local kind cluster and runs the browser suite of `installer/e2e/kubernetes`. +- `tests/e2e/run-openshift.sh` does the same on OpenShift, in the namespace of the current `oc login`, under the `restricted-v2` SCC. +- `tests/e2e/run-rotation.sh` rotates the seal key and the generated secrets; `tests/e2e/run-resilience.sh` kills pods and restarts the node; `tests/e2e/run-soak.sh` runs one installation for hours. +- `tests/e2e/run-restore.sh` backs up, deletes the namespace and restores; `tests/e2e/run-offline.sh` installs the packaged archive from an image mirror with no network for Helm. diff --git a/charts/libredb-platform/files/libredb-jobs.cjs.gz b/charts/libredb-platform/files/libredb-jobs.cjs.gz new file mode 100644 index 0000000..c65b44b Binary files /dev/null and b/charts/libredb-platform/files/libredb-jobs.cjs.gz differ diff --git a/charts/libredb-platform/templates/NOTES.txt b/charts/libredb-platform/templates/NOTES.txt new file mode 100644 index 0000000..43946bb --- /dev/null +++ b/charts/libredb-platform/templates/NOTES.txt @@ -0,0 +1,47 @@ +LibreDB Platform {{ .Chart.AppVersion }} is installing in namespace {{ .Release.Namespace }}. + + Platform https://{{ .Values.hosts.platform }} + Studio https://{{ .Values.hosts.studio }} + Keycloak {{ include "libredb-platform.issuer" . }} + +Watch it come up; Platform turns ready once its migrate job has run: + + kubectl -n {{ .Release.Namespace }} get pods,jobs + kubectl -n {{ .Release.Namespace }} logs deploy/libredb-platform | grep -i licen + +{{- if eq .Values.tls.mode "generated" }} + +The internal CA issued the ingress certificates. Import it into the trust store of every workstation that opens the +three URLs (it can sign a certificate for any name, so import it only where the cluster's operators are trusted): + + kubectl -n {{ .Release.Namespace }} get secret libredb-tls-platform -o jsonpath='{.data.ca\.crt}' | base64 -d > libredb-ca.crt +{{- end }} +{{- if include "libredb-platform.managedKeycloak" . }} + +First sign-in: open Platform, choose Continue with SSO and sign in as {{ .Values.keycloak.firstAdmin.username }} with +its temporary password; Keycloak then asks for a new password and a TOTP device. + + kubectl -n {{ .Release.Namespace }} get secret libredb-keycloak-config -o jsonpath='{.data.first-admin-password}' | base64 -d; echo + +{{ .Values.keycloak.userManager.username }} manages the realm's users in Keycloak's console, which is not exposed by the ingress: + + kubectl -n {{ .Release.Namespace }} get secret libredb-keycloak-config -o jsonpath='{.data.user-manager-password}' | base64 -d; echo + kubectl -n {{ .Release.Namespace }} port-forward svc/keycloak 18080:8080 + # then open http://127.0.0.1:18080/admin/{{ .Values.keycloak.realm }}/console/ +{{- end }} +{{- if eq .Values.postgresql.mode "external" }} + +The external PostgreSQL server needs its one-time provisioning by a DBA. The script holds SCRAM verifiers only: + + kubectl -n {{ .Release.Namespace }} get secret libredb-postgres-dba-script -o jsonpath='{.data.provision\.sql}' | base64 -d > libredb-provision.sql + psql "host={{ .Values.postgresql.external.host }} port={{ .Values.postgresql.external.port }} dbname=postgres user= sslmode=verify-full sslrootcert=" -f libredb-provision.sql + +The migrate job retries until the roles exist. +{{- end }} + +BACK UP THE GENERATED SECRETS NOW. Secret libredb-secrets holds the seal keys that open every database credential +Platform stores; it outlives `helm uninstall` on purpose, and a lost key cannot be recovered: + + kubectl -n {{ .Release.Namespace }} get secret libredb-secrets libredb-tls libredb-installer-state -o yaml > libredb-secrets-backup.yaml + +A full backup set adds a dump of each database; "Back up and restore" in docs/deploy/install-kubernetes.md has both. diff --git a/charts/libredb-platform/templates/_helpers.tpl b/charts/libredb-platform/templates/_helpers.tpl new file mode 100644 index 0000000..3884cb0 --- /dev/null +++ b/charts/libredb-platform/templates/_helpers.tpl @@ -0,0 +1,372 @@ +{{/* +Names. Every object has a fixed name, like the installer's Kubernetes constants (installer/src/kubernetes/names.ts), +so a namespace holds one installation; installer/test/unit/kubernetes/chart.test.ts holds the bootstrap Role's names +to the objects the bootstrap job writes. +*/}} + +{{- define "libredb-platform.labels" -}} +app.kubernetes.io/name: libredb-platform +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/part-of: libredb +helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" }} +{{- end }} + +{{/* Selector labels of one component: platform, postgres, keycloak or a job. */}} +{{- define "libredb-platform.selectorLabels" -}} +app.kubernetes.io/name: libredb-platform +app.kubernetes.io/instance: {{ .root.Release.Name }} +app.kubernetes.io/component: {{ .component }} +{{- end }} + +{{- define "libredb-platform.image" -}} +{{- printf "%s:%s" .repository .tag }} +{{- end }} + +{{- define "libredb-platform.managedPostgres" -}} +{{- if eq .Values.postgresql.mode "managed" }}true{{ end }} +{{- end }} + +{{- define "libredb-platform.managedKeycloak" -}} +{{- if eq .Values.keycloak.mode "managed" }}true{{ end }} +{{- end }} + +{{/* Whether the chart runs the keycloak-configure job: a managed Keycloak, or an external one with provisioning installer. */}} +{{- define "libredb-platform.keycloakJob" -}} +{{- if or (include "libredb-platform.managedKeycloak" .) (eq .Values.keycloak.external.provisioning "installer") }}true{{ end }} +{{- end }} + +{{- define "libredb-platform.issuer" -}} +{{- if include "libredb-platform.managedKeycloak" . -}} +{{- printf "https://%s/realms/%s" .Values.hosts.auth .Values.keycloak.realm }} +{{- else -}} +{{- printf "%s/realms/%s" (trimSuffix "/" .Values.keycloak.external.baseUrl) .Values.keycloak.realm }} +{{- end }} +{{- end }} + +{{/* Where the bootstrap job finds the operator's inputs, as the installation document names them. */}} +{{- define "libredb-platform.inputDir" -}}/etc/libredb/inputs{{- end }} + +{{/* +The installation document: the Helm values as a LibreDBInstallation of mode kubernetes, which the jobs validate with +the installer's schema (installer/src/config/schema.ts). It holds no secret, only the files that hold them. +*/}} +{{- define "libredb-platform.installation" -}} +{{- $in := include "libredb-platform.inputDir" . -}} +{{- $tls := dict "mode" "generated" -}} +{{- if eq .Values.tls.mode "secrets" -}} +{{- $tls = dict "mode" "external" "caBundleFile" (printf "%s/ca/tls-ca.crt" $in) "external" (dict "trustedProxies" (list "private_ranges")) -}} +{{- end -}} +{{- $postgres := dict "mode" "managed" -}} +{{- if eq .Values.postgresql.mode "external" -}} +{{- $ext := dict "host" .Values.postgresql.external.host "port" (int .Values.postgresql.external.port) "provisioning" "manual" -}} +{{- if .Values.postgresql.external.caBundle }}{{ $_ := set $ext "caBundleFile" (printf "%s/ca/postgres-ca.crt" $in) }}{{ end -}} +{{- $postgres = dict "mode" "external" "external" $ext -}} +{{- end -}} +{{- $kc := .Values.keycloak -}} +{{- $account := dict "firstAdmin" (dict "username" $kc.firstAdmin.username) "userManager" (dict "username" $kc.userManager.username) -}} +{{- if not (kindIs "invalid" $kc.firstAdmin.enabled) }}{{ $_ := set $account.firstAdmin "enabled" $kc.firstAdmin.enabled }}{{ end -}} +{{- if not (kindIs "invalid" $kc.userManager.enabled) }}{{ $_ := set $account.userManager "enabled" $kc.userManager.enabled }}{{ end -}} +{{- $keycloak := dict "mode" $kc.mode "realm" $kc.realm "firstAdmin" $account.firstAdmin "userManager" $account.userManager -}} +{{- if eq $kc.mode "external" -}} +{{- $x := $kc.external -}} +{{- $ext := dict "baseUrl" $x.baseUrl "provisioning" $x.provisioning -}} +{{- if $x.adminUrl }}{{ $_ := set $ext "adminUrl" $x.adminUrl }}{{ end -}} +{{- if $x.caBundle }}{{ $_ := set $ext "caBundleFile" (printf "%s/ca/keycloak-ca.crt" $in) }}{{ end -}} +{{- if eq $x.provisioning "installer" -}} +{{- $_ := set $ext "adminRealm" $x.adminRealm -}} +{{- if $x.adminClientId -}} +{{- $_ := set $ext "adminClientId" $x.adminClientId -}} +{{- $_ := set $ext "adminClientSecretFile" (printf "%s/keycloak-admin/admin-client-secret" $in) -}} +{{- end -}} +{{- if $x.adminUser -}} +{{- $_ := set $ext "adminUser" $x.adminUser -}} +{{- $_ := set $ext "adminPasswordFile" (printf "%s/keycloak-admin/admin-password" $in) -}} +{{- end -}} +{{- else -}} +{{- $_ := set $ext "studioClientSecretFile" (printf "%s/keycloak-clients/studio-client-secret" $in) -}} +{{- $_ := set $ext "platformClientSecretFile" (printf "%s/keycloak-clients/platform-client-secret" $in) -}} +{{- end -}} +{{- $_ := set $keycloak "external" $ext -}} +{{- end -}} +{{- $platform := dict "organizationName" .Values.platform.organizationName "licenseFile" (printf "%s/license/license" $in) -}} +{{- if .Values.platform.authSecretPrevious.existingSecret }}{{ $_ := set $platform "authSecretPreviousFile" (printf "%s/auth-secret-previous/auth-secret-previous" $in) }}{{ end -}} +{{- if not (kindIs "invalid" .Values.platform.queryLogRetentionDays) }}{{ $_ := set $platform "queryLogRetentionDays" (int .Values.platform.queryLogRetentionDays) }}{{ end -}} +{{- $doc := dict "apiVersion" "install.libredb.org/v1" "kind" "LibreDBInstallation" "metadata" (dict "name" .Values.installation.name) "mode" "kubernetes" "endpoints" (dict "style" "hostnames" "hosts" .Values.hosts) "tls" $tls "postgres" $postgres "keycloak" $keycloak "platform" $platform "kubernetes" (dict "namespace" .Release.Namespace) -}} +{{- if .Values.network.httpsProxy }}{{ $_ := set $doc "network" (dict "httpsProxy" .Values.network.httpsProxy) }}{{ end -}} +{{- $seal := .Values.platform.sealKeys -}} +{{- $sealKeys := dict "issueNew" $seal.issueNew "retire" (include "libredb-platform.retiredKids" . | fromJsonArray) -}} +{{- if $seal.activeKid }}{{ $_ := set $sealKeys "activeKid" (include "libredb-platform.kid" (dict "value" $seal.activeKid "name" "platform.sealKeys.activeKid")) }}{{ end -}} +{{- $_ := set $doc "bridge" (dict "sealKeys" $sealKeys) -}} +{{- toPrettyJson $doc }} +{{- end }} + +{{/* A seal key id, 16 lowercase hex digits, or a failed render that names the value: dict "value" "name". */}} +{{- define "libredb-platform.kid" -}} +{{- if not (regexMatch "^[0-9a-f]{16}$" (toString .value)) }}{{ fail (printf "%s: %q is no seal key id, which is 16 lowercase hex digits" .name (toString .value)) }}{{ end -}} +{{- .value -}} +{{- end }} + +{{/* platform.sealKeys.retire as a JSON array, each kid checked, since the jobs' SQL quotes them as literals. */}} +{{- define "libredb-platform.retiredKids" -}} +{{- $kids := list -}} +{{- range $index, $kid := .Values.platform.sealKeys.retire -}} +{{- $kids = append $kids (include "libredb-platform.kid" (dict "value" $kid "name" (printf "platform.sealKeys.retire[%d]" $index))) -}} +{{- end -}} +{{- toJson $kids -}} +{{- end }} + +{{/* +The installer's envelopeUsesScript (installer/src/secrets/seal-rotation.ts) for platform.sealKeys.retire, which the +bootstrap job's seal-envelopes init container runs as libredb_platform; installer/test/unit/kubernetes/chart.test.ts +holds the two to the same text. +*/}} +{{- define "libredb-platform.sealEnvelopesSql" -}} +{{- $quoted := list -}} +{{- range (include "libredb-platform.retiredKids" . | fromJsonArray) }}{{ $quoted = append $quoted (printf "'%s'" .) }}{{ end -}} +SELECT 'libredb-result envelope-uses ' || ((SELECT coalesce(jsonb_agg(jsonb_build_object('kid', e.kid, 'connectionId', e.connection_id::text, 'name', e.name) ORDER BY e.kid, e.name), '[]'::jsonb) FROM (SELECT connection_id, name, split_part(read_secret, '.', 2) AS kid FROM bridge.connections_v1 UNION SELECT connection_id, name, split_part(write_secret, '.', 2) FROM bridge.connections_v1 WHERE write_secret IS NOT NULL) e WHERE e.kid IN ({{ join ", " $quoted }})))::text; +{{ end }} + +{{/* The installer's deleteCanaryScript for each retired kid, which the restart job runs once the release applied. */}} +{{- define "libredb-platform.sealCanarySql" -}} +{{- range (include "libredb-platform.retiredKids" . | fromJsonArray) -}} +DELETE FROM platform.seal_canary WHERE kid = '{{ . }}'; +{{ end -}} +{{- end }} + +{{/* +An init container that runs one SQL file of the chart as libredb_platform over verified TLS, with the role's password +from Secret libredb-platform-migrate-env in PGPASSWORD, never on a command line: dict "root" . "name" "file" "output". +*/}} +{{- define "libredb-platform.platformSqlContainer" -}} +- name: {{ .name }} + image: {{ include "libredb-platform.image" .root.Values.postgresql.managed.image }} + imagePullPolicy: {{ .root.Values.postgresql.managed.image.pullPolicy }} + command: + - /bin/sh + - -c + - | + set -eu + rest="${DATABASE_URL#postgresql://libredb_platform:}" + export PGPASSWORD="${rest%%@*}" + address="${rest#*@}" + address="${address%%/*}" + conninfo="host=${address%:*} port=${address##*:} dbname=libredb user=libredb_platform sslmode=verify-full sslrootcert=/etc/libredb/ca/ca.crt connect_timeout=10" + # A network policy controller such as k3s' kube-router may refuse a new pod's first connections until it holds + # the pod's address, so the container waits up to 30 s for one; the script itself runs once, and its failure + # is the job's. + tries=0 + until psql "$conninfo" -X -q -A -t -c 'SELECT 1' >/dev/null 2>&1 || [ "$tries" -ge 30 ]; do tries=$((tries + 1)); sleep 1; done + psql "$conninfo" -X -q -A -t -v ON_ERROR_STOP=1 -f {{ .file }}{{ with .output }} > {{ . }}{{ end }} + env: + - name: DATABASE_URL + valueFrom: + secretKeyRef: + name: libredb-platform-migrate-env + key: DATABASE_URL + securityContext: + {{- include "libredb-platform.containerSecurity" .root | nindent 4 }} + resources: + {{- toYaml .root.Values.jobs.resources | nindent 4 }} + volumeMounts: + - name: installation-ca + mountPath: /etc/libredb/ca + readOnly: true + - name: chart + mountPath: /etc/libredb/chart + readOnly: true + - name: tmp + mountPath: /tmp +{{- end }} + +{{- define "libredb-platform.chartSettings" -}} +{{- /* No Helm or Argo CD tracking label (app.kubernetes.io/instance): a tool that tracked these objects would prune them. */}} +{{- toPrettyJson (dict "studioSecretName" .Values.studio.secrets.existingSecret "labels" (dict "app.kubernetes.io/part-of" "libredb" "app.kubernetes.io/managed-by" "libredb-bootstrap" "install.libredb.org/installation" .Values.installation.name)) }} +{{- end }} + +{{/* A short hash that names a job after what it runs, so a changed job is a new Job and an unchanged one stays done. */}} +{{- define "libredb-platform.hash" -}} +{{- . | sha256sum | trunc 8 }} +{{- end }} + +{{/* The restricted security context of every LibreDB container (uid 1001, as the installer's compose file). */}} +{{- define "libredb-platform.containerSecurity" -}} +allowPrivilegeEscalation: false +readOnlyRootFilesystem: true +runAsNonRoot: true +capabilities: + drop: ["ALL"] +seccompProfile: + type: RuntimeDefault +{{- end }} + +{{/* +Whether to leave out the pods' fixed uids for OpenShift, whose restricted-v2 SCC assigns runAsUser and fsGroup from the +namespace's range and rejects any other: global.compatibility.openshift.adaptSecurityContext, the value the Studio +chart reads too, "auto" when the API server serves security.openshift.io/v1, "force" always, "disabled" never. +*/}} +{{- define "libredb-platform.adaptOpenShift" -}} +{{- $mode := dig "compatibility" "openshift" "adaptSecurityContext" "auto" (.Values.global | default dict) }} +{{- if or (eq $mode "force") (and (eq $mode "auto") (.Capabilities.APIVersions.Has "security.openshift.io/v1")) }} +{{- true }} +{{- end }} +{{- end }} + +{{/* The pod security context, with the image's uid unless OpenShift assigns one: dict "root" . "uid" . */}} +{{- define "libredb-platform.podSecurity" -}} +runAsNonRoot: true +{{- if not (include "libredb-platform.adaptOpenShift" .root) }} +runAsUser: {{ .uid }} +runAsGroup: {{ .uid }} +fsGroup: {{ .uid }} +{{- end }} +seccompProfile: + type: RuntimeDefault +{{- end }} + +{{/* +Where a pod of the Platform image may run: platform.nodeSelector and platform.tolerations, with kubernetes.io/arch amd64 +unless platform.nodeSelector names another, since the Platform image is built for linux/amd64 alone. Platform and the +jobs that run its image share it, so a job never lands on a node the image cannot run on. +*/}} +{{- define "libredb-platform.platformScheduling" -}} +nodeSelector: + {{- toYaml (merge (dict) .Values.platform.nodeSelector (dict "kubernetes.io/arch" "amd64")) | nindent 2 }} +{{- with .Values.platform.tolerations }} +tolerations: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- end }} + +{{/* +The node command of a chart job: the gzipped bundle decompressed into /tmp, then run with the cluster CA and, for the +keycloak job, the installation's CA bundle trusted. +*/}} +{{- define "libredb-platform.jobCommand" -}} +- /bin/sh +- -c +- | + set -eu + cat /var/run/secrets/kubernetes.io/serviceaccount/ca.crt {{ if .withBundle }}/etc/libredb/ca/ca.crt {{ end }}> /tmp/trust.pem + node -e 'const z=require("zlib"),f=require("fs");f.writeFileSync("/tmp/libredb-jobs.cjs",z.gunzipSync(f.readFileSync("/etc/libredb/jobs/libredb-jobs.cjs.gz")))' + NODE_EXTRA_CA_CERTS=/tmp/trust.pem exec node /tmp/libredb-jobs.cjs {{ .command }} +{{- end }} + +{{/* +An init container that waits until the installation's PostgreSQL server accepts connections, so a job that starts +beside a server that is still starting waits instead of failing. pg_isready checks the protocol, without credentials. +*/}} +{{- define "libredb-platform.waitForPostgres" -}} +- name: wait-for-postgres + image: {{ include "libredb-platform.image" .Values.postgresql.managed.image }} + imagePullPolicy: {{ .Values.postgresql.managed.image.pullPolicy }} + command: + - /bin/sh + - -c + - | + until pg_isready -q -t 5 -h "$PGHOST" -p "$PGPORT"; do echo "waiting for PostgreSQL at $PGHOST:$PGPORT"; sleep 3; done + env: + # libpq looks the OS user up when PGUSER is unset, which fails for a uid the image has no passwd entry for. + - name: PGUSER + value: postgres + - name: PGHOST + value: {{ if include "libredb-platform.managedPostgres" . }}libredb-postgres{{ else }}{{ .Values.postgresql.external.host | quote }}{{ end }} + - name: PGPORT + value: {{ if include "libredb-platform.managedPostgres" . }}"5432"{{ else }}{{ .Values.postgresql.external.port | quote }}{{ end }} + securityContext: + {{- include "libredb-platform.containerSecurity" . | nindent 4 }} + resources: + {{- toYaml .Values.jobs.resources | nindent 4 }} +{{- end }} + +{{/* +Render guards: every combination the installation could not run with fails here, naming the value, before anything +reaches the cluster. The installer schema checks the rest in the bootstrap job. +*/}} +{{- define "libredb-platform.guards" -}} +{{- range $endpoint := list "platform" "studio" "auth" }} +{{- if not (get $.Values.hosts $endpoint) }} +{{- fail (printf "hosts.%s is required: the public host of %s" $endpoint $endpoint) }} +{{- end }} +{{- end }} +{{- if not .Values.platform.organizationName }} +{{- fail "platform.organizationName is required: the name of the installation's one tenant" }} +{{- end }} +{{- if and (not .Values.platform.license.value) (not .Values.platform.license.existingSecret) }} +{{- fail "platform.license is required: every install needs a license from LibreDB (info@libredb.org); pass it with --set-file platform.license.value=, or name a Secret in platform.license.existingSecret" }} +{{- end }} +{{- if and .Values.platform.license.value (not (regexMatch "^ldl1\\.[0-9a-f]{16}\\.[A-Za-z0-9_-]+\\.[A-Za-z0-9_-]+$" (trim .Values.platform.license.value))) }} +{{- fail "platform.license.value must hold the license token alone, one line of the form ldl1..." }} +{{- end }} +{{- if not (has .Values.tls.mode (list "generated" "secrets")) }} +{{- fail (printf "tls.mode must be generated or secrets, not %q" .Values.tls.mode) }} +{{- end }} +{{- if eq .Values.tls.mode "secrets" }} +{{- range $endpoint := list "platform" "studio" }} +{{- if not (get $.Values.tls.secrets $endpoint) }} +{{- fail (printf "tls.mode secrets needs tls.secrets.%s, the TLS Secret of hosts.%s" $endpoint $endpoint) }} +{{- end }} +{{- end }} +{{- if and (include "libredb-platform.managedKeycloak" .) (not .Values.tls.secrets.auth) }} +{{- fail "tls.mode secrets with a managed Keycloak needs tls.secrets.auth, the TLS Secret of hosts.auth" }} +{{- end }} +{{- if not .Values.tls.caBundle }} +{{- fail "tls.mode secrets needs tls.caBundle, the PEM of the CA that issued the ingress certificates, which every pod trusts" }} +{{- end }} +{{- end }} +{{- if not (has .Values.postgresql.mode (list "managed" "external")) }} +{{- fail (printf "postgresql.mode must be managed or external, not %q" .Values.postgresql.mode) }} +{{- end }} +{{- if and (eq .Values.postgresql.mode "external") (not .Values.postgresql.external.host) }} +{{- fail "postgresql.mode external needs postgresql.external.host, the DNS name its certificate covers" }} +{{- end }} +{{- if not (has .Values.keycloak.mode (list "managed" "external")) }} +{{- fail (printf "keycloak.mode must be managed or external, not %q" .Values.keycloak.mode) }} +{{- end }} +{{- if eq .Values.keycloak.mode "external" }} +{{- $x := .Values.keycloak.external }} +{{- if not $x.baseUrl }} +{{- fail "keycloak.mode external needs keycloak.external.baseUrl" }} +{{- end }} +{{- if not (has $x.provisioning (list "installer" "manual")) }} +{{- fail (printf "keycloak.external.provisioning must be installer or manual, not %q" $x.provisioning) }} +{{- end }} +{{- if and (eq $x.provisioning "installer") (not $x.adminSecret) }} +{{- fail "keycloak.external.provisioning installer needs keycloak.external.adminSecret, the Secret with the admin credential" }} +{{- end }} +{{- if and (eq $x.provisioning "installer") (ne (len (compact (list $x.adminClientId $x.adminUser))) 1) }} +{{- fail "keycloak.external.provisioning installer needs exactly one of keycloak.external.adminClientId and keycloak.external.adminUser" }} +{{- end }} +{{- if and (eq $x.provisioning "manual") (not $x.clientSecretsSecret) }} +{{- fail "keycloak.external.provisioning manual needs keycloak.external.clientSecretsSecret, with the keys studio-client-secret and platform-client-secret" }} +{{- end }} +{{- end }} +{{- if and .Values.network.httpsProxy (regexMatch "(?i)^[a-z][a-z0-9+.-]*://[^/]*@" .Values.network.httpsProxy) }} +{{- fail "network.httpsProxy must not hold credentials: the installation document that holds it is a ConfigMap" }} +{{- end }} +{{- if .Values.studio.enabled }} +{{- $issuer := include "libredb-platform.issuer" . }} +{{- if ne .Values.studio.config.oidcIssuer $issuer }} +{{- fail (printf "studio.config.oidcIssuer must be %s, the issuer of hosts.auth and keycloak.realm, not %q" $issuer .Values.studio.config.oidcIssuer) }} +{{- end }} +{{- if ne (.Values.studio.fullnameOverride | toString) "libredb-studio" }} +{{- fail "studio.fullnameOverride must stay libredb-studio: the restart job of the chart names Studio's Deployment so" }} +{{- end }} +{{- if ne .Values.studio.secrets.existingSecret "libredb-studio-env" }} +{{- fail "studio.secrets.existingSecret must stay libredb-studio-env: the bootstrap job writes Studio's secrets there" }} +{{- end }} +{{- if .Values.studio.ingress.enabled }} +{{- $hosts := list }} +{{- range .Values.studio.ingress.hosts }}{{ $hosts = append $hosts .host }}{{ end }} +{{- if ne (toJson $hosts) (toJson (list .Values.hosts.studio)) }} +{{- fail (printf "studio.ingress.hosts must hold exactly one host, hosts.studio (%s)" .Values.hosts.studio) }} +{{- end }} +{{- $tlsSecret := ternary "libredb-tls-studio" .Values.tls.secrets.studio (eq .Values.tls.mode "generated") }} +{{- $tls := .Values.studio.ingress.tls | default list }} +{{- if or (ne (len $tls) 1) (ne (get (first $tls | default dict) "secretName" | default "") $tlsSecret) (ne (toJson (get (first $tls | default dict) "hosts" | default list)) (toJson (list .Values.hosts.studio))) }} +{{- fail (printf "studio.ingress.tls must hold one entry with secretName %s and hosts [%s]: the TLS Secret of hosts.studio under tls.mode %s" $tlsSecret .Values.hosts.studio .Values.tls.mode) }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/libredb-platform/templates/bootstrap-hook.yaml b/charts/libredb-platform/templates/bootstrap-hook.yaml new file mode 100644 index 0000000..f4e8132 --- /dev/null +++ b/charts/libredb-platform/templates/bootstrap-hook.yaml @@ -0,0 +1,269 @@ +{{- include "libredb-platform.guards" . }} +{{- /* +The bootstrap job (installer/src/kubernetes/bootstrap.ts), a pre-install and pre-upgrade hook: it generates the +secrets, the seal keys and the internal CA once, and writes every Secret and ConfigMap the workloads mount, before any +of them starts. Its ServiceAccount, Role and inputs exist only while it runs. The Secrets it writes carry no Helm +ownership, so they outlive `helm uninstall`: losing the seal keys loses every sealed database credential. +*/}} +{{- /* Also before a rollback, whose bootstrap refuses an older release than the installation ran (checkRelease). */}} +{{- $hook := dict "helm.sh/hook" "pre-install,pre-upgrade,pre-rollback" "helm.sh/hook-delete-policy" "before-hook-creation,hook-succeeded" }} +{{- $managedPg := include "libredb-platform.managedPostgres" . }} +{{- $managedKc := include "libredb-platform.managedKeycloak" . }} +{{- $secrets := list "libredb-secrets" "libredb-installer-state" "libredb-tls" "libredb-platform-env" "libredb-platform-migrate-env" "libredb-keycloak-config" "libredb-studio-network-env" .Values.studio.secrets.existingSecret }} +{{- if $managedPg }}{{ $secrets = concat $secrets (list "libredb-postgres-superuser" "libredb-postgres-tls" "libredb-postgres-provision") }}{{ end }} +{{- if not $managedPg }}{{ $secrets = append $secrets "libredb-postgres-dba-script" }}{{ end }} +{{- if $managedKc }}{{ $secrets = append $secrets "libredb-keycloak-env" }}{{ end }} +{{- if eq .Values.tls.mode "generated" }}{{ $secrets = concat $secrets (list "libredb-tls-platform" "libredb-tls-studio") }}{{ if $managedKc }}{{ $secrets = append $secrets "libredb-tls-auth" }}{{ end }}{{ end }} +{{- $configMaps := list "libredb-ca" "libredb-jobs" }} +{{- if $managedPg }}{{ $configMaps = append $configMaps "libredb-postgres-config" }}{{ end }} +{{- if $managedKc }}{{ $configMaps = append $configMaps "libredb-keycloak-edge" }}{{ end }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: libredb-bootstrap + labels: + {{- include "libredb-platform.labels" . | nindent 4 }} + annotations: + {{- toYaml $hook | nindent 4 }} + helm.sh/hook-weight: "-30" +automountServiceAccountToken: false +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: libredb-bootstrap + labels: + {{- include "libredb-platform.labels" . | nindent 4 }} + annotations: + {{- toYaml $hook | nindent 4 }} + helm.sh/hook-weight: "-30" +rules: + # create cannot be limited by name; reads and replaces are limited to the job's own objects. + - apiGroups: [""] + resources: ["secrets", "configmaps"] + verbs: ["create"] + - apiGroups: [""] + resources: ["secrets"] + resourceNames: {{ toJson $secrets }} + verbs: ["get", "update"] + - apiGroups: [""] + resources: ["configmaps"] + resourceNames: {{ toJson $configMaps }} + verbs: ["get", "update"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: libredb-bootstrap + labels: + {{- include "libredb-platform.labels" . | nindent 4 }} + annotations: + {{- toYaml $hook | nindent 4 }} + helm.sh/hook-weight: "-30" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: libredb-bootstrap +subjects: + - kind: ServiceAccount + name: libredb-bootstrap + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: libredb-bootstrap-chart + labels: + {{- include "libredb-platform.labels" . | nindent 4 }} + annotations: + {{- toYaml $hook | nindent 4 }} + helm.sh/hook-weight: "-20" +data: + installation.json: | + {{- include "libredb-platform.installation" . | nindent 4 }} + chart.json: | + {{- include "libredb-platform.chartSettings" . | nindent 4 }} + {{- if .Values.platform.sealKeys.retire }} + seal-envelopes.sql: | + {{- include "libredb-platform.sealEnvelopesSql" . | nindent 4 }} + {{- end }} + {{- with .Values.tls.caBundle }} + tls-ca.crt: {{ . | quote }} + {{- end }} + {{- with .Values.postgresql.external.caBundle }} + postgres-ca.crt: {{ . | quote }} + {{- end }} + {{- with .Values.keycloak.external.caBundle }} + keycloak-ca.crt: {{ . | quote }} + {{- end }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: libredb-bootstrap-jobs + labels: + {{- include "libredb-platform.labels" . | nindent 4 }} + annotations: + {{- toYaml $hook | nindent 4 }} + helm.sh/hook-weight: "-20" +binaryData: + libredb-jobs.cjs.gz: {{ .Files.Get "files/libredb-jobs.cjs.gz" | b64enc }} +{{- if .Values.platform.license.value }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: libredb-bootstrap-license + labels: + {{- include "libredb-platform.labels" . | nindent 4 }} + annotations: + {{- toYaml $hook | nindent 4 }} + helm.sh/hook-weight: "-20" +type: Opaque +stringData: + license: {{ trim .Values.platform.license.value | quote }} +{{- end }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: libredb-bootstrap + labels: + {{- include "libredb-platform.labels" . | nindent 4 }} + app.kubernetes.io/component: bootstrap + annotations: + {{- toYaml $hook | nindent 4 }} + helm.sh/hook-weight: "-10" +spec: + # A refusal a retry cannot change (exit 2 invalid installation, 3 failed precondition, 6 unsafe change, as the + # installer's exit codes) fails the Job at once, so Helm reports it instead of waiting out the retries. + podFailurePolicy: + rules: + - action: FailJob + onExitCodes: + containerName: bootstrap + operator: In + values: [2, 3, 6] + backoffLimit: {{ .Values.jobs.backoffLimit }} + activeDeadlineSeconds: {{ .Values.jobs.activeDeadlineSeconds }} + template: + metadata: + labels: + {{- include "libredb-platform.selectorLabels" (dict "root" . "component" "bootstrap") | nindent 8 }} + spec: + serviceAccountName: libredb-bootstrap + automountServiceAccountToken: true + restartPolicy: Never + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + securityContext: + {{- include "libredb-platform.podSecurity" (dict "root" $ "uid" 1001) | nindent 8 }} + {{- include "libredb-platform.platformScheduling" $ | nindent 6 }} + {{- if .Values.platform.sealKeys.retire }} + # A retirement needs the connections each retired kid still seals, which the bootstrap job reads from /tmp. + initContainers: + {{- include "libredb-platform.platformSqlContainer" (dict "root" . "name" "seal-envelopes" "file" "/etc/libredb/chart/seal-envelopes.sql" "output" "/tmp/seal-envelopes.out") | nindent 8 }} + {{- end }} + containers: + - name: bootstrap + image: {{ include "libredb-platform.image" .Values.platform.image }} + imagePullPolicy: {{ .Values.platform.image.pullPolicy }} + command: + {{- include "libredb-platform.jobCommand" (dict "command" "bootstrap" "withBundle" false) | nindent 12 }} + securityContext: + {{- include "libredb-platform.containerSecurity" . | nindent 12 }} + resources: + {{- toYaml .Values.jobs.resources | nindent 12 }} + volumeMounts: + - name: chart + mountPath: /etc/libredb/chart + readOnly: true + - name: jobs + mountPath: /etc/libredb/jobs + readOnly: true + - name: inputs-ca + mountPath: {{ include "libredb-platform.inputDir" . }}/ca + readOnly: true + - name: license + mountPath: {{ include "libredb-platform.inputDir" . }}/license + readOnly: true + {{- if .Values.platform.authSecretPrevious.existingSecret }} + - name: auth-secret-previous + mountPath: {{ include "libredb-platform.inputDir" . }}/auth-secret-previous + readOnly: true + {{- end }} + {{- if and (eq .Values.keycloak.mode "external") (eq .Values.keycloak.external.provisioning "manual") }} + - name: keycloak-clients + mountPath: {{ include "libredb-platform.inputDir" . }}/keycloak-clients + readOnly: true + {{- end }} + - name: tmp + mountPath: /tmp + volumes: + - name: chart + configMap: + name: libredb-bootstrap-chart + items: + - key: installation.json + path: installation.json + - key: chart.json + path: chart.json + {{- if .Values.platform.sealKeys.retire }} + - key: seal-envelopes.sql + path: seal-envelopes.sql + {{- end }} + - name: jobs + configMap: + name: libredb-bootstrap-jobs + - name: inputs-ca + configMap: + name: libredb-bootstrap-chart + optional: true + items: + {{- if .Values.tls.caBundle }} + - key: tls-ca.crt + path: tls-ca.crt + {{- end }} + {{- if .Values.postgresql.external.caBundle }} + - key: postgres-ca.crt + path: postgres-ca.crt + {{- end }} + {{- if .Values.keycloak.external.caBundle }} + - key: keycloak-ca.crt + path: keycloak-ca.crt + {{- end }} + - key: chart.json + path: .chart.json + - name: license + secret: + secretName: {{ .Values.platform.license.existingSecret | default "libredb-bootstrap-license" }} + items: + - key: {{ if .Values.platform.license.existingSecret }}{{ .Values.platform.license.key }}{{ else }}license{{ end }} + path: license + {{- with .Values.platform.authSecretPrevious.existingSecret }} + - name: auth-secret-previous + secret: + secretName: {{ . }} + items: + - key: {{ $.Values.platform.authSecretPrevious.key }} + path: auth-secret-previous + {{- end }} + {{- if .Values.platform.sealKeys.retire }} + - name: installation-ca + configMap: + name: libredb-ca + {{- end }} + {{- if and (eq .Values.keycloak.mode "external") (eq .Values.keycloak.external.provisioning "manual") }} + - name: keycloak-clients + secret: + secretName: {{ .Values.keycloak.external.clientSecretsSecret }} + items: + - key: studio-client-secret + path: studio-client-secret + - key: platform-client-secret + path: platform-client-secret + {{- end }} + - name: tmp + emptyDir: {} diff --git a/charts/libredb-platform/templates/jobs.yaml b/charts/libredb-platform/templates/jobs.yaml new file mode 100644 index 0000000..9f162a7 --- /dev/null +++ b/charts/libredb-platform/templates/jobs.yaml @@ -0,0 +1,377 @@ +{{- /* +The jobs that run after the bootstrap hook: Platform's migrate job and the keycloak-configure job. Each is named by a +hash of the release revision and what it runs, so every install and upgrade runs it again, as every installer apply +does, which is safe because both converge, and a job that failed is retried by the next upgrade; Helm removes the +previous revision's Job. Each retries until what it needs is up: the migrate job until PostgreSQL is provisioned, the keycloak +job until Keycloak answers. Platform's readiness (/api/health/ready) stays 503 until the migration ran. +*/}} +{{- $migrate := dict "revision" .Release.Revision "image" (include "libredb-platform.image" .Values.platform.image) "installation" (include "libredb-platform.installation" .) }} +apiVersion: batch/v1 +kind: Job +metadata: + name: libredb-platform-migrate-{{ include "libredb-platform.hash" (toJson $migrate) }} + labels: + {{- include "libredb-platform.labels" . | nindent 4 }} + app.kubernetes.io/component: platform-migrate + annotations: + # Argo CD renders with helm template, where every revision is 1: as a Sync hook replaced on each sync, the job + # runs again on every sync as on every helm upgrade. Helm ignores these annotations. + argocd.argoproj.io/hook: Sync + argocd.argoproj.io/hook-delete-policy: BeforeHookCreation +spec: + # The job retries until what it needs is up, so it gets more attempts than the bootstrap hook. + backoffLimit: {{ add .Values.jobs.backoffLimit 4 }} + activeDeadlineSeconds: {{ .Values.jobs.activeDeadlineSeconds }} + template: + metadata: + labels: + {{- include "libredb-platform.selectorLabels" (dict "root" . "component" "platform-migrate") | nindent 8 }} + spec: + automountServiceAccountToken: false + restartPolicy: Never + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + securityContext: + {{- include "libredb-platform.podSecurity" (dict "root" $ "uid" 1001) | nindent 8 }} + {{- include "libredb-platform.platformScheduling" $ | nindent 6 }} + initContainers: + # Waits until libredb_platform can sign in over verified TLS, which happens once the server is provisioned: + # by the provisioning job of a managed server, by the DBA's script on an external one. + - name: wait-for-provisioning + image: {{ include "libredb-platform.image" .Values.postgresql.managed.image }} + imagePullPolicy: {{ .Values.postgresql.managed.image.pullPolicy }} + command: + - /bin/sh + - -c + - | + # The password goes to libpq in PGPASSWORD, never on a command line; the bootstrap job generates it in + # base64url, which a URL carries unescaped. + rest="${DATABASE_URL#postgresql://libredb_platform:}" + export PGPASSWORD="${rest%%@*}" + address="${rest#*@}" + address="${address%%/*}" + conninfo="host=${address%:*} port=${address##*:} dbname=libredb user=libredb_platform sslmode=verify-full sslrootcert=/etc/libredb/ca/ca.crt connect_timeout=5" + until psql "$conninfo" -X -A -t -q -c 'SELECT 1' >/dev/null 2>&1; do echo "waiting until PostgreSQL is provisioned for libredb_platform"; sleep 3; done + env: + - name: DATABASE_URL + valueFrom: + secretKeyRef: + name: libredb-platform-migrate-env + key: DATABASE_URL + securityContext: + {{- include "libredb-platform.containerSecurity" . | nindent 12 }} + resources: + {{- toYaml .Values.jobs.resources | nindent 12 }} + volumeMounts: + - name: ca + mountPath: /etc/libredb/ca + readOnly: true + containers: + - name: migrate + image: {{ include "libredb-platform.image" .Values.platform.image }} + imagePullPolicy: {{ .Values.platform.image.pullPolicy }} + args: ["migrate"] + envFrom: + # The only place MIGRATE_DATABASE_URL, the owner role's URL, lives. + - secretRef: + name: libredb-platform-migrate-env + securityContext: + {{- include "libredb-platform.containerSecurity" . | nindent 12 }} + resources: + {{- toYaml .Values.jobs.resources | nindent 12 }} + volumeMounts: + - name: ca + mountPath: /etc/libredb/ca + readOnly: true + - name: tmp + mountPath: /tmp + volumes: + - name: ca + configMap: + name: libredb-ca + - name: tmp + emptyDir: {} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: libredb-chart + labels: + {{- include "libredb-platform.labels" . | nindent 4 }} +data: + installation.json: | + {{- include "libredb-platform.installation" . | nindent 4 }} + chart.json: | + {{- include "libredb-platform.chartSettings" . | nindent 4 }} + {{- if .Values.platform.sealKeys.retire }} + seal-canary.sql: | + {{- include "libredb-platform.sealCanarySql" . | nindent 4 }} + {{- end }} +--- +{{- /* +The restart job, a post-install, post-upgrade and post-rollback hook (Argo CD: PostSync): it restarts each workload +whose Secret or ConfigMap the bootstrap job changed, once Helm applied the release's pod templates, unless its pods +already started after the change. Its ServiceAccount and Role are regular objects, which exist by then. +*/}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: libredb-restart + labels: + {{- include "libredb-platform.labels" . | nindent 4 }} +automountServiceAccountToken: false +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: libredb-restart + labels: + {{- include "libredb-platform.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["secrets"] + resourceNames: ["libredb-installer-state"] + verbs: ["get", "update"] + - apiGroups: ["apps"] + resources: ["deployments"] + resourceNames: ["libredb-platform", "libredb-studio", "keycloak"] + verbs: ["get", "patch"] + - apiGroups: ["apps"] + resources: ["statefulsets"] + resourceNames: ["libredb-postgres"] + verbs: ["get", "patch"] + # Whether a workload's pods already started after the change. + - apiGroups: [""] + resources: ["pods"] + verbs: ["list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: libredb-restart + labels: + {{- include "libredb-platform.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: libredb-restart +subjects: + - kind: ServiceAccount + name: libredb-restart + namespace: {{ .Release.Namespace }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: libredb-restart + labels: + {{- include "libredb-platform.labels" . | nindent 4 }} + app.kubernetes.io/component: restart + annotations: + helm.sh/hook: post-install,post-upgrade,post-rollback + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded +spec: + backoffLimit: {{ .Values.jobs.backoffLimit }} + activeDeadlineSeconds: {{ .Values.jobs.activeDeadlineSeconds }} + template: + metadata: + labels: + {{- include "libredb-platform.selectorLabels" (dict "root" . "component" "restart") | nindent 8 }} + spec: + serviceAccountName: libredb-restart + automountServiceAccountToken: true + restartPolicy: Never + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + securityContext: + {{- include "libredb-platform.podSecurity" (dict "root" $ "uid" 1001) | nindent 8 }} + {{- include "libredb-platform.platformScheduling" $ | nindent 6 }} + {{- if .Values.platform.sealKeys.retire }} + # A retired kid's canary row would make every Studio heartbeat report UNKNOWN_KID for good, and Platform only + # inserts canaries; it goes before the restarts, so Studio starts without the kid and without its canary. + initContainers: + {{- include "libredb-platform.platformSqlContainer" (dict "root" . "name" "seal-canary" "file" "/etc/libredb/chart/seal-canary.sql") | nindent 8 }} + {{- end }} + containers: + - name: restart + image: {{ include "libredb-platform.image" .Values.platform.image }} + imagePullPolicy: {{ .Values.platform.image.pullPolicy }} + command: + {{- include "libredb-platform.jobCommand" (dict "command" "restart" "withBundle" false) | nindent 12 }} + securityContext: + {{- include "libredb-platform.containerSecurity" . | nindent 12 }} + resources: + {{- toYaml .Values.jobs.resources | nindent 12 }} + volumeMounts: + - name: chart + mountPath: /etc/libredb/chart + readOnly: true + - name: jobs + mountPath: /etc/libredb/jobs + readOnly: true + - name: tmp + mountPath: /tmp + volumes: + - name: chart + configMap: + name: libredb-chart + items: + - key: chart.json + path: chart.json + {{- if .Values.platform.sealKeys.retire }} + - key: seal-canary.sql + path: seal-canary.sql + {{- end }} + - name: jobs + configMap: + name: libredb-jobs + - name: tmp + emptyDir: {} + {{- if .Values.platform.sealKeys.retire }} + - name: installation-ca + configMap: + name: libredb-ca + {{- end }} +{{- if include "libredb-platform.keycloakJob" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: libredb-keycloak-configure + labels: + {{- include "libredb-platform.labels" . | nindent 4 }} +automountServiceAccountToken: false +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: libredb-keycloak-configure + labels: + {{- include "libredb-platform.labels" . | nindent 4 }} +rules: + # The installer state, which records the Keycloak accounts the job created. + - apiGroups: [""] + resources: ["secrets"] + resourceNames: ["libredb-installer-state"] + verbs: ["get", "update"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: libredb-keycloak-configure + labels: + {{- include "libredb-platform.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: libredb-keycloak-configure +subjects: + - kind: ServiceAccount + name: libredb-keycloak-configure + namespace: {{ .Release.Namespace }} +--- +{{- $keycloak := dict "revision" .Release.Revision "image" (include "libredb-platform.image" .Values.platform.image) "installation" (include "libredb-platform.installation" .) "jobs" (.Files.Get "files/libredb-jobs.cjs.gz" | sha256sum) }} +apiVersion: batch/v1 +kind: Job +metadata: + name: libredb-keycloak-configure-{{ include "libredb-platform.hash" (toJson $keycloak) }} + labels: + {{- include "libredb-platform.labels" . | nindent 4 }} + app.kubernetes.io/component: keycloak-configure + annotations: + # Argo CD renders with helm template, where every revision is 1: as a Sync hook replaced on each sync, the job + # runs again on every sync as on every helm upgrade. Helm ignores these annotations. + argocd.argoproj.io/hook: Sync + argocd.argoproj.io/hook-delete-policy: BeforeHookCreation +spec: + # A refusal a retry cannot change (exit 2 invalid installation, 3 failed precondition, 6 unsafe change, as the + # installer's exit codes) fails the Job at once, so Helm reports it instead of waiting out the retries. + podFailurePolicy: + rules: + - action: FailJob + onExitCodes: + containerName: keycloak-configure + operator: In + values: [2, 3, 6] + # The job retries until what it needs is up, so it gets more attempts than the bootstrap hook. + backoffLimit: {{ add .Values.jobs.backoffLimit 4 }} + activeDeadlineSeconds: {{ .Values.jobs.activeDeadlineSeconds }} + template: + metadata: + labels: + {{- include "libredb-platform.selectorLabels" (dict "root" . "component" "keycloak-configure") | nindent 8 }} + spec: + serviceAccountName: libredb-keycloak-configure + automountServiceAccountToken: true + restartPolicy: Never + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + securityContext: + {{- include "libredb-platform.podSecurity" (dict "root" $ "uid" 1001) | nindent 8 }} + {{- include "libredb-platform.platformScheduling" $ | nindent 6 }} + containers: + - name: keycloak-configure + image: {{ include "libredb-platform.image" .Values.platform.image }} + imagePullPolicy: {{ .Values.platform.image.pullPolicy }} + command: + {{- include "libredb-platform.jobCommand" (dict "command" "keycloak" "withBundle" true) | nindent 12 }} + securityContext: + {{- include "libredb-platform.containerSecurity" . | nindent 12 }} + resources: + {{- toYaml .Values.jobs.resources | nindent 12 }} + volumeMounts: + - name: chart + mountPath: /etc/libredb/chart + readOnly: true + - name: jobs + mountPath: /etc/libredb/jobs + readOnly: true + - name: keycloak-config + mountPath: /etc/libredb/keycloak-config + readOnly: true + - name: ca + mountPath: /etc/libredb/ca + readOnly: true + {{- if eq .Values.keycloak.mode "external" }} + - name: keycloak-admin + mountPath: {{ include "libredb-platform.inputDir" . }}/keycloak-admin + readOnly: true + {{- end }} + - name: tmp + mountPath: /tmp + volumes: + - name: chart + configMap: + name: libredb-chart + # Written by the bootstrap job from its hook ConfigMap, so the release carries the bundle once. + - name: jobs + configMap: + name: libredb-jobs + - name: keycloak-config + secret: + secretName: libredb-keycloak-config + - name: ca + configMap: + name: libredb-ca + {{- if eq .Values.keycloak.mode "external" }} + - name: keycloak-admin + secret: + secretName: {{ .Values.keycloak.external.adminSecret }} + items: + {{- if .Values.keycloak.external.adminClientId }} + - key: admin-client-secret + path: admin-client-secret + {{- else }} + - key: admin-password + path: admin-password + {{- end }} + {{- end }} + - name: tmp + emptyDir: {} +{{- end }} diff --git a/charts/libredb-platform/templates/keycloak.yaml b/charts/libredb-platform/templates/keycloak.yaml new file mode 100644 index 0000000..85f0e97 --- /dev/null +++ b/charts/libredb-platform/templates/keycloak.yaml @@ -0,0 +1,220 @@ +{{- if include "libredb-platform.managedKeycloak" . }} +{{- /* +The managed Keycloak, with the environment of the installer's compose service (installer/src/render/compose.ts), and +the installer's edge as a sidecar: the ingress reaches only the edge, which forwards /realms// and /resources/ +alone. Service keycloak serves the Admin API in-cluster at http://keycloak:8080, Platform's KEYCLOAK_ADMIN_URL; the +admin console opens through `kubectl port-forward svc/keycloak 18080:8080` at http://127.0.0.1:18080, the URL +KC_HOSTNAME_ADMIN names, as on a Docker install. +*/}} +{{- $kc := .Values.keycloak.managed }} +apiVersion: v1 +kind: Service +metadata: + name: keycloak + labels: + {{- include "libredb-platform.labels" . | nindent 4 }} + app.kubernetes.io/component: keycloak +spec: + selector: + {{- include "libredb-platform.selectorLabels" (dict "root" . "component" "keycloak") | nindent 4 }} + ports: + - name: http + port: 8080 + targetPort: http +--- +apiVersion: v1 +kind: Service +metadata: + name: keycloak-edge + labels: + {{- include "libredb-platform.labels" . | nindent 4 }} + app.kubernetes.io/component: keycloak +spec: + selector: + {{- include "libredb-platform.selectorLabels" (dict "root" . "component" "keycloak") | nindent 4 }} + ports: + - name: edge + port: 8081 + targetPort: edge +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: keycloak + labels: + {{- include "libredb-platform.labels" . | nindent 4 }} + app.kubernetes.io/component: keycloak +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + {{- include "libredb-platform.selectorLabels" (dict "root" . "component" "keycloak") | nindent 6 }} + template: + metadata: + labels: + {{- include "libredb-platform.selectorLabels" (dict "root" . "component" "keycloak") | nindent 8 }} + spec: + automountServiceAccountToken: false + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + securityContext: + {{- include "libredb-platform.podSecurity" (dict "root" $ "uid" 1000) | nindent 8 }} + containers: + - name: keycloak + image: {{ include "libredb-platform.image" $kc.image }} + imagePullPolicy: {{ $kc.image.pullPolicy }} + args: ["start"] + env: + - name: KC_DB + value: postgres + - name: KC_DB_URL + value: "jdbc:postgresql://{{ if include "libredb-platform.managedPostgres" . }}libredb-postgres:5432{{ else }}{{ .Values.postgresql.external.host }}:{{ .Values.postgresql.external.port }}{{ end }}/keycloak?sslmode=verify-full&sslrootcert=/etc/libredb/ca/ca.crt" + - name: KC_DB_USERNAME + value: libredb_keycloak + - name: KC_DB_POOL_MAX_SIZE + value: "20" + - name: KC_HOSTNAME + value: "https://{{ .Values.hosts.auth }}" + - name: KC_HOSTNAME_ADMIN + value: "http://127.0.0.1:18080" + - name: KC_PROXY_HEADERS + value: xforwarded + - name: KC_HTTP_ENABLED + value: "true" + - name: KC_HEALTH_ENABLED + value: "true" + - name: KC_TRUSTSTORE_PATHS + value: /etc/libredb/ca/ca.crt + - name: LIQUIBASE_ANALYTICS_ENABLED + value: "false" + envFrom: + - secretRef: + name: libredb-keycloak-env + ports: + - name: http + containerPort: 8080 + - name: management + containerPort: 9000 + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + startupProbe: + httpGet: + path: /health/started + port: management + periodSeconds: 5 + failureThreshold: 60 + readinessProbe: + httpGet: + path: /health/ready + port: management + periodSeconds: 10 + livenessProbe: + httpGet: + path: /health/live + port: management + periodSeconds: 20 + resources: + {{- toYaml $kc.resources | nindent 12 }} + volumeMounts: + - name: ca + mountPath: /etc/libredb/ca + readOnly: true + - name: edge + image: {{ include "libredb-platform.image" $kc.edge.image }} + imagePullPolicy: {{ $kc.edge.image.pullPolicy }} + args: ["caddy", "run", "--config", "/etc/caddy/Caddyfile", "--adapter", "caddyfile"] + env: + - name: XDG_DATA_HOME + value: /tmp/caddy/data + - name: XDG_CONFIG_HOME + value: /tmp/caddy/config + ports: + - name: edge + containerPort: 8081 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + capabilities: + drop: ["ALL"] + # The image's caddy binary carries the file capability cap_net_bind_service, and the kernel refuses to + # exec it when that capability is outside the bounding set; the edge itself listens on 8081. + add: ["NET_BIND_SERVICE"] + seccompProfile: + type: RuntimeDefault + readinessProbe: + tcpSocket: + port: edge + periodSeconds: 10 + resources: + {{- toYaml $kc.edge.resources | nindent 12 }} + volumeMounts: + - name: edge-config + mountPath: /etc/caddy + readOnly: true + - name: edge-tmp + mountPath: /tmp + volumes: + - name: ca + configMap: + name: libredb-ca + - name: edge-config + configMap: + name: libredb-keycloak-edge + - name: edge-tmp + emptyDir: {} + {{- with $kc.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $kc.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $kc.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- if .Values.ingress.enabled }} +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: keycloak + labels: + {{- include "libredb-platform.labels" . | nindent 4 }} + app.kubernetes.io/component: keycloak + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .Values.ingress.className }} + ingressClassName: {{ . }} + {{- end }} + tls: + - hosts: + - {{ .Values.hosts.auth | quote }} + secretName: {{ if eq .Values.tls.mode "generated" }}libredb-tls-auth{{ else }}{{ .Values.tls.secrets.auth }}{{ end }} + rules: + - host: {{ .Values.hosts.auth | quote }} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: keycloak-edge + port: + name: edge +{{- end }} +{{- end }} diff --git a/charts/libredb-platform/templates/networkpolicy.yaml b/charts/libredb-platform/templates/networkpolicy.yaml new file mode 100644 index 0000000..ba27039 --- /dev/null +++ b/charts/libredb-platform/templates/networkpolicy.yaml @@ -0,0 +1,52 @@ +{{- if .Values.networkPolicy.enabled }} +{{- /* +The installer's internal data network: the managed PostgreSQL server accepts connections only from this release's +pods, and Keycloak's port 8080 (the Admin API) only from them; the ingress reaches Keycloak through its edge alone. +*/}} +{{- if include "libredb-platform.managedPostgres" . }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: libredb-postgres + labels: + {{- include "libredb-platform.labels" . | nindent 4 }} +spec: + podSelector: + matchLabels: + {{- include "libredb-platform.selectorLabels" (dict "root" . "component" "postgres") | nindent 6 }} + policyTypes: ["Ingress"] + ingress: + - from: + - podSelector: + matchLabels: + app.kubernetes.io/instance: {{ .Release.Name }} + ports: + - port: 5432 +--- +{{- end }} +{{- if include "libredb-platform.managedKeycloak" . }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: keycloak + labels: + {{- include "libredb-platform.labels" . | nindent 4 }} +spec: + podSelector: + matchLabels: + {{- include "libredb-platform.selectorLabels" (dict "root" . "component" "keycloak") | nindent 6 }} + policyTypes: ["Ingress"] + ingress: + # Platform, Studio and the jobs reach the Admin API and the token endpoint in-cluster. + - from: + - podSelector: + matchLabels: + app.kubernetes.io/instance: {{ .Release.Name }} + ports: + - port: 8080 + # Anyone may reach the edge, which forwards only the realm's public paths; restrict it to your ingress + # controller's namespace with an extra policy if you like. + - ports: + - port: 8081 +{{- end }} +{{- end }} diff --git a/charts/libredb-platform/templates/platform.yaml b/charts/libredb-platform/templates/platform.yaml new file mode 100644 index 0000000..305f3df --- /dev/null +++ b/charts/libredb-platform/templates/platform.yaml @@ -0,0 +1,172 @@ +{{- /* +Platform's `serve` (docs/deploy/platform-runtime.md), with the environment the bootstrap job rendered with the +installer's renderPlatformEnv. It never receives MIGRATE_DATABASE_URL: that lives only in the migrate job's Secret. +This release runs one replica. +*/}} +{{- $p := .Values.platform }} +{{- if and $p.persistence.enabled }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: libredb-platform-uploads + labels: + {{- include "libredb-platform.labels" . | nindent 4 }} + app.kubernetes.io/component: platform + annotations: + helm.sh/resource-policy: keep +spec: + accessModes: + {{- toYaml $p.persistence.accessModes | nindent 4 }} + {{- with $p.persistence.storageClass }} + storageClassName: {{ . | quote }} + {{- end }} + resources: + requests: + storage: {{ $p.persistence.size }} +--- +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: libredb-platform + labels: + {{- include "libredb-platform.labels" . | nindent 4 }} + app.kubernetes.io/component: platform +spec: + selector: + {{- include "libredb-platform.selectorLabels" (dict "root" . "component" "platform") | nindent 4 }} + ports: + - name: http + port: 3000 + targetPort: http +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: libredb-platform + labels: + {{- include "libredb-platform.labels" . | nindent 4 }} + app.kubernetes.io/component: platform +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + {{- include "libredb-platform.selectorLabels" (dict "root" . "component" "platform") | nindent 6 }} + template: + metadata: + labels: + {{- include "libredb-platform.selectorLabels" (dict "root" . "component" "platform") | nindent 8 }} + {{- with $p.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- /* The bootstrap job restarts Platform when its environment or CA bundle changed. */}} + {{- with $p.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + automountServiceAccountToken: false + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + securityContext: + {{- include "libredb-platform.podSecurity" (dict "root" $ "uid" 1001) | nindent 8 }} + containers: + - name: platform + image: {{ include "libredb-platform.image" $p.image }} + imagePullPolicy: {{ $p.image.pullPolicy }} + args: ["serve"] + envFrom: + - secretRef: + name: libredb-platform-env + ports: + - name: http + containerPort: 3000 + securityContext: + {{- include "libredb-platform.containerSecurity" . | nindent 12 }} + startupProbe: + httpGet: + path: /api/health/live + port: http + periodSeconds: 5 + failureThreshold: 60 + livenessProbe: + httpGet: + path: /api/health/live + port: http + periodSeconds: 20 + timeoutSeconds: 5 + readinessProbe: + httpGet: + path: /api/health/ready + port: http + periodSeconds: 10 + timeoutSeconds: 5 + resources: + {{- toYaml $p.resources | nindent 12 }} + volumeMounts: + - name: ca + mountPath: /etc/libredb/ca + readOnly: true + - name: uploads + mountPath: /app/uploads + - name: tmp + mountPath: /tmp + - name: next-cache + mountPath: /app/.next/cache + volumes: + - name: ca + configMap: + name: libredb-ca + - name: uploads + {{- if $p.persistence.enabled }} + persistentVolumeClaim: + claimName: libredb-platform-uploads + {{- else }} + emptyDir: {} + {{- end }} + - name: tmp + emptyDir: {} + - name: next-cache + emptyDir: {} + {{- include "libredb-platform.platformScheduling" . | nindent 6 }} + {{- with $p.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- if .Values.ingress.enabled }} +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: libredb-platform + labels: + {{- include "libredb-platform.labels" . | nindent 4 }} + app.kubernetes.io/component: platform + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .Values.ingress.className }} + ingressClassName: {{ . }} + {{- end }} + tls: + - hosts: + - {{ .Values.hosts.platform | quote }} + secretName: {{ if eq .Values.tls.mode "generated" }}libredb-tls-platform{{ else }}{{ .Values.tls.secrets.platform }}{{ end }} + rules: + - host: {{ .Values.hosts.platform | quote }} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: libredb-platform + port: + name: http +{{- end }} diff --git a/charts/libredb-platform/templates/postgres.yaml b/charts/libredb-platform/templates/postgres.yaml new file mode 100644 index 0000000..b9a9da3 --- /dev/null +++ b/charts/libredb-platform/templates/postgres.yaml @@ -0,0 +1,223 @@ +{{- if include "libredb-platform.managedPostgres" . }} +{{- /* +The managed PostgreSQL server, as the installer's compose service: TLS with the internal CA's leaf, a pg_hba.conf that +accepts only hostssl with SCRAM, and the superuser password from the bootstrap job. The provisioning job then runs the +installer's provisioning script (installer/src/sql/provision.ts), which sends SCRAM verifiers only. +*/}} +{{- $pg := .Values.postgresql.managed }} +apiVersion: v1 +kind: Service +metadata: + name: libredb-postgres + labels: + {{- include "libredb-platform.labels" . | nindent 4 }} + app.kubernetes.io/component: postgres +spec: + selector: + {{- include "libredb-platform.selectorLabels" (dict "root" . "component" "postgres") | nindent 4 }} + ports: + - name: postgres + port: 5432 + targetPort: postgres +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: libredb-postgres + labels: + {{- include "libredb-platform.labels" . | nindent 4 }} + app.kubernetes.io/component: postgres +spec: + serviceName: libredb-postgres + replicas: 1 + selector: + matchLabels: + {{- include "libredb-platform.selectorLabels" (dict "root" . "component" "postgres") | nindent 6 }} + template: + metadata: + labels: + {{- include "libredb-platform.selectorLabels" (dict "root" . "component" "postgres") | nindent 8 }} + spec: + automountServiceAccountToken: false + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + securityContext: + {{- include "libredb-platform.podSecurity" (dict "root" $ "uid" 999) | nindent 8 }} + containers: + - name: postgres + image: {{ include "libredb-platform.image" $pg.image }} + imagePullPolicy: {{ $pg.image.pullPolicy }} + args: + - postgres + - -c + - ssl=on + - -c + - ssl_cert_file=/etc/libredb/tls/server.crt + - -c + - ssl_key_file=/etc/libredb/tls/server.key + - -c + - hba_file=/etc/libredb/postgres/pg_hba.conf + - -c + - password_encryption=scram-sha-256 + env: + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: libredb-postgres-superuser + key: POSTGRES_PASSWORD + - name: POSTGRES_INITDB_ARGS + value: --auth-host=scram-sha-256 --auth-local=scram-sha-256 + ports: + - name: postgres + containerPort: 5432 + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + readinessProbe: + exec: + command: ["pg_isready", "-h", "127.0.0.1", "-U", "postgres", "-d", "postgres"] + periodSeconds: 10 + timeoutSeconds: 5 + livenessProbe: + exec: + command: ["pg_isready", "-h", "127.0.0.1", "-U", "postgres", "-d", "postgres"] + initialDelaySeconds: 60 + periodSeconds: 20 + timeoutSeconds: 5 + resources: + {{- toYaml $pg.resources | nindent 12 }} + volumeMounts: + - name: data + mountPath: /var/lib/postgresql + - name: tls + mountPath: /etc/libredb/tls + readOnly: true + - name: config + mountPath: /etc/libredb/postgres + readOnly: true + - name: run + mountPath: /var/run/postgresql + volumes: + - name: tls + secret: + secretName: libredb-postgres-tls + # PostgreSQL accepts a key owned by root only with mode 0640 or stricter. + defaultMode: 0640 + - name: config + configMap: + name: libredb-postgres-config + - name: run + emptyDir: {} + {{- with $pg.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $pg.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $pg.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + volumeClaimTemplates: + # A StatefulSet's volume claims outlive the StatefulSet, and helm uninstall, by Kubernetes' own rule. + - metadata: + name: data + spec: + accessModes: ["ReadWriteOnce"] + {{- with $pg.persistence.storageClass }} + storageClassName: {{ . | quote }} + {{- end }} + resources: + requests: + storage: {{ $pg.persistence.size }} +--- +{{- $provision := dict "revision" .Release.Revision "image" (include "libredb-platform.image" $pg.image) "installation" (include "libredb-platform.installation" .) }} +apiVersion: batch/v1 +kind: Job +metadata: + name: libredb-postgres-provision-{{ include "libredb-platform.hash" (toJson $provision) }} + labels: + {{- include "libredb-platform.labels" . | nindent 4 }} + app.kubernetes.io/component: postgres-provision + annotations: + # Argo CD renders with helm template, where every revision is 1: as a Sync hook replaced on each sync, the job + # runs again on every sync as on every helm upgrade. Helm ignores these annotations. + argocd.argoproj.io/hook: Sync + argocd.argoproj.io/hook-delete-policy: BeforeHookCreation +spec: + # The job retries until what it needs is up, so it gets more attempts than the bootstrap hook. + backoffLimit: {{ add .Values.jobs.backoffLimit 4 }} + activeDeadlineSeconds: {{ .Values.jobs.activeDeadlineSeconds }} + template: + metadata: + labels: + {{- include "libredb-platform.selectorLabels" (dict "root" . "component" "postgres-provision") | nindent 8 }} + spec: + automountServiceAccountToken: false + restartPolicy: Never + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + securityContext: + {{- include "libredb-platform.podSecurity" (dict "root" $ "uid" 999) | nindent 8 }} + initContainers: + {{- include "libredb-platform.waitForPostgres" . | nindent 8 }} + containers: + - name: provision + image: {{ include "libredb-platform.image" $pg.image }} + imagePullPolicy: {{ $pg.image.pullPolicy }} + # The installer's provisioning script as the superuser, over verified TLS; it converges an existing server. + # A renewed leaf restarts the server instead: the bootstrap job restarts it when its leaf Secret changed. + command: ["psql", "-X", "-v", "ON_ERROR_STOP=1", "-f", "/etc/libredb/provision/provision.sql"] + env: + - name: PGHOST + value: libredb-postgres + - name: PGPORT + value: "5432" + - name: PGUSER + value: postgres + - name: PGDATABASE + value: postgres + - name: PGSSLMODE + value: verify-full + - name: PGSSLROOTCERT + value: /etc/libredb/ca/ca.crt + - name: PGCONNECT_TIMEOUT + value: "10" + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: libredb-postgres-superuser + key: POSTGRES_PASSWORD + securityContext: + {{- include "libredb-platform.containerSecurity" . | nindent 12 }} + resources: + {{- toYaml .Values.jobs.resources | nindent 12 }} + volumeMounts: + - name: provision + mountPath: /etc/libredb/provision + readOnly: true + - name: ca + mountPath: /etc/libredb/ca + readOnly: true + - name: tmp + mountPath: /tmp + volumes: + - name: provision + secret: + secretName: libredb-postgres-provision + - name: ca + configMap: + name: libredb-ca + - name: tmp + emptyDir: {} +{{- end }} diff --git a/charts/libredb-platform/tests/e2e/coredns-rewrite.sh b/charts/libredb-platform/tests/e2e/coredns-rewrite.sh new file mode 100755 index 0000000..4c8b5d5 --- /dev/null +++ b/charts/libredb-platform/tests/e2e/coredns-rewrite.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Makes every *.libredb.test name resolve, inside the kind cluster, to the Traefik Service, the way a real site's DNS +# points the three hosts at its ingress: Platform and Studio call Keycloak's public issuer URL from their pods. +set -euo pipefail +corefile=$(kubectl -n kube-system get configmap coredns -o jsonpath='{.data.Corefile}') +if grep -q 'libredb\\.test' <<<"$corefile"; then + echo "CoreDNS already rewrites *.libredb.test" + exit 0 +fi +rewritten=$(sed 's|^\( ready\)$|\1\n rewrite name regex (.*)\\.libredb\\.test traefik.traefik.svc.cluster.local answer auto|' <<<"$corefile") +kubectl -n kube-system create configmap coredns --from-literal=Corefile="$rewritten" --dry-run=client -o yaml | kubectl -n kube-system apply -f - >/dev/null +kubectl -n kube-system rollout restart deployment coredns >/dev/null +kubectl -n kube-system rollout status deployment coredns --timeout=120s diff --git a/charts/libredb-platform/tests/e2e/kind-config.yaml b/charts/libredb-platform/tests/e2e/kind-config.yaml new file mode 100644 index 0000000..63f1bee --- /dev/null +++ b/charts/libredb-platform/tests/e2e/kind-config.yaml @@ -0,0 +1,17 @@ +# The kind cluster of the end-to-end test: Traefik's node ports reach the host on 127.0.0.1:80 and 127.0.0.1:443. +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +name: libredb-helm +nodes: + - role: control-plane + labels: + ingress-ready: "true" + extraPortMappings: + - containerPort: 30443 + hostPort: 443 + listenAddress: "127.0.0.1" + protocol: TCP + - containerPort: 30080 + hostPort: 80 + listenAddress: "127.0.0.1" + protocol: TCP diff --git a/charts/libredb-platform/tests/e2e/run-argocd.sh b/charts/libredb-platform/tests/e2e/run-argocd.sh new file mode 100755 index 0000000..6329ac4 --- /dev/null +++ b/charts/libredb-platform/tests/e2e/run-argocd.sh @@ -0,0 +1,153 @@ +#!/usr/bin/env bash +# The Argo CD test of the chart, on the kind cluster run-e2e.sh set up (Traefik, the *.libredb.test DNS), with no Helm +# release of the chart in the cluster, since it uses the same hosts: +# +# LIBREDB_E2E_LICENSE=kind.license LIBREDB_E2E_CHART_REPO=/dir/with/index.yaml/and/tgz \ +# charts/libredb-platform/tests/e2e/run-argocd.sh +# +# It installs Argo CD with label resource tracking, the method under which a tool would prune the objects the jobs +# write if they carried the release's instance label, serves the packaged chart from a Helm repository in the cluster, +# and syncs an Application with automated prune and self-heal into a namespace that enforces the restricted Pod +# Security Standard. It checks that the app turns Synced and Healthy, the endpoints answer, the jobs' Secrets survive +# the prune of every sync, a new sync runs the jobs again without touching the seal keys, and a changed value reaches +# the running Platform through the PostSync restart job. +set -euo pipefail + +here=$(cd "$(dirname "$0")" && pwd) +license=${LIBREDB_E2E_LICENSE:?a license for platform.libredb.test} +repo=${LIBREDB_E2E_CHART_REPO:?a directory with the packaged chart and its index.yaml} +argocd_version=${LIBREDB_E2E_ARGOCD_VERSION:-v3.5.5} +namespace=libredb-argo +# The chart's own version, at the indent of an index entry, not a dependency's. +version=$(awk '/^ version:/{print $2; exit}' "$repo/index.yaml" | tr -d '"') + +step() { printf '\n=== %s\n' "$*"; } +fail() { printf 'FAIL: %s\n' "$*" >&2; exit 1; } +app() { kubectl -n argocd get application libredb -o "jsonpath=$1"; } +ready() { curl -s --resolve "$1:443:127.0.0.1" -k -o /dev/null -w '%{http_code}' "https://$1$2"; } +hash_of() { kubectl -n "$namespace" get secret "$1" -o "jsonpath={.data.$2}" | sha256sum | cut -d' ' -f1; } +wait_synced() { + for _ in $(seq 1 180); do + if [ "$(app '{.status.sync.status}')" = "Synced" ] && [ "$(app '{.status.health.status}')" = "Healthy" ] \ + && [ "$(app '{.status.operationState.phase}')" = "Succeeded" ]; then return; fi + sleep 5 + done + fail "the app is $(app '{.status.sync.status}')/$(app '{.status.health.status}'), operation $(app '{.status.operationState.phase}'): $(app '{.status.operationState.message}')" +} + +step "Argo CD $argocd_version with label resource tracking" +kubectl create namespace argocd --dry-run=client -o yaml | kubectl apply -f - >/dev/null +# LIBREDB_E2E_ARGOCD_MANIFEST names a downloaded install.yaml, for a cluster whose nodes reach no registry or GitHub. +kubectl apply -n argocd --server-side --force-conflicts \ + -f "${LIBREDB_E2E_ARGOCD_MANIFEST:-https://raw.githubusercontent.com/argoproj/argo-cd/$argocd_version/manifests/install.yaml}" >/dev/null +kubectl -n argocd patch configmap argocd-cm --type merge -p '{"data":{"application.resourceTrackingMethod":"label"}}' >/dev/null +kubectl -n argocd rollout restart statefulset/argocd-application-controller >/dev/null +for deploy in argocd-repo-server argocd-server; do kubectl -n argocd rollout status "deploy/$deploy" --timeout=300s >/dev/null; done +kubectl -n argocd rollout status statefulset/argocd-application-controller --timeout=300s >/dev/null + +step "a Helm repository in the cluster serving libredb-platform $version" +kubectl create namespace chartrepo --dry-run=client -o yaml | kubectl apply -f - >/dev/null +kubectl -n chartrepo create configmap charts --from-file="$repo" --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts -f - >/dev/null +kubectl -n chartrepo apply -f - >/dev/null <<'EOF' +apiVersion: apps/v1 +kind: Deployment +metadata: {name: chartrepo} +spec: + selector: {matchLabels: {app: chartrepo}} + template: + metadata: {labels: {app: chartrepo}} + spec: + containers: + - name: httpd + image: busybox:1.37 + command: ["httpd", "-f", "-p", "8080", "-h", "/srv"] + volumeMounts: [{name: charts, mountPath: /srv}] + volumes: [{name: charts, configMap: {name: charts}}] +--- +apiVersion: v1 +kind: Service +metadata: {name: chartrepo} +spec: {selector: {app: chartrepo}, ports: [{port: 8080}]} +EOF +kubectl -n chartrepo rollout restart deploy/chartrepo >/dev/null +kubectl -n chartrepo rollout status deploy/chartrepo --timeout=120s >/dev/null + +step "the Application: automated sync with prune and self-heal into $namespace" +kubectl create namespace "$namespace" --dry-run=client -o yaml | kubectl apply -f - >/dev/null +kubectl label namespace "$namespace" pod-security.kubernetes.io/enforce=restricted --overwrite >/dev/null +kubectl -n "$namespace" create secret generic libredb-license --from-file=license="$license" --dry-run=client -o yaml | kubectl apply -f - >/dev/null +application() { + cat </dev/null +started=$(date +%s) +wait_synced +echo "Synced and Healthy in $(( $(date +%s) - started )) s" + +step "endpoints, license and the jobs' Secrets" +[ "$(ready platform.libredb.test /api/health/ready)" = "200" ] || fail "Platform is not ready" +[ "$(ready studio.libredb.test /api/ready)" = "200" ] || fail "Studio is not ready" +[ "$(ready auth.libredb.test /realms/libredb/.well-known/openid-configuration)" = "200" ] || fail "Keycloak's realm does not answer" +kubectl -n "$namespace" logs deploy/libredb-platform | grep -c '"message":"License active"' >/dev/null || fail "Platform did not log License active" +seal=$(hash_of libredb-secrets 'seal-keys\.json') +annotations=$(kubectl -n "$namespace" get secret libredb-secrets -o 'jsonpath={.metadata.annotations}') +grep -q 'Prune=false' <<<"$annotations" || fail "libredb-secrets lacks Argo CD's Prune=false: $annotations" +labels=$(kubectl -n "$namespace" get secret libredb-secrets -o 'jsonpath={.metadata.labels}') +grep -q 'app.kubernetes.io/instance' <<<"$labels" && fail "libredb-secrets carries the tracking label: $labels" +echo "Platform, Studio and Keycloak answer, license active, the jobs' Secrets carry no tracking label" + +step "a new sync with prune runs the jobs again and keeps every Secret of the jobs" +migrate=$(kubectl -n "$namespace" get jobs -l app.kubernetes.io/component=platform-migrate -o 'jsonpath={.items[0].metadata.uid}') +kubectl -n argocd patch application libredb --type merge \ + -p "{\"operation\":{\"initiatedBy\":{\"username\":\"e2e\"},\"sync\":{\"revision\":\"$version\",\"prune\":true}}}" >/dev/null +sleep 5 +wait_synced +again=$(kubectl -n "$namespace" get jobs -l app.kubernetes.io/component=platform-migrate -o 'jsonpath={.items[0].metadata.uid}') +[ -n "$again" ] && [ "$again" != "$migrate" ] || fail "the sync did not run the migrate job again" +for secret in libredb-secrets libredb-tls libredb-installer-state libredb-platform-env libredb-studio-env; do + kubectl -n "$namespace" get secret "$secret" >/dev/null || fail "the sync pruned Secret $secret" +done +[ "$(hash_of libredb-secrets 'seal-keys\.json')" = "$seal" ] || fail "the sync changed the seal keys" +echo "the migrate job ran again, every Secret of the jobs is there, the seal keys unchanged" + +step "a changed value reaches the running Platform through the PostSync restart job" +application "queryLogRetentionDays: 30" | kubectl apply -f - >/dev/null +for _ in $(seq 1 60); do + [ "$(kubectl -n "$namespace" get secret libredb-platform-env -o 'jsonpath={.data.QUERY_LOG_RETENTION_DAYS}' | base64 -d)" = "30" ] && break + sleep 5 +done +wait_synced +kubectl -n "$namespace" rollout status deploy/libredb-platform --timeout=300s >/dev/null +running=$(kubectl -n "$namespace" exec deploy/libredb-platform -- printenv QUERY_LOG_RETENTION_DAYS) +[ "$running" = "30" ] || fail "the running Platform has QUERY_LOG_RETENTION_DAYS=$running" +[ "$(hash_of libredb-secrets 'seal-keys\.json')" = "$seal" ] || fail "the change changed the seal keys" +echo "the running Platform has QUERY_LOG_RETENTION_DAYS=30, the seal keys unchanged" + +printf '\nPASS\n' diff --git a/charts/libredb-platform/tests/e2e/run-e2e.sh b/charts/libredb-platform/tests/e2e/run-e2e.sh new file mode 100755 index 0000000..fce2dd3 --- /dev/null +++ b/charts/libredb-platform/tests/e2e/run-e2e.sh @@ -0,0 +1,158 @@ +#!/usr/bin/env bash +# The end-to-end test of the Helm chart on a local kind cluster, which it creates when missing: +# +# LIBREDB_E2E_LICENSE=kind.license charts/libredb-platform/tests/e2e/run-e2e.sh +# +# The license must name the host platform.libredb.test. The test installs the chart with a managed PostgreSQL and +# Keycloak, checks the endpoints and the edge's path rules against the internal CA, the license, Studio's heartbeat +# and its seal canary, runs the browser suite (installer/e2e/kubernetes), upgrades with a changed value and checks that +# the seal keys and the CA stay, and finally uninstalls and installs again and checks the same. It uses only the kind +# cluster it names; it never touches other containers or volumes of the host's Docker. +# Needs kind, kubectl, helm, curl and bun on PATH. LIBREDB_E2E_KEEP=1 keeps the installation running at the end. +set -euo pipefail + +here=$(cd "$(dirname "$0")" && pwd) +chart=$(cd "$here/../.." && pwd) +installer=$(cd "$chart/../../installer" && pwd) +license=${LIBREDB_E2E_LICENSE:?set LIBREDB_E2E_LICENSE to a license file for platform.libredb.test} +cluster=${LIBREDB_E2E_CLUSTER:-libredb-helm} +namespace=${LIBREDB_E2E_NAMESPACE:-libredb} +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT + +step() { printf '\n=== %s\n' "$*"; } +fail() { printf 'FAIL: %s\n' "$*" >&2; exit 1; } + +install_chart() { + helm upgrade --install libredb "$chart" -n "$namespace" --create-namespace -f "$here/values-kind.yaml" \ + --set-file platform.license.value="$license" "$@" --wait --wait-for-jobs --timeout 10m +} + +secret_hash() { kubectl -n "$namespace" get secret "$1" -o "jsonpath={.data.$2}" | sha256sum | cut -d' ' -f1; } + +if [ "${LIBREDB_E2E_EXISTING_CLUSTER:-0}" = "1" ]; then + # A cluster this script set up before, reached through KUBECONFIG, such as from a container on kind's network. + step "the cluster of KUBECONFIG, set up earlier" +else + step "kind cluster $cluster" + if ! kind get clusters | grep -qx "$cluster"; then + sed "s/^name: .*/name: $cluster/" "$here/kind-config.yaml" > "$work/kind-config.yaml" + kind create cluster --config "$work/kind-config.yaml" --wait 180s + fi + kubectl config use-context "kind-$cluster" >/dev/null + + step "Traefik and the in-cluster DNS of *.libredb.test" + helm repo add traefik https://traefik.github.io/charts >/dev/null 2>&1 || true + helm repo update traefik >/dev/null + helm upgrade --install traefik traefik/traefik --version 41.7.1 -n traefik --create-namespace \ + -f "$here/traefik-values.yaml" --wait --timeout 5m >/dev/null + "$here/coredns-rewrite.sh" +fi + +step "chart dependencies" +if [ ! -f "$chart/charts/libredb-studio-0.1.100.tgz" ]; then + # helm dependency build takes Chart.lock's Studio chart only from a repository Helm knows. + helm repo add libredb-studio https://libredb.org/libredb-studio/ >/dev/null 2>&1 || true + helm dependency build "$chart" +fi + +step "install into a namespace that enforces the restricted Pod Security Standard" +kubectl create namespace "$namespace" --dry-run=client -o yaml | kubectl apply -f - >/dev/null +kubectl label namespace "$namespace" pod-security.kubernetes.io/enforce=restricted --overwrite >/dev/null +started=$(date +%s) +install_chart >/dev/null +echo "installed in $(( $(date +%s) - started )) s" +errors=$(kubectl -n "$namespace" get pods --no-headers | awk '$3 == "Error" || $3 == "CrashLoopBackOff"' | wc -l) +[ "$errors" -eq 0 ] || fail "$errors pods failed during the install" + +step "endpoints, the edge's path rules and TLS against the internal CA" +kubectl -n "$namespace" get secret libredb-tls-platform -o 'jsonpath={.data.ca\.crt}' | base64 -d > "$work/ca.crt" +resolve=(--resolve platform.libredb.test:443:127.0.0.1 --resolve studio.libredb.test:443:127.0.0.1 --resolve auth.libredb.test:443:127.0.0.1) +expect_status() { + local status + status=$(curl -s "${resolve[@]}" --cacert "$work/ca.crt" --path-as-is -o /dev/null -w '%{http_code}' "$2") + [ "$status" = "$1" ] || fail "$2 answered $status, not $1" + printf '%s %s\n' "$status" "$2" +} +expect_status 200 https://platform.libredb.test/api/health/ready +expect_status 200 https://studio.libredb.test/api/ready +expect_status 200 https://auth.libredb.test/realms/libredb/.well-known/openid-configuration +expect_status 404 https://auth.libredb.test/admin/ +expect_status 404 https://auth.libredb.test/realms/master/ +expect_status 400 "https://auth.libredb.test/realms/libredb/..;/master/" + +step "network policies: a pod of another namespace reaches Platform but not Keycloak's Admin API or PostgreSQL" +probe() { + kubectl -n default run "np-probe-$RANDOM" --rm -i --restart=Never --quiet --image=busybox:1.37 -- \ + sh -c "nc -z -w 3 $1 $2 && echo open || echo closed" 2>/dev/null | tail -1 +} +[ "$(probe "libredb-platform.$namespace.svc" 3000)" = "open" ] || fail "the probe pod cannot reach Platform at all" +[ "$(probe "keycloak.$namespace.svc" 8080)" = "closed" ] || fail "a pod outside the release reaches Keycloak's Admin API" +[ "$(probe "libredb-postgres.$namespace.svc" 5432)" = "closed" ] || fail "a pod outside the release reaches PostgreSQL" +echo "Keycloak's Admin API and PostgreSQL refuse a pod outside the release" + +step "license and Studio's heartbeat" +kubectl -n "$namespace" logs deploy/libredb-platform | grep -c '"message":"License active"' >/dev/null || fail "Platform did not log License active" +# Platform publishes the canary when it starts, and Studio reports it with its next heartbeat, every 30 seconds. +for _ in $(seq 1 30); do + heartbeat=$(kubectl -n "$namespace" exec libredb-postgres-0 -- sh -c \ + 'PGPASSWORD="$POSTGRES_PASSWORD" psql -X -A -t -h 127.0.0.1 -U postgres -d libredb -c "SELECT canary FROM bridge.studio_instances_v1"') + grep -q '"ok"' <<<"$heartbeat" && break + sleep 5 +done +grep -q '"ok"' <<<"$heartbeat" || fail "Studio's seal canary is not ok: $heartbeat" +echo "license active; seal canary $heartbeat" + +step "browser suite" +account="$work/first-admin.json" +first_admin_file="${LIBREDB_E2E_ACCOUNT_FILE:-$account}" +# The license's id, from its payload, the third dot-separated field in base64url. +license_id=$(cut -d. -f3 "$license" | tr '_-' '/+' | base64 -d 2>/dev/null | grep -o '"id":"[^"]*"' | cut -d'"' -f4 || true) +(cd "$installer" && LIBREDB_E2E_NAMESPACE="$namespace" LIBREDB_E2E_ACCOUNT_FILE="$first_admin_file" LIBREDB_E2E_LICENSE_ID="$license_id" \ + bunx playwright test --config e2e/kubernetes/playwright.config.ts) + +step "upgrade with a changed value keeps the seal keys and the CA" +seal=$(secret_hash libredb-secrets 'seal-keys\.json') +ca=$(secret_hash libredb-tls 'ca\.crt') +install_chart --set platform.queryLogRetentionDays=30 >/dev/null +[ "$(secret_hash libredb-secrets 'seal-keys\.json')" = "$seal" ] || fail "the upgrade changed the seal keys" +[ "$(secret_hash libredb-tls 'ca\.crt')" = "$ca" ] || fail "the upgrade changed the internal CA" +retention=$(kubectl -n "$namespace" get secret libredb-platform-env -o 'jsonpath={.data.QUERY_LOG_RETENTION_DAYS}' | base64 -d) +[ "$retention" = "30" ] || fail "QUERY_LOG_RETENTION_DAYS is $retention after the upgrade" +# The bootstrap hook restarted Platform, whose environment changed, so the running server has the new value. +kubectl -n "$namespace" rollout status deploy/libredb-platform --timeout=180s >/dev/null +running=$(kubectl -n "$namespace" exec deploy/libredb-platform -- printenv QUERY_LOG_RETENTION_DAYS) +[ "$running" = "30" ] || fail "the running Platform has QUERY_LOG_RETENTION_DAYS=$running after the upgrade" +expect_status 200 https://platform.libredb.test/api/health/ready +echo "seal keys and CA kept, QUERY_LOG_RETENTION_DAYS=30 in the Secret and the running server" + +step "a rollback within the release runs the bootstrap with the earlier values and restarts Platform" +helm -n "$namespace" rollback libredb --wait --timeout 10m >/dev/null +kubectl -n "$namespace" rollout status deploy/libredb-platform --timeout=180s >/dev/null +running=$(kubectl -n "$namespace" exec deploy/libredb-platform -- printenv QUERY_LOG_RETENTION_DAYS) +[ "$running" = "90" ] || fail "the running Platform has QUERY_LOG_RETENTION_DAYS=$running after the rollback" +[ "$(secret_hash libredb-secrets 'seal-keys\.json')" = "$seal" ] || fail "the rollback changed the seal keys" +expect_status 200 https://platform.libredb.test/api/health/ready +echo "rolled back to QUERY_LOG_RETENTION_DAYS=90, seal keys kept" + +step "uninstall keeps the generated Secrets and the volumes; a new install reuses them" +helm -n "$namespace" uninstall libredb --wait >/dev/null +for name in libredb-secrets libredb-tls libredb-installer-state; do + kubectl -n "$namespace" get secret "$name" >/dev/null || fail "uninstall removed Secret $name" +done +kubectl -n "$namespace" get pvc data-libredb-postgres-0 >/dev/null || fail "uninstall removed the database volume" +install_chart >/dev/null +[ "$(secret_hash libredb-secrets 'seal-keys\.json')" = "$seal" ] || fail "the reinstall changed the seal keys" +[ "$(secret_hash libredb-tls 'ca\.crt')" = "$ca" ] || fail "the reinstall changed the internal CA" +kubectl -n "$namespace" logs -l app.kubernetes.io/component=keycloak-configure --tail=20 | grep -c "realm already converged" >/dev/null \ + || fail "the reinstall's keycloak job did not find the realm converged" +expect_status 200 https://platform.libredb.test/api/health/ready +expect_status 200 https://studio.libredb.test/api/ready +echo "reinstall reused the seal keys, the CA, the database and the realm" + +if [ "${LIBREDB_E2E_KEEP:-0}" != "1" ]; then + step "cleanup" + helm -n "$namespace" uninstall libredb --wait >/dev/null + kubectl delete namespace "$namespace" --wait=true >/dev/null +fi +printf '\nPASS\n' diff --git a/charts/libredb-platform/tests/e2e/run-external.sh b/charts/libredb-platform/tests/e2e/run-external.sh new file mode 100755 index 0000000..00908cd --- /dev/null +++ b/charts/libredb-platform/tests/e2e/run-external.sh @@ -0,0 +1,224 @@ +#!/usr/bin/env bash +# The end-to-end test of the chart against a customer's own PostgreSQL and Keycloak, on the kind cluster that +# run-e2e.sh set up (Traefik, the *.libredb.test DNS): +# +# LIBREDB_E2E_LICENSE=kind-ext.license charts/libredb-platform/tests/e2e/run-external.sh +# +# The license must name platform-ext.libredb.test. Namespace corp plays the customer: a PostgreSQL server with TLS +# from a CA of its own, at corp-pg.corp.svc.cluster.local, and a Keycloak at https://sso-ext.libredb.test behind +# Traefik with a certificate of the same CA and an admin user in realm master. The chart installs into libredb-ext +# with postgresql.mode external (a DBA runs the script the chart writes) and keycloak.mode external with provisioning +# installer, its own TLS Secrets for the ingress (tls.mode secrets) and the license in a Secret of its own +# (platform.license.existingSecret). The test checks that the install converges, the realm holds the LibreDB objects, +# Platform's client can list users, the license is active, Studio's seal canary is ok, and the ingress serves the +# customer's certificates. +set -euo pipefail + +here=$(cd "$(dirname "$0")" && pwd) +chart=$(cd "$here/../.." && pwd) +license=${LIBREDB_E2E_LICENSE:?set LIBREDB_E2E_LICENSE to a license file for platform-ext.libredb.test} +namespace=libredb-ext +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT + +step() { printf '\n=== %s\n' "$*"; } +fail() { printf 'FAIL: %s\n' "$*" >&2; exit 1; } + +step "a fresh start: namespaces corp and $namespace" +helm -n "$namespace" uninstall libredb --wait >/dev/null 2>&1 || true +kubectl delete namespace corp "$namespace" --ignore-not-found --wait=true >/dev/null + +step "the customer's CA and certificates" +openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:P-256 -nodes -days 30 -subj "/CN=Corp test CA" \ + -keyout "$work/ca.key" -out "$work/ca.crt" 2>/dev/null +leaf() { + openssl req -newkey ec -pkeyopt ec_paramgen_curve:P-256 -nodes -subj "/CN=$2" -keyout "$work/$1.key" -out "$work/$1.csr" 2>/dev/null + openssl x509 -req -in "$work/$1.csr" -CA "$work/ca.crt" -CAkey "$work/ca.key" -CAcreateserial -days 30 \ + -extfile <(printf 'subjectAltName=DNS:%s\nextendedKeyUsage=serverAuth\n' "$2") -out "$work/$1.crt" 2>/dev/null +} +leaf pg corp-pg.corp.svc.cluster.local +leaf sso sso-ext.libredb.test +leaf platform platform-ext.libredb.test +leaf studio studio-ext.libredb.test + +step "the customer's PostgreSQL and Keycloak in namespace corp" +kubectl create namespace corp --dry-run=client -o yaml | kubectl apply -f - >/dev/null +superuser=$(openssl rand -hex 16) +admin_password=$(openssl rand -hex 16) +kubectl -n corp create secret generic corp-pg-tls --from-file=server.crt="$work/pg.crt" --from-file=server.key="$work/pg.key" \ + --dry-run=client -o yaml | kubectl apply -f - >/dev/null +kubectl -n corp create secret tls sso-ext-tls --cert="$work/sso.crt" --key="$work/sso.key" --dry-run=client -o yaml | kubectl apply -f - >/dev/null +postgres_image=$(helm show values "$chart" | awk '/^postgresql:/{p=1} p && /repository:/{r=$2} p && /tag:/{gsub(/"/,"",$2); print r":"$2; exit}') +keycloak_image=$(helm show values "$chart" | awk '/^keycloak:/{p=1} p && /repository: quay/{r=$2} p && r && /tag:/{gsub(/"/,"",$2); print r":"$2; exit}') +kubectl -n corp apply -f - >/dev/null </dev/null +kubectl -n corp rollout status deploy/corp-sso --timeout=300s >/dev/null +resolve=(--resolve sso-ext.libredb.test:443:127.0.0.1 --resolve platform-ext.libredb.test:443:127.0.0.1 --resolve studio-ext.libredb.test:443:127.0.0.1) +# Traefik serves the new Ingress and its certificate a few seconds after it exists. +token="" +for _ in $(seq 1 30); do + token=$(curl -sf "${resolve[@]}" --cacert "$work/ca.crt" https://sso-ext.libredb.test/realms/master/protocol/openid-connect/token \ + -d grant_type=password -d client_id=admin-cli -d username=admin --data-urlencode "password=$admin_password" \ + | sed 's/.*"access_token":"\([^"]*\)".*/\1/') && [ -n "$token" ] && break + sleep 2 +done +[ -n "$token" ] || fail "the customer's Keycloak gave its admin no token" +# The customer's realm exists before the install: provisioning installer configures objects in it, never the realm. +curl -sf "${resolve[@]}" --cacert "$work/ca.crt" -X POST https://sso-ext.libredb.test/admin/realms \ + -H "authorization: Bearer $token" -H 'content-type: application/json' -d '{"realm":"corp","enabled":true}' || true + +step "install into $namespace against them" +kubectl create namespace "$namespace" --dry-run=client -o yaml | kubectl apply -f - >/dev/null +kubectl label namespace "$namespace" pod-security.kubernetes.io/enforce=restricted --overwrite >/dev/null +kubectl -n "$namespace" create secret generic corp-keycloak-admin --from-literal=admin-password="$admin_password" \ + --dry-run=client -o yaml | kubectl apply -f - >/dev/null +kubectl -n "$namespace" create secret generic corp-license --from-file=token="$license" --dry-run=client -o yaml | kubectl apply -f - >/dev/null +kubectl -n "$namespace" create secret tls corp-platform-tls --cert="$work/platform.crt" --key="$work/platform.key" --dry-run=client -o yaml | kubectl apply -f - >/dev/null +kubectl -n "$namespace" create secret tls corp-studio-tls --cert="$work/studio.crt" --key="$work/studio.key" --dry-run=client -o yaml | kubectl apply -f - >/dev/null +cat > "$work/values.yaml" </dev/null + +step "the DBA runs the provisioning script the chart wrote" +for _ in $(seq 1 60); do kubectl -n "$namespace" get secret libredb-postgres-dba-script >/dev/null 2>&1 && break; sleep 2; done +kubectl -n "$namespace" get secret libredb-postgres-dba-script -o 'jsonpath={.data.provision\.sql}' | base64 -d > "$work/provision.sql" +grep -q 'SCRAM-SHA-256\$' "$work/provision.sql" || fail "the DBA script holds no SCRAM verifier" +kubectl -n corp exec -i corp-pg -- sh -c "PGPASSWORD='$superuser' psql -X -q -v ON_ERROR_STOP=1 -h 127.0.0.1 -U postgres -d postgres -f -" < "$work/provision.sql" >/dev/null + +step "the installation converges" +for _ in $(seq 1 120); do + ready=$(kubectl -n "$namespace" get deploy libredb-platform libredb-studio -o 'jsonpath={.items[*].status.readyReplicas}') + [ "$ready" = "1 1" ] && break + sleep 5 +done +[ "$ready" = "1 1" ] || fail "Platform and Studio did not turn ready: $ready" +kubectl -n "$namespace" wait --for=condition=complete job -l app.kubernetes.io/component=keycloak-configure --timeout=300s >/dev/null +kubectl -n "$namespace" wait --for=condition=complete job -l app.kubernetes.io/component=platform-migrate --timeout=300s >/dev/null +kubectl -n "$namespace" logs deploy/libredb-platform | grep -c '"message":"License active"' >/dev/null || fail "Platform did not log License active" +kubectl -n "$namespace" get statefulset libredb-postgres >/dev/null 2>&1 && fail "an external PostgreSQL install runs a managed server" +kubectl -n "$namespace" get deploy keycloak >/dev/null 2>&1 && fail "an external Keycloak install runs a managed Keycloak" + +step "Platform's client signs in to the customer's realm and lists users through its service account" +platform_secret=$(kubectl -n "$namespace" get secret libredb-keycloak-config -o 'jsonpath={.data.keycloak-client-libredb-platform}' | base64 -d) +client_token=$(curl -sf "${resolve[@]}" --cacert "$work/ca.crt" https://sso-ext.libredb.test/realms/corp/protocol/openid-connect/token \ + -d grant_type=client_credentials -d client_id=libredb-platform --data-urlencode "client_secret=$platform_secret" | sed 's/.*"access_token":"\([^"]*\)".*/\1/') +status=$(curl -s "${resolve[@]}" --cacert "$work/ca.crt" -o /dev/null -w '%{http_code}' -H "authorization: Bearer $client_token" \ + https://sso-ext.libredb.test/admin/realms/corp/users?max=1) +[ "$status" = "200" ] || fail "libredb-platform's service account cannot list users: $status" +roles=$(curl -sf "${resolve[@]}" --cacert "$work/ca.crt" -H "authorization: Bearer $token" "https://sso-ext.libredb.test/admin/realms/corp/clients?clientId=libredb") +grep -q '"clientId":"libredb"' <<<"$roles" || fail "the realm holds no client libredb" + +step "Studio's seal canary" +for _ in $(seq 1 30); do + canary=$(kubectl -n corp exec corp-pg -- sh -c "PGPASSWORD='$superuser' psql -X -A -t -h 127.0.0.1 -U postgres -d libredb -c 'SELECT canary FROM bridge.studio_instances_v1'") + grep -q '"ok"' <<<"$canary" && break + sleep 5 +done +grep -q '"ok"' <<<"$canary" || fail "Studio's seal canary is not ok: $canary" +step "the ingress serves the customer's certificates" +for url in https://platform-ext.libredb.test/api/health/ready https://studio-ext.libredb.test/api/ready; do + status=$(curl -s "${resolve[@]}" --cacert "$work/ca.crt" -o /dev/null -w '%{http_code}' "$url") + [ "$status" = "200" ] || fail "$url answered $status over the customer's certificate" + echo "200 $url" +done +if kubectl -n "$namespace" get secret libredb-tls-platform >/dev/null 2>&1; then fail "tls.mode secrets wrote an ingress Secret of the internal CA"; fi + +if [ "${LIBREDB_E2E_KEEP:-0}" != "1" ]; then + step "cleanup" + helm -n "$namespace" uninstall libredb --wait >/dev/null + kubectl delete namespace "$namespace" corp --wait=true >/dev/null +fi +printf '\nPASS\n' diff --git a/charts/libredb-platform/tests/e2e/run-lifecycle.sh b/charts/libredb-platform/tests/e2e/run-lifecycle.sh new file mode 100755 index 0000000..b1770c8 --- /dev/null +++ b/charts/libredb-platform/tests/e2e/run-lifecycle.sh @@ -0,0 +1,125 @@ +#!/usr/bin/env bash +# The lifecycle test of a Helm install, on the installation run-e2e.sh left running (LIBREDB_E2E_KEEP=1): +# +# LIBREDB_E2E_LICENSE=active.license LIBREDB_E2E_LICENSE_EXPIRING=expiring.license \ +# LIBREDB_E2E_LICENSE_EXPIRED=expired.license charts/libredb-platform/tests/e2e/run-lifecycle.sh +# +# Certificate renewal: the clock of a kind node cannot move, so the test puts the installation where it would stand +# after two years: it replaces the PostgreSQL leaf and Platform's ingress leaf with leaves of the same internal CA that +# expire in 10 days, and restarts PostgreSQL so it serves its short leaf. An upgrade with unchanged values must then +# re-issue both, PostgreSQL must serve its new leaf after the restart job, the ingress its new one, and the apps keep +# working over verify-full. License lifecycle: an upgrade with a license that expires within 30 days, one that has +# expired and then the active one again must each reach the running Platform, which logs the state. +set -euo pipefail + +here=$(cd "$(dirname "$0")" && pwd) +chart=$(cd "$here/../.." && pwd) +namespace=${LIBREDB_E2E_NAMESPACE:-libredb} +active=${LIBREDB_E2E_LICENSE:?the active license for platform.libredb.test} +expiring=${LIBREDB_E2E_LICENSE_EXPIRING:?a license for platform.libredb.test that expires within 30 days} +expired=${LIBREDB_E2E_LICENSE_EXPIRED:?a license for platform.libredb.test past its grace period} +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT + +step() { printf '\n=== %s\n' "$*"; } +fail() { printf 'FAIL: %s\n' "$*" >&2; exit 1; } +# Days until a certificate's notAfter; a read that failed gives none, so a wait never takes it for an answer. +days_left() { + [ -n "$1" ] || { echo -1; return; } + echo $(( ( $(date -d "$1" +%s) - $(date +%s) ) / 86400 )) +} + +upgrade() { + local license=$1 + shift + helm upgrade libredb "$chart" -n "$namespace" -f "$here/values-kind.yaml" --set-file platform.license.value="$license" "$@" \ + --wait --wait-for-jobs --timeout 10m >/dev/null +} + +# The leaf PostgreSQL serves, read after the SSLRequest of the protocol from Platform's pod, which the network +# policies let in. +pg_not_after() { + kubectl -n "$namespace" exec deploy/libredb-platform -- node -e ' + const net = require("net"), tls = require("tls"); + const socket = net.connect(5432, "libredb-postgres", () => { const b = Buffer.alloc(8); b.writeInt32BE(8); b.writeInt32BE(80877103, 4); socket.write(b); }); + socket.once("data", (reply) => { + if (String(reply) !== "S") { console.error("no TLS"); process.exit(1); } + const secure = tls.connect({ socket, servername: "libredb-postgres", rejectUnauthorized: false }, () => { console.log(secure.getPeerCertificate().valid_to); process.exit(0); }); + });' +} + +ingress_not_after() { + openssl s_client -connect 127.0.0.1:443 -servername "$1" /dev/null | openssl x509 -noout -enddate | cut -d= -f2 +} + +platform_license_line() { + kubectl -n "$namespace" logs deploy/libredb-platform | grep '"message":"License' | tail -1 +} + +wait_platform() { + kubectl -n "$namespace" rollout status deploy/libredb-platform --timeout=300s >/dev/null + for _ in $(seq 1 60); do + [ "$(curl -s --resolve platform.libredb.test:443:127.0.0.1 -k -o /dev/null -w '%{http_code}' https://platform.libredb.test/api/health/ready)" = "200" ] && return + sleep 3 + done + fail "Platform did not turn ready" +} + +step "leaves of the internal CA that expire in 10 days, as after two years" +kubectl -n "$namespace" get secret libredb-tls -o 'jsonpath={.data.ca\.crt}' | base64 -d > "$work/ca.crt" +kubectl -n "$namespace" get secret libredb-tls -o 'jsonpath={.data.ca\.key}' | base64 -d > "$work/ca.key" +short_leaf() { + local name=$1 sans=$2 + openssl req -newkey ec -pkeyopt ec_paramgen_curve:P-256 -nodes -subj "/CN=$name" -keyout "$work/$name.key" -out "$work/$name.csr" 2>/dev/null + openssl x509 -req -in "$work/$name.csr" -CA "$work/ca.crt" -CAkey "$work/ca.key" -CAcreateserial -days 10 \ + -extfile <(printf 'subjectAltName=%s\nextendedKeyUsage=serverAuth\nbasicConstraints=CA:FALSE\n' "$sans") -out "$work/$name.crt" 2>/dev/null +} +b64() { base64 -w0 < "$1"; } +short_leaf postgres "DNS:libredb-postgres,DNS:libredb-postgres.$namespace,DNS:libredb-postgres.$namespace.svc,DNS:libredb-postgres.$namespace.svc.cluster.local" +short_leaf platform "DNS:platform.libredb.test" +kubectl -n "$namespace" patch secret libredb-tls --type merge -p "{\"data\":{\"postgres.crt\":\"$(b64 "$work/postgres.crt")\",\"postgres.key\":\"$(b64 "$work/postgres.key")\",\"platform.crt\":\"$(b64 "$work/platform.crt")\",\"platform.key\":\"$(b64 "$work/platform.key")\"}}" >/dev/null +kubectl -n "$namespace" patch secret libredb-postgres-tls --type merge -p "{\"data\":{\"server.crt\":\"$(b64 "$work/postgres.crt")\",\"server.key\":\"$(b64 "$work/postgres.key")\"}}" >/dev/null +kubectl -n "$namespace" patch secret libredb-tls-platform --type merge -p "{\"data\":{\"tls.crt\":\"$(b64 "$work/platform.crt")\",\"tls.key\":\"$(b64 "$work/platform.key")\"}}" >/dev/null +kubectl -n "$namespace" rollout restart statefulset/libredb-postgres >/dev/null +kubectl -n "$namespace" rollout status statefulset/libredb-postgres --timeout=300s >/dev/null +for _ in $(seq 1 30); do + left=$(days_left "$(pg_not_after 2>/dev/null || true)") && [ "$left" -ge 0 ] && [ "$left" -le 10 ] && break + sleep 5 +done +{ [ "$left" -ge 0 ] && [ "$left" -le 10 ]; } || fail "PostgreSQL does not serve the short leaf: $left days left" +for _ in $(seq 1 30); do + ingress=$(days_left "$(ingress_not_after platform.libredb.test || true)") && [ "$ingress" -ge 0 ] && [ "$ingress" -le 10 ] && break + sleep 3 +done +{ [ "$ingress" -ge 0 ] && [ "$ingress" -le 10 ]; } || fail "the ingress does not serve the short leaf: $ingress days left" +echo "PostgreSQL serves a leaf with $left days left, the ingress one with $ingress" + +step "an upgrade with unchanged values re-issues both leaves and restarts PostgreSQL" +upgrade "$active" +kubectl -n "$namespace" rollout status statefulset/libredb-postgres --timeout=300s >/dev/null +for _ in $(seq 1 60); do + left=$(days_left "$(pg_not_after 2>/dev/null || true)") && [ "$left" -gt 800 ] && break + sleep 5 +done +[ "$left" -gt 800 ] || fail "PostgreSQL still serves a leaf with $left days left after the upgrade" +for _ in $(seq 1 40); do + ingress=$(days_left "$(ingress_not_after platform.libredb.test || true)") && [ "$ingress" -gt 800 ] && break + sleep 3 +done +[ "$ingress" -gt 800 ] || fail "the ingress still serves a leaf with $ingress days left" +wait_platform +curl -sf --resolve studio.libredb.test:443:127.0.0.1 --cacert "$work/ca.crt" -o /dev/null https://studio.libredb.test/api/ready || fail "Studio is not ready after PostgreSQL's restart" +echo "PostgreSQL serves a leaf with $left days left, the ingress one with $ingress; Platform and Studio ready" + +step "license lifecycle: expiring, expired, active again, each in the running Platform" +for case in "$expiring:expiring:License expiring within 30 days" "$expired:expired:License expired, no seat can be added" "$active:active:License active"; do + file=${case%%:*}; rest=${case#*:}; state=${rest%%:*}; message=${rest#*:} + upgrade "$file" + wait_platform + line=$(platform_license_line) + grep -q "\"message\":\"$message\"" <<<"$line" || fail "after the $state license Platform logged: $line" + grep -q "\"state\":\"$state\"" <<<"$line" || fail "after the $state license Platform logged: $line" + echo "$state: $(sed 's/.*"licenseId":"\([^"]*\)".*/\1/' <<<"$line")" +done + +printf '\nPASS\n' diff --git a/charts/libredb-platform/tests/e2e/run-offline.sh b/charts/libredb-platform/tests/e2e/run-offline.sh new file mode 100755 index 0000000..fcec9a4 --- /dev/null +++ b/charts/libredb-platform/tests/e2e/run-offline.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash +# The install without network access of docs/deploy/install-kubernetes.md, on the kind cluster run-e2e.sh set up: +# +# LIBREDB_E2E_LICENSE=kind.license LIBREDB_E2E_MIRROR=libredb-mirror:5000 LIBREDB_E2E_MIRROR_PUSH=localhost:5055 \ +# charts/libredb-platform/tests/e2e/run-offline.sh +# +# LIBREDB_E2E_MIRROR is a registry the cluster's nodes pull from, LIBREDB_E2E_MIRROR_PUSH the same registry as this +# host reaches it. On the connected side the test packages the chart with its dependency and copies the release's +# images into the mirror by digest; on the site side Helm runs with no repository configured and every proxy pointing +# at a closed port, installs the archive with each image repository set to the mirror, and the test checks that every +# container ran an image of the mirror, the endpoints, the license and Studio's seal canary. +# LIBREDB_E2E_KEEP=1 keeps the installation. +set -euo pipefail + +here=$(cd "$(dirname "$0")" && pwd) +chart=$(cd "$here/../.." && pwd) +license=${LIBREDB_E2E_LICENSE:?set LIBREDB_E2E_LICENSE to a license file for platform.libredb.test} +mirror=${LIBREDB_E2E_MIRROR:?the mirror registry as the nodes reach it} +push=${LIBREDB_E2E_MIRROR_PUSH:?the mirror registry as this host reaches it} +namespace=${LIBREDB_E2E_NAMESPACE:-libredb} +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT + +step() { printf '\n=== %s\n' "$*"; } +fail() { printf 'FAIL: %s\n' "$*" >&2; exit 1; } + +step "connected side: the chart archive with its dependency" +helm package --dependency-update "$chart" -d "$work" >/dev/null +archive=$(ls "$work"/libredb-platform-*.tgz) +tar -tzf "$archive" | grep -cx 'libredb-platform/charts/libredb-studio/Chart.yaml' >/dev/null || fail "the archive lacks the Studio chart" +echo "$(basename "$archive") holds the Studio chart" + +step "connected side: the release's images into the mirror, by digest" +images=$(helm template libredb "$archive" -f "$here/values-kind.yaml" --set-file platform.license.value="$license" \ + | grep -oE 'image: "?[^" ]+' | awk '{print $2}' | tr -d '"' | sort -u) +for image in $images; do + # registry/repository:tag@digest, copied as repository:tag with its index, so the digest stays. + name=${image#*/} + name=${name%@*} + docker buildx imagetools create -t "$push/$name" "$image" >/dev/null 2>&1 + echo "$name" +done + +# LIBREDB_E2E_KIND_NODE: a kind node whose copies of these images go first, so its pods pull them from the mirror. +if [ -n "${LIBREDB_E2E_KIND_NODE:-}" ]; then + step "the node $LIBREDB_E2E_KIND_NODE forgets the release's images" + for image in $images; do + for ref in "$image" "$mirror/${image#*/}"; do + docker exec "$LIBREDB_E2E_KIND_NODE" crictl rmi "$ref" >/dev/null 2>&1 || true + docker exec "$LIBREDB_E2E_KIND_NODE" crictl inspecti "$ref" >/dev/null 2>&1 && fail "the node still holds $ref" + done + done + echo "none of them left on the node" +fi + +step "site side: Helm with no repository and no way out" +export HELM_REPOSITORY_CONFIG="$work/repositories.yaml" HELM_REPOSITORY_CACHE="$work/cache" HELM_REGISTRY_CONFIG="$work/registry.json" +export HTTPS_PROXY=http://127.0.0.1:9 HTTP_PROXY=http://127.0.0.1:9 https_proxy=http://127.0.0.1:9 http_proxy=http://127.0.0.1:9 +export NO_PROXY=127.0.0.1,localhost no_proxy=127.0.0.1,localhost +# helm repo list fails when no repository is configured, which is what this checks. +repos=$(helm repo list 2>&1 || true) +grep -q "no repositories" <<<"$repos" || fail "Helm has a repository configured: $repos" +helm -n "$namespace" uninstall libredb --wait >/dev/null 2>&1 || true +kubectl delete namespace "$namespace" --ignore-not-found --wait=true >/dev/null +kubectl create namespace "$namespace" >/dev/null +kubectl label namespace "$namespace" pod-security.kubernetes.io/enforce=restricted >/dev/null +mirrored=( + --set "platform.image.repository=$mirror/libredb/libredb-platform" + --set "postgresql.managed.image.repository=$mirror/library/postgres" + --set "keycloak.managed.image.repository=$mirror/keycloak/keycloak" + --set "keycloak.managed.edge.image.repository=$mirror/library/caddy" + --set "studio.image.repository=$mirror/libredb/libredb-studio" +) +rendered=$(helm template libredb "$archive" -n "$namespace" -f "$here/values-kind.yaml" --set-file platform.license.value="$license" "${mirrored[@]}" \ + | grep -oE 'image: "?[^" ]+' | awk '{print $2}' | tr -d '"' | sort -u) +outside=$(grep -v "^$mirror/" <<<"$rendered" || true) +[ -z "$outside" ] || fail "images outside the mirror: $outside" +started=$(date +%s) +helm install libredb "$archive" -n "$namespace" -f "$here/values-kind.yaml" --set-file platform.license.value="$license" \ + "${mirrored[@]}" --wait --wait-for-jobs --timeout 10m >/dev/null +echo "installed from the archive in $(( $(date +%s) - started )) s, every image from $mirror" +unset HTTPS_PROXY HTTP_PROXY https_proxy http_proxy + +step "every container ran an image the nodes pulled from the mirror" +ids=$(kubectl -n "$namespace" get pods -o 'jsonpath={range .items[*]}{range .status.initContainerStatuses[*]}{.imageID}{"\n"}{end}{range .status.containerStatuses[*]}{.imageID}{"\n"}{end}{end}' | sort -u) +outside=$(grep -v "^$mirror/" <<<"$ids" || true) +[ -z "$outside" ] || fail "containers ran images from elsewhere: $outside" +sed 's/^/ /' <<<"$ids" + +step "endpoints, license and Studio's heartbeat" +kubectl -n "$namespace" get secret libredb-tls-platform -o 'jsonpath={.data.ca\.crt}' | base64 -d > "$work/ca.crt" +for url in https://platform.libredb.test/api/health/ready https://studio.libredb.test/api/ready \ + https://auth.libredb.test/realms/libredb/.well-known/openid-configuration; do + host=${url#https://} + status=$(curl -s --resolve "${host%%/*}:443:127.0.0.1" --cacert "$work/ca.crt" -o /dev/null -w '%{http_code}' "$url") + [ "$status" = "200" ] || fail "$url answered $status" + echo "200 $url" +done +kubectl -n "$namespace" logs deploy/libredb-platform | grep -c '"message":"License active"' >/dev/null || fail "Platform did not log License active" +for _ in $(seq 1 30); do + heartbeat=$(kubectl -n "$namespace" exec libredb-postgres-0 -- sh -c \ + 'PGPASSWORD="$POSTGRES_PASSWORD" psql -X -A -t -h 127.0.0.1 -U postgres -d libredb -c "SELECT canary FROM bridge.studio_instances_v1"') + grep -q '"ok"' <<<"$heartbeat" && break + sleep 5 +done +grep -q '"ok"' <<<"$heartbeat" || fail "Studio's seal canary is not ok: $heartbeat" +echo "license active; seal canary $heartbeat" + +if [ "${LIBREDB_E2E_KEEP:-0}" != "1" ]; then + step "cleanup" + helm -n "$namespace" uninstall libredb --wait >/dev/null + kubectl delete namespace "$namespace" --wait=true >/dev/null +fi +printf '\nPASS\n' diff --git a/charts/libredb-platform/tests/e2e/run-openshift.sh b/charts/libredb-platform/tests/e2e/run-openshift.sh new file mode 100755 index 0000000..3757be8 --- /dev/null +++ b/charts/libredb-platform/tests/e2e/run-openshift.sh @@ -0,0 +1,155 @@ +#!/usr/bin/env bash +# The end-to-end test of the Helm chart on OpenShift, in the one namespace of the current `oc login`, such as a Red +# Hat Developer Sandbox's: +# +# LIBREDB_E2E_LICENSE=os.license LIBREDB_E2E_DOMAIN=apps.example.openshiftapps.com \ +# charts/libredb-platform/tests/e2e/run-openshift.sh +# +# The hosts are ldb-platform-., ldb-studio-. and ldb-auth-., +# under the apps domain the router serves, and the license must name the first. OpenShift turns each Ingress into a +# Route that terminates TLS with the Ingress's Secret, so the endpoints answer with leaves of the internal CA. The test +# needs no cluster scope: it checks that the restricted-v2 SCC admits every pod with a uid from the namespace's range, +# the endpoints, the edge's path rules, the license and Studio's seal canary, runs the browser suite, upgrades, rolls +# back, uninstalls and installs again, as run-e2e.sh does on kind. It removes everything the chart left in the +# namespace before it starts and when it ends (LIBREDB_E2E_KEEP=1 keeps the last installation), and nothing else. +# Needs oc, kubectl, helm, curl and bun on PATH. +set -euo pipefail + +here=$(cd "$(dirname "$0")" && pwd) +chart=$(cd "$here/../.." && pwd) +installer=$(cd "$chart/../../installer" && pwd) +license=${LIBREDB_E2E_LICENSE:?set LIBREDB_E2E_LICENSE to a license file for ldb-platform-.} +domain=${LIBREDB_E2E_DOMAIN:?set LIBREDB_E2E_DOMAIN to the apps domain of the router} +namespace=$(oc project -q) +platform=ldb-platform-$namespace.$domain +studio=ldb-studio-$namespace.$domain +auth=ldb-auth-$namespace.$domain +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT + +step() { printf '\n=== %s\n' "$*"; } +fail() { printf 'FAIL: %s\n' "$*" >&2; exit 1; } + +sed -e "s/ldb-platform\.apps\.example\.com/$platform/g" -e "s/ldb-studio\.apps\.example\.com/$studio/g" \ + -e "s/ldb-auth\.apps\.example\.com/$auth/g" "$here/values-openshift.yaml" > "$work/values.yaml" + +install_chart() { + helm upgrade --install libredb "$chart" -n "$namespace" -f "$work/values.yaml" \ + --set-file platform.license.value="$license" "$@" --wait --wait-for-jobs --timeout 15m +} + +secret_hash() { kubectl -n "$namespace" get secret "$1" -o "jsonpath={.data.$2}" | sha256sum | cut -d' ' -f1; } + +# What the chart leaves on purpose after helm uninstall: the jobs' Secrets and ConfigMaps and the volumes. +remove_installation() { + helm -n "$namespace" uninstall libredb --wait >/dev/null 2>&1 || true + kubectl -n "$namespace" delete pod/e2e-target service/e2e-target --ignore-not-found >/dev/null + kubectl -n "$namespace" delete secret,configmap -l app.kubernetes.io/managed-by=libredb-bootstrap --ignore-not-found >/dev/null + kubectl -n "$namespace" delete pvc -l app.kubernetes.io/instance=libredb --ignore-not-found --wait=true >/dev/null +} + +step "a clean namespace $namespace" +remove_installation + +step "install with OpenShift's restricted-v2 SCC" +started=$(date +%s) +install_chart >/dev/null +echo "installed in $(( $(date +%s) - started )) s" +errors=$(kubectl -n "$namespace" get pods --no-headers | awk '$3 == "Error" || $3 == "CrashLoopBackOff"' | wc -l) +[ "$errors" -eq 0 ] || fail "$errors pods failed during the install" + +step "every pod runs under restricted-v2 with a uid from the namespace's range" +range=$(oc get namespace "$namespace" -o 'jsonpath={.metadata.annotations.openshift\.io/sa\.scc\.uid-range}') +first=${range%/*} +size=${range#*/} +pods=$(kubectl -n "$namespace" get pods -l app.kubernetes.io/instance=libredb --field-selector=status.phase=Running -o name) +[ -n "$pods" ] || fail "no running pod of the release" +for pod in $pods; do + scc=$(kubectl -n "$namespace" get "$pod" -o 'jsonpath={.metadata.annotations.openshift\.io/scc}') + [ "$scc" = "restricted-v2" ] || fail "$pod runs under SCC $scc" + # The SCC writes the uid it assigned into each container's security context. + uid=$(kubectl -n "$namespace" get "$pod" -o 'jsonpath={.spec.containers[0].securityContext.runAsUser}') + [ "$uid" -ge "$first" ] && [ "$uid" -lt $(( first + size )) ] || fail "$pod runs as uid $uid, outside $range" + printf '%s restricted-v2 uid %s\n' "$pod" "$uid" +done + +step "the Routes OpenShift made of the Ingresses" +for host in "$platform" "$studio" "$auth"; do + termination=$(kubectl -n "$namespace" get routes -o "jsonpath={.items[?(@.spec.host=='$host')].spec.tls.termination}") + [ "$termination" = "edge" ] || fail "no edge Route for $host" +done +echo "edge Routes for the three hosts" + +step "endpoints, the edge's path rules and TLS against the internal CA" +kubectl -n "$namespace" get secret libredb-tls-platform -o 'jsonpath={.data.ca\.crt}' | base64 -d > "$work/ca.crt" +expect_status() { + local status + status=$(curl -s --cacert "$work/ca.crt" --path-as-is -o /dev/null -w '%{http_code}' "$2") + [ "$status" = "$1" ] || fail "$2 answered $status, not $1" + printf '%s %s\n' "$status" "$2" +} +expect_status 200 "https://$platform/api/health/ready" +expect_status 200 "https://$studio/api/ready" +expect_status 200 "https://$auth/realms/libredb/.well-known/openid-configuration" +expect_status 404 "https://$auth/admin/" +expect_status 404 "https://$auth/realms/master/" +expect_status 400 "https://$auth/realms/libredb/..;/master/" + +step "license and Studio's heartbeat" +kubectl -n "$namespace" logs deploy/libredb-platform | grep -c '"message":"License active"' >/dev/null || fail "Platform did not log License active" +for _ in $(seq 1 30); do + heartbeat=$(kubectl -n "$namespace" exec libredb-postgres-0 -- sh -c \ + 'PGPASSWORD="$POSTGRES_PASSWORD" psql -X -A -t -h 127.0.0.1 -U postgres -d libredb -c "SELECT canary FROM bridge.studio_instances_v1"') + grep -q '"ok"' <<<"$heartbeat" && break + sleep 5 +done +grep -q '"ok"' <<<"$heartbeat" || fail "Studio's seal canary is not ok: $heartbeat" +echo "license active; seal canary $heartbeat" + +step "browser suite" +license_id=$(cut -d. -f3 "$license" | tr '_-' '/+' | base64 -d 2>/dev/null | grep -o '"id":"[^"]*"' | cut -d'"' -f4 || true) +(cd "$installer" && LIBREDB_E2E_NAMESPACE="$namespace" LIBREDB_E2E_OPENSHIFT=1 \ + LIBREDB_E2E_PLATFORM_URL="https://$platform" LIBREDB_E2E_STUDIO_URL="https://$studio" \ + LIBREDB_E2E_LICENSE_ID="$license_id" LIBREDB_E2E_ACCOUNT_FILE="${LIBREDB_E2E_ACCOUNT_FILE:-$work/first-admin.json}" \ + bunx playwright test --config e2e/kubernetes/playwright.config.ts) + +step "upgrade with a changed value keeps the seal keys and the CA" +seal=$(secret_hash libredb-secrets 'seal-keys\.json') +ca=$(secret_hash libredb-tls 'ca\.crt') +install_chart --set platform.queryLogRetentionDays=30 >/dev/null +[ "$(secret_hash libredb-secrets 'seal-keys\.json')" = "$seal" ] || fail "the upgrade changed the seal keys" +[ "$(secret_hash libredb-tls 'ca\.crt')" = "$ca" ] || fail "the upgrade changed the internal CA" +kubectl -n "$namespace" rollout status deploy/libredb-platform --timeout=180s >/dev/null +running=$(kubectl -n "$namespace" exec deploy/libredb-platform -- printenv QUERY_LOG_RETENTION_DAYS) +[ "$running" = "30" ] || fail "the running Platform has QUERY_LOG_RETENTION_DAYS=$running after the upgrade" +expect_status 200 "https://$platform/api/health/ready" +echo "seal keys and CA kept, QUERY_LOG_RETENTION_DAYS=30 in the running server" + +step "a rollback within the release restores the earlier values" +helm -n "$namespace" rollback libredb --wait --timeout 15m >/dev/null +kubectl -n "$namespace" rollout status deploy/libredb-platform --timeout=180s >/dev/null +running=$(kubectl -n "$namespace" exec deploy/libredb-platform -- printenv QUERY_LOG_RETENTION_DAYS) +[ "$running" = "90" ] || fail "the running Platform has QUERY_LOG_RETENTION_DAYS=$running after the rollback" +[ "$(secret_hash libredb-secrets 'seal-keys\.json')" = "$seal" ] || fail "the rollback changed the seal keys" +echo "rolled back to QUERY_LOG_RETENTION_DAYS=90, seal keys kept" + +step "uninstall keeps the generated Secrets and the volumes; a new install reuses them" +helm -n "$namespace" uninstall libredb --wait >/dev/null +for name in libredb-secrets libredb-tls libredb-installer-state; do + kubectl -n "$namespace" get secret "$name" >/dev/null || fail "uninstall removed Secret $name" +done +kubectl -n "$namespace" get pvc data-libredb-postgres-0 >/dev/null || fail "uninstall removed the database volume" +install_chart >/dev/null +[ "$(secret_hash libredb-secrets 'seal-keys\.json')" = "$seal" ] || fail "the reinstall changed the seal keys" +[ "$(secret_hash libredb-tls 'ca\.crt')" = "$ca" ] || fail "the reinstall changed the internal CA" +kubectl -n "$namespace" logs -l app.kubernetes.io/component=keycloak-configure --tail=20 | grep -c "realm already converged" >/dev/null \ + || fail "the reinstall's keycloak job did not find the realm converged" +expect_status 200 "https://$platform/api/health/ready" +expect_status 200 "https://$studio/api/ready" +echo "reinstall reused the seal keys, the CA, the database and the realm" + +if [ "${LIBREDB_E2E_KEEP:-0}" != "1" ]; then + step "cleanup" + remove_installation +fi +printf '\nPASS\n' diff --git a/charts/libredb-platform/tests/e2e/run-resilience.sh b/charts/libredb-platform/tests/e2e/run-resilience.sh new file mode 100755 index 0000000..a7303d4 --- /dev/null +++ b/charts/libredb-platform/tests/e2e/run-resilience.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash +# The resilience test of the Helm chart, on the kind cluster run-e2e.sh set up: +# +# LIBREDB_E2E_LICENSE=kind.license charts/libredb-platform/tests/e2e/run-resilience.sh +# +# It installs the chart, and the browser suite restore.k8s.ts (phase seed) has the first admin connect a target +# PostgreSQL and grant a Keycloak user, who reads the table in Studio. Then it kills, without a grace period, the +# PostgreSQL pod, then Platform, Studio and Keycloak at once, and finally restarts the kind node itself, the cluster's +# power cut. After each it waits, with no other action, until the three endpoints answer again, reports how long that +# took, and checks that the connections, the grants and the Keycloak users are what they were. The browser suite +# (phase verify) then signs the first admin in with the seed's password and TOTP device and has the user read the table +# through the credential sealed in the seed. LIBREDB_E2E_KEEP=1 keeps the installation. +set -euo pipefail + +here=$(cd "$(dirname "$0")" && pwd) +chart=$(cd "$here/../.." && pwd) +installer=$(cd "$chart/../../installer" && pwd) +license=${LIBREDB_E2E_LICENSE:?set LIBREDB_E2E_LICENSE to a license file for platform.libredb.test} +namespace=${LIBREDB_E2E_NAMESPACE:-libredb} +node=${LIBREDB_E2E_KIND_NODE:-libredb-helm-control-plane} +target_namespace=libredb-resilience-target +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT + +step() { printf '\n=== %s\n' "$*"; } +fail() { printf 'FAIL: %s\n' "$*" >&2; exit 1; } + +status() { curl -s --max-time 5 --resolve "$1:443:127.0.0.1" --cacert "$work/ca.crt" -o /dev/null -w '%{http_code}' "https://$1$2" || true; } +answering() { + [ "$(status platform.libredb.test /api/health/ready)" = 200 ] && [ "$(status studio.libredb.test /api/ready)" = 200 ] \ + && [ "$(status auth.libredb.test /realms/libredb/.well-known/openid-configuration)" = 200 ] +} +# Waits, doing nothing else, until the three endpoints answer 200, and prints how long that took since $1. +recovered() { + local since=$1 + for _ in $(seq 1 240); do + if answering; then echo "every endpoint answers again $(( $(date +%s) - since )) s after it"; return; fi + sleep 5 + done + fail "the endpoints did not answer within 20 minutes" +} + +psql_super() { + kubectl -n "$namespace" exec -i libredb-postgres-0 -- sh -c \ + 'PGPASSWORD="$POSTGRES_PASSWORD" psql -X -A -t -q -h 127.0.0.1 -U postgres -d "$1"' sh "$1" +} +# What a crash must not lose: the connections with their envelopes, the grants, and the realm's users. +fingerprint() { + { + echo "SELECT md5(coalesce(string_agg(t::text, '|' ORDER BY t::text), '')) FROM (SELECT id, \"readSecret\", \"writeSecret\" FROM platform.database_connections) t; + SELECT md5(coalesce(string_agg(t::text, '|' ORDER BY t::text), '')) FROM platform.connection_grants t;" | psql_super libredb + echo "SELECT md5(coalesce(string_agg(username, '|' ORDER BY username), '')) FROM user_entity;" | psql_super keycloak + } | tr '\n' ' ' +} + +browser() { + (cd "$installer" && LIBREDB_E2E_NAMESPACE="$namespace" LIBREDB_E2E_TARGET_NAMESPACE="$target_namespace" \ + LIBREDB_E2E_RESTORE_PHASE="$1" LIBREDB_E2E_RESTORE_STATE="$work/state.json" LIBREDB_E2E_ACCOUNT_FILE="$work/first-admin.json" \ + bunx playwright test --config e2e/kubernetes/playwright.config.ts restore.k8s.ts) +} + +step "a fresh installation in namespace $namespace, the target database in $target_namespace" +helm -n "$namespace" uninstall libredb --wait >/dev/null 2>&1 || true +kubectl delete namespace "$namespace" "$target_namespace" --ignore-not-found --wait=true >/dev/null +for ns in "$namespace" "$target_namespace"; do + kubectl create namespace "$ns" >/dev/null + kubectl label namespace "$ns" pod-security.kubernetes.io/enforce=restricted >/dev/null +done +helm install libredb "$chart" -n "$namespace" -f "$here/values-kind.yaml" --set-file platform.license.value="$license" \ + --wait --wait-for-jobs --timeout 10m >/dev/null +kubectl -n "$namespace" get secret libredb-tls-platform -o 'jsonpath={.data.ca\.crt}' | base64 -d > "$work/ca.crt" + +step "seed: a connection, a grant and a read in Studio" +browser seed +before=$(fingerprint) +echo "fingerprint $before" + +step "PostgreSQL killed without a grace period" +started=$(date +%s) +kubectl -n "$namespace" delete pod libredb-postgres-0 --grace-period=0 --force >/dev/null 2>&1 +recovered "$started" +[ "$(fingerprint)" = "$before" ] || fail "the data changed: $(fingerprint)" + +step "Platform, Studio and Keycloak killed at once" +started=$(date +%s) +kubectl -n "$namespace" delete pod -l 'app.kubernetes.io/component in (platform,keycloak)' --grace-period=0 --force >/dev/null 2>&1 +kubectl -n "$namespace" delete pod -l app.kubernetes.io/name=studio --grace-period=0 --force >/dev/null 2>&1 +recovered "$started" +[ "$(fingerprint)" = "$before" ] || fail "the data changed: $(fingerprint)" + +step "the node $node restarted, the cluster's power cut" +started=$(date +%s) +docker restart "$node" >/dev/null +for _ in $(seq 1 120); do kubectl get nodes >/dev/null 2>&1 && break; sleep 5; done +kubectl wait --for=condition=Ready "node/$node" --timeout=600s >/dev/null +recovered "$started" +[ "$(fingerprint)" = "$before" ] || fail "the data changed: $(fingerprint)" +restarts=$(kubectl -n "$namespace" get pods -o 'jsonpath={range .items[*]}{.metadata.name}={range .status.containerStatuses[*]}{.restartCount}{" "}{end}{"\n"}{end}') +printf 'container restarts after the power cut:\n%s\n' "$restarts" + +step "verify: the first admin signs in, and the user reads the table through the credential of the seed" +browser verify + +if [ "${LIBREDB_E2E_KEEP:-0}" != "1" ]; then + step "cleanup" + helm -n "$namespace" uninstall libredb --wait >/dev/null + kubectl delete namespace "$namespace" "$target_namespace" --wait=true >/dev/null +fi +printf '\nPASS\n' diff --git a/charts/libredb-platform/tests/e2e/run-restore.sh b/charts/libredb-platform/tests/e2e/run-restore.sh new file mode 100755 index 0000000..81b7847 --- /dev/null +++ b/charts/libredb-platform/tests/e2e/run-restore.sh @@ -0,0 +1,115 @@ +#!/usr/bin/env bash +# The backup and restore test of the Helm chart, on the kind cluster run-e2e.sh set up, with the steps of "Back up and +# restore" in docs/deploy/install-kubernetes.md: +# +# LIBREDB_E2E_LICENSE=kind.license charts/libredb-platform/tests/e2e/run-restore.sh +# +# It installs the chart, and the browser suite restore.k8s.ts (phase seed) has the first admin connect a target +# PostgreSQL in a namespace of its own and grant a Keycloak user, who reads the table in Studio. Then it takes the +# backup set of the guide (the three Secrets and a pg_dump of each database), deletes the installation's namespace, +# volumes included, restores the set into a new namespace of the same name and checks the seal keys, the CA, the +# license and Studio's seal canary; the browser suite (phase verify) then signs the first admin in with the seed's +# password and TOTP device and has the user read the table again, through the credential sealed before the backup. +# LIBREDB_E2E_KEEP=1 keeps the restored installation and the target database. +set -euo pipefail + +here=$(cd "$(dirname "$0")" && pwd) +chart=$(cd "$here/../.." && pwd) +installer=$(cd "$chart/../../installer" && pwd) +license=${LIBREDB_E2E_LICENSE:?set LIBREDB_E2E_LICENSE to a license file for platform.libredb.test} +namespace=${LIBREDB_E2E_NAMESPACE:-libredb} +target_namespace=libredb-restore-target +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT +backup="$work/backup" + +step() { printf '\n=== %s\n' "$*"; } +fail() { printf 'FAIL: %s\n' "$*" >&2; exit 1; } +secret_hash() { kubectl -n "$namespace" get secret "$1" -o "jsonpath={.data.$2}" | sha256sum | cut -d' ' -f1; } + +install_chart() { + helm upgrade --install libredb "$chart" -n "$namespace" -f "$here/values-kind.yaml" \ + --set-file platform.license.value="$license" --wait --wait-for-jobs --timeout 10m +} + +restricted_namespace() { + kubectl create namespace "$1" --dry-run=client -o yaml | kubectl apply -f - >/dev/null + kubectl label namespace "$1" pod-security.kubernetes.io/enforce=restricted --overwrite >/dev/null +} + +browser() { + (cd "$installer" && LIBREDB_E2E_NAMESPACE="$namespace" LIBREDB_E2E_TARGET_NAMESPACE="$target_namespace" \ + LIBREDB_E2E_RESTORE_PHASE="$1" LIBREDB_E2E_RESTORE_STATE="$work/state.json" LIBREDB_E2E_ACCOUNT_FILE="$work/first-admin.json" \ + bunx playwright test --config e2e/kubernetes/playwright.config.ts restore.k8s.ts) +} + +databases=(libredb keycloak) + +step "a fresh installation in namespace $namespace, the target database in $target_namespace" +helm -n "$namespace" uninstall libredb --wait >/dev/null 2>&1 || true +kubectl delete namespace "$namespace" "$target_namespace" --ignore-not-found --wait=true >/dev/null +restricted_namespace "$namespace" +restricted_namespace "$target_namespace" +install_chart >/dev/null + +step "seed: a connection, a grant and a read in Studio" +browser seed + +step "the backup set of the guide" +mkdir -m 0700 "$backup" +kubectl -n "$namespace" get secret libredb-secrets libredb-tls libredb-installer-state -o yaml > "$backup/secrets.yaml" +for db in "${databases[@]}"; do + kubectl -n "$namespace" exec libredb-postgres-0 -- sh -c \ + 'PGPASSWORD="$POSTGRES_PASSWORD" pg_dump -h 127.0.0.1 -U postgres -Fc -d "$1"' sh "$db" > "$backup/$db.dump" +done +seal=$(secret_hash libredb-secrets 'seal-keys\.json') +ca=$(secret_hash libredb-tls 'ca\.crt') +ls -l "$backup" | tail -n +2 + +step "the disaster: namespace $namespace deleted, its volumes with it" +helm -n "$namespace" uninstall libredb --wait >/dev/null +kubectl delete namespace "$namespace" --wait=true >/dev/null +volumes() { kubectl get pv -o 'jsonpath={range .items[*]}{.spec.claimRef.namespace}{"\n"}{end}' | grep -cx "$namespace" || true; } +for _ in $(seq 1 60); do [ "$(volumes)" = "0" ] && break; sleep 2; done +[ "$(volumes)" = "0" ] || fail "a volume of $namespace survived" +echo "namespace and volumes gone" + +step "restore: the Secrets, a new install, then each database while the apps are stopped" +restricted_namespace "$namespace" +kubectl -n "$namespace" create -f "$backup/secrets.yaml" >/dev/null +install_chart >/dev/null +kubectl -n "$namespace" scale deployment libredb-platform libredb-studio keycloak --replicas=0 >/dev/null +kubectl -n "$namespace" wait --for=delete pod -l 'app.kubernetes.io/component in (platform,keycloak)' --timeout=180s >/dev/null +kubectl -n "$namespace" wait --for=delete pod -l app.kubernetes.io/name=studio --timeout=180s >/dev/null +for db in "${databases[@]}"; do + kubectl -n "$namespace" exec -i libredb-postgres-0 -- sh -c \ + 'PGPASSWORD="$POSTGRES_PASSWORD" pg_restore -h 127.0.0.1 -U postgres --clean --if-exists --exit-on-error -d "$1"' sh "$db" \ + < "$backup/$db.dump" +done +kubectl -n "$namespace" scale deployment libredb-platform libredb-studio keycloak --replicas=1 >/dev/null +for deploy in libredb-platform libredb-studio keycloak; do + kubectl -n "$namespace" rollout status "deploy/$deploy" --timeout=300s >/dev/null +done + +step "the restored installation" +[ "$(secret_hash libredb-secrets 'seal-keys\.json')" = "$seal" ] || fail "the restore changed the seal keys" +[ "$(secret_hash libredb-tls 'ca\.crt')" = "$ca" ] || fail "the restore changed the internal CA" +kubectl -n "$namespace" logs deploy/libredb-platform | grep -c '"message":"License active"' >/dev/null || fail "Platform did not log License active" +for _ in $(seq 1 30); do + heartbeat=$(kubectl -n "$namespace" exec libredb-postgres-0 -- sh -c \ + 'PGPASSWORD="$POSTGRES_PASSWORD" psql -X -A -t -h 127.0.0.1 -U postgres -d libredb -c "SELECT canary FROM bridge.studio_instances_v1"') + grep -q '"ok"' <<<"$heartbeat" && break + sleep 5 +done +grep -q '"ok"' <<<"$heartbeat" || fail "Studio's seal canary is not ok: $heartbeat" +echo "seal keys and CA of the backup, license active, seal canary ok" + +step "verify: the first admin of the backup signs in, and the user reads the table through the restored credential" +browser verify + +if [ "${LIBREDB_E2E_KEEP:-0}" != "1" ]; then + step "cleanup" + helm -n "$namespace" uninstall libredb --wait >/dev/null + kubectl delete namespace "$namespace" "$target_namespace" --wait=true >/dev/null +fi +printf '\nPASS\n' diff --git a/charts/libredb-platform/tests/e2e/run-rotation.sh b/charts/libredb-platform/tests/e2e/run-rotation.sh new file mode 100755 index 0000000..546a055 --- /dev/null +++ b/charts/libredb-platform/tests/e2e/run-rotation.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# The rotation test of the Helm chart, on the kind cluster run-e2e.sh set up: a fresh installation, then the browser +# suite rotation.k8s.ts, which walks the seal key rotation and the secret rotations of "Rotate secrets and the seal +# key" in docs/deploy/install-kubernetes.md, one helm upgrade per step: +# +# LIBREDB_E2E_LICENSE=kind.license charts/libredb-platform/tests/e2e/run-rotation.sh +# +# LIBREDB_E2E_KEEP=1 keeps the installation. +set -euo pipefail + +here=$(cd "$(dirname "$0")" && pwd) +chart=$(cd "$here/../.." && pwd) +installer=$(cd "$chart/../../installer" && pwd) +license=${LIBREDB_E2E_LICENSE:?set LIBREDB_E2E_LICENSE to a license file for platform.libredb.test} +namespace=${LIBREDB_E2E_NAMESPACE:-libredb} +target_namespace=libredb-rotation-target +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT + +step() { printf '\n=== %s\n' "$*"; } + +step "a fresh installation in namespace $namespace, the target database in $target_namespace" +helm -n "$namespace" uninstall libredb --wait >/dev/null 2>&1 || true +kubectl delete namespace "$namespace" "$target_namespace" --ignore-not-found --wait=true >/dev/null +for ns in "$namespace" "$target_namespace"; do + kubectl create namespace "$ns" >/dev/null + kubectl label namespace "$ns" pod-security.kubernetes.io/enforce=restricted >/dev/null +done +helm install libredb "$chart" -n "$namespace" -f "$here/values-kind.yaml" --set-file platform.license.value="$license" \ + --wait --wait-for-jobs --timeout 10m >/dev/null + +step "the rotations, one helm upgrade per step" +(cd "$installer" && LIBREDB_E2E_ROTATION=1 LIBREDB_E2E_NAMESPACE="$namespace" LIBREDB_E2E_TARGET_NAMESPACE="$target_namespace" \ + LIBREDB_E2E_CHART="$chart" LIBREDB_E2E_VALUES="$here/values-kind.yaml" LIBREDB_E2E_LICENSE="$license" \ + LIBREDB_E2E_ACCOUNT_FILE="$work/first-admin.json" \ + bunx playwright test --config e2e/kubernetes/playwright.config.ts rotation.k8s.ts) + +if [ "${LIBREDB_E2E_KEEP:-0}" != "1" ]; then + step "cleanup" + helm -n "$namespace" uninstall libredb --wait >/dev/null + kubectl delete namespace "$namespace" "$target_namespace" --wait=true >/dev/null +fi +printf '\nPASS\n' diff --git a/charts/libredb-platform/tests/e2e/run-soak.sh b/charts/libredb-platform/tests/e2e/run-soak.sh new file mode 100755 index 0000000..68dcb0d --- /dev/null +++ b/charts/libredb-platform/tests/e2e/run-soak.sh @@ -0,0 +1,155 @@ +#!/usr/bin/env bash +# The soak test of the Helm chart, on the kind cluster run-e2e.sh set up: one installation left running for hours. +# +# LIBREDB_E2E_LICENSE=kind.license LIBREDB_E2E_SOAK_HOURS=8 LIBREDB_E2E_SOAK_OUT=/dir \ +# charts/libredb-platform/tests/e2e/run-soak.sh +# +# After the seed of restore.k8s.ts it checks the three endpoints every 30 seconds, records every 5 minutes each +# container's memory and CPU from the node's CRI, the restart counts, the databases' sizes and the size of the pods' +# logs, and every 30 minutes has the browser suite (phase verify) sign the first admin in and the user read the table. +# The CSV files and summary.txt land in LIBREDB_E2E_SOAK_OUT. It fails when a verify run fails or a container restarts; +# availability, latency, memory and growth go to the summary. LIBREDB_E2E_KEEP=1 keeps the installation. +set -euo pipefail + +here=$(cd "$(dirname "$0")" && pwd) +chart=$(cd "$here/../.." && pwd) +installer=$(cd "$chart/../../installer" && pwd) +license=${LIBREDB_E2E_LICENSE:?set LIBREDB_E2E_LICENSE to a license file for platform.libredb.test} +hours=${LIBREDB_E2E_SOAK_HOURS:-8} +# LIBREDB_E2E_SOAK_MINUTES, for a short trial run, wins over the hours. +seconds=$(( ${LIBREDB_E2E_SOAK_MINUTES:-$(( hours * 60 ))} * 60 )) +out=${LIBREDB_E2E_SOAK_OUT:?a directory for the measurements} +namespace=${LIBREDB_E2E_NAMESPACE:-libredb} +node=${LIBREDB_E2E_KIND_NODE:-libredb-helm-control-plane} +target_namespace=libredb-soak-target +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT +mkdir -p "$out" + +step() { printf '\n=== %s\n' "$*"; } +now() { date -u +%Y-%m-%dT%H:%M:%SZ; } + +browser() { + (cd "$installer" && LIBREDB_E2E_NAMESPACE="$namespace" LIBREDB_E2E_TARGET_NAMESPACE="$target_namespace" \ + LIBREDB_E2E_RESTORE_PHASE="$1" LIBREDB_E2E_RESTORE_STATE="$work/state.json" LIBREDB_E2E_ACCOUNT_FILE="$work/first-admin.json" \ + bunx playwright test --config e2e/kubernetes/playwright.config.ts restore.k8s.ts) +} +psql_super() { + kubectl -n "$namespace" exec -i libredb-postgres-0 -- sh -c \ + 'PGPASSWORD="$POSTGRES_PASSWORD" psql -X -A -t -q -h 127.0.0.1 -U postgres -d libredb' +} + +step "a fresh installation in namespace $namespace, the target database in $target_namespace" +helm -n "$namespace" uninstall libredb --wait >/dev/null 2>&1 || true +kubectl delete namespace "$namespace" "$target_namespace" --ignore-not-found --wait=true >/dev/null +for ns in "$namespace" "$target_namespace"; do + kubectl create namespace "$ns" >/dev/null + kubectl label namespace "$ns" pod-security.kubernetes.io/enforce=restricted >/dev/null +done +helm install libredb "$chart" -n "$namespace" -f "$here/values-kind.yaml" --set-file platform.license.value="$license" \ + --wait --wait-for-jobs --timeout 10m >/dev/null +kubectl -n "$namespace" get secret libredb-tls-platform -o 'jsonpath={.data.ca\.crt}' | base64 -d > "$work/ca.crt" +browser seed + +echo "time,endpoint,status,seconds" > "$out/endpoints.csv" +echo "time,pod,container,memory_bytes,cpu_nanocores" > "$out/memory.csv" +echo "time,restarts,libredb_db_bytes,keycloak_db_bytes,pod_log_bytes" > "$out/growth.csv" +echo "time,verify" > "$out/verify.csv" + +probe() { + local host=$1 path=$2 result + result=$(curl -s --max-time 20 --resolve "$host:443:127.0.0.1" --cacert "$work/ca.crt" -o /dev/null -w '%{http_code},%{time_total}' "https://$host$path" || echo "000,20") + echo "$(now),$host,$result" >> "$out/endpoints.csv" +} +sample() { + local pods + pods=$(docker exec "$node" crictl pods --namespace "$namespace" -q | tr '\n' ' ') + for pod in $pods; do + docker exec "$node" crictl stats -p "$pod" -o json 2>/dev/null | python3 -I -c ' +import json, sys +stats = json.load(sys.stdin).get("stats", []) +for s in stats: + attrs = s["attributes"] + pod = attrs["labels"].get("io.kubernetes.pod.name", "") + mem = s.get("memory", {}).get("workingSetBytes", {}).get("value", 0) + # crictl matches the namespace as a pattern, and a finished job container reports nothing. + if attrs["labels"].get("io.kubernetes.pod.namespace") != sys.argv[2] or mem == 0: + continue + cpu = s.get("cpu", {}).get("usageNanoCores", {}).get("value", 0) + name = attrs["metadata"]["name"] + print(f"{sys.argv[1]},{pod},{name},{mem},{cpu}") +' "$(now)" "$namespace" >> "$out/memory.csv" + done + local restarts sizes logs + restarts=$(kubectl -n "$namespace" get pods -o 'jsonpath={range .items[*]}{range .status.containerStatuses[*]}{.restartCount}{"\n"}{end}{end}' | awk '{s+=$1} END {print s+0}') + sizes=$(echo "SELECT pg_database_size('libredb') || ',' || pg_database_size('keycloak')" | psql_super) + logs=$(docker exec "$node" sh -c "du -sb /var/log/pods/${namespace}_* 2>/dev/null | awk '{s+=\$1} END {print s+0}'") + echo "$(now),$restarts,$sizes,$logs" >> "$out/growth.csv" +} + +step "soak for $(( seconds / 60 )) minutes" +end=$(( $(date +%s) + seconds )) +next_sample=0 +# Every 30 minutes, or twice in a run shorter than an hour. +verify_every=$(( seconds < 3600 ? seconds / 2 : 1800 )) +next_verify=$(( $(date +%s) + verify_every )) +failures=0 +while [ "$(date +%s)" -lt "$end" ]; do + probe platform.libredb.test /api/health/ready + probe studio.libredb.test /api/ready + probe auth.libredb.test /realms/libredb/.well-known/openid-configuration + if [ "$(date +%s)" -ge "$next_sample" ]; then + sample || echo "$(now) sample failed" + next_sample=$(( $(date +%s) + 300 )) + fi + if [ "$(date +%s)" -ge "$next_verify" ]; then + if browser verify > "$work/verify.log" 2>&1; then echo "$(now),pass" >> "$out/verify.csv"; else + echo "$(now),fail" >> "$out/verify.csv"; failures=$((failures + 1)); cp "$work/verify.log" "$out/verify-fail-$(date +%s).log" + fi + next_verify=$(( $(date +%s) + verify_every )) + fi + sleep 30 +done +sample + +step "summary" +python3 -I - "$out" > "$out/summary.txt" <<'EOF' +import csv, sys, statistics, collections +out = sys.argv[1] +rows = list(csv.DictReader(open(f"{out}/endpoints.csv"))) +print(f"endpoint checks: {len(rows)}") +for host in sorted({r["endpoint"] for r in rows}): + mine = [r for r in rows if r["endpoint"] == host] + ok = [r for r in mine if r["status"] == "200"] + times = sorted(float(r["seconds"]) for r in ok) + p99 = times[int(len(times) * 0.99) - 1] if times else 0 + print(f" {host}: {len(ok)}/{len(mine)} answered 200 ({100 * len(ok) / max(len(mine), 1):.3f}%), median {statistics.median(times) if times else 0:.3f} s, p99 {p99:.3f} s, max {max(times) if times else 0:.3f} s") +mem = list(csv.DictReader(open(f"{out}/memory.csv"))) +by = collections.defaultdict(list) +for r in mem: + by[r["container"]].append(int(r["memory_bytes"])) +print("memory, first hour average -> last hour average (MiB):") +for name, values in sorted(by.items()): + first, last = values[:12], values[-12:] + a, b = statistics.mean(first) / 2**20, statistics.mean(last) / 2**20 + print(f" {name}: {a:.0f} -> {b:.0f} ({100 * (b - a) / a if a else 0:+.1f}%), peak {max(values) / 2**20:.0f}") +growth = list(csv.DictReader(open(f"{out}/growth.csv"))) +if growth: + f, l = growth[0], growth[-1] + print(f"restarts: {f['restarts']} -> {l['restarts']}") + for key in ("libredb_db_bytes", "keycloak_db_bytes", "pod_log_bytes"): + print(f"{key}: {int(f[key]) / 2**20:.1f} MiB -> {int(l[key]) / 2**20:.1f} MiB") +verify = list(csv.DictReader(open(f"{out}/verify.csv"))) +print(f"browser verify runs: {sum(r['verify'] == 'pass' for r in verify)} passed, {sum(r['verify'] == 'fail' for r in verify)} failed") +EOF +cat "$out/summary.txt" +restarts=$(tail -1 "$out/growth.csv" | cut -d, -f2) + +if [ "${LIBREDB_E2E_KEEP:-0}" != "1" ]; then + step "cleanup" + helm -n "$namespace" uninstall libredb --wait >/dev/null + kubectl delete namespace "$namespace" "$target_namespace" --wait=true >/dev/null +fi +[ "$failures" -eq 0 ] || { echo "FAIL: $failures browser verify runs failed"; exit 1; } +[ "$restarts" = "0" ] || { echo "FAIL: containers restarted $restarts times"; exit 1; } +printf '\nPASS\n' diff --git a/charts/libredb-platform/tests/e2e/run-upgrade.sh b/charts/libredb-platform/tests/e2e/run-upgrade.sh new file mode 100755 index 0000000..b67ccb7 --- /dev/null +++ b/charts/libredb-platform/tests/e2e/run-upgrade.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +# The release upgrade test, on the installation run-e2e.sh left running (LIBREDB_E2E_KEEP=1), with this chart as the +# installed release and a chart of a later release: +# +# LIBREDB_E2E_LICENSE=kind.license LIBREDB_E2E_NEXT_CHART=/path/to/next/charts/libredb-platform \ +# charts/libredb-platform/tests/e2e/run-upgrade.sh +# +# First an upgrade to the next release that fails after its hooks (Studio's image does not exist) under --atomic +# must roll back to this release, which the installation still counts as its own. Then it upgrades to the next release and checks that the +# migrate job and Platform run the next image, the installation records the next release, the seal keys and the CA +# stay, and Platform and Studio keep working; then that going back, by an upgrade with this chart or by helm rollback, +# is refused at once by the bootstrap job and leaves the next release running, and that the documented recovery from +# the refused rollback's pending record lets the next upgrade through. +set -euo pipefail + +here=$(cd "$(dirname "$0")" && pwd) +chart=$(cd "$here/../.." && pwd) +next=${LIBREDB_E2E_NEXT_CHART:?the chart directory of the next release} +license=${LIBREDB_E2E_LICENSE:?a license for platform.libredb.test} +namespace=${LIBREDB_E2E_NAMESPACE:-libredb} + +step() { printf '\n=== %s\n' "$*"; } +fail() { printf 'FAIL: %s\n' "$*" >&2; exit 1; } +release() { kubectl -n "$namespace" get secret libredb-installer-state -o 'jsonpath={.data.release}' | base64 -d; } +hash_of() { kubectl -n "$namespace" get secret "$1" -o "jsonpath={.data.$2}" | sha256sum | cut -d' ' -f1; } +image() { kubectl -n "$namespace" get deploy libredb-platform -o 'jsonpath={.spec.template.spec.containers[0].image}'; } +ready() { curl -s --resolve "$1:443:127.0.0.1" -k -o /dev/null -w '%{http_code}' "https://$1$2"; } +deploy() { + helm upgrade libredb "$1" -n "$namespace" -f "$here/values-kind.yaml" --set-file platform.license.value="$license" \ + --wait --wait-for-jobs --timeout 10m +} + +next_version=$(awk '/^appVersion:/{gsub(/"/,"",$2); print $2}' "$next/Chart.yaml") +this_version=$(awk '/^appVersion:/{gsub(/"/,"",$2); print $2}' "$chart/Chart.yaml") +seal=$(hash_of libredb-secrets 'seal-keys\.json') +ca=$(hash_of libredb-tls 'ca\.crt') +[ "$(release)" = "$this_version" ] || fail "the installation runs $(release), not $this_version" + +step "a failed upgrade under --atomic rolls back to $this_version" +if helm upgrade libredb "$next" -n "$namespace" -f "$here/values-kind.yaml" --set-file platform.license.value="$license" \ + --set studio.image.tag=does-not-exist --atomic --wait --wait-for-jobs --timeout 3m > /tmp/atomic.log 2>&1; then + fail "the upgrade to a missing image succeeded" +fi +grep -q "rolled back\|rollback" /tmp/atomic.log || fail "--atomic did not roll back: $(tail -2 /tmp/atomic.log)" +status=$(helm -n "$namespace" status libredb -o json | node -e 'let t="";process.stdin.on("data",c=>t+=c).on("end",()=>console.log(JSON.parse(t).info.status))') +[ "$status" = "deployed" ] || fail "the release is $status after the atomic rollback" +case "$(image)" in *":$this_version"*) ;; *) fail "Platform runs $(image) after the atomic rollback" ;; esac +[ "$(release)" = "$this_version" ] || fail "the installation records $(release) after the atomic rollback" +for _ in $(seq 1 60); do [ "$(ready platform.libredb.test /api/health/ready)" = "200" ] && break; sleep 3; done +[ "$(ready platform.libredb.test /api/health/ready)" = "200" ] || fail "Platform is not ready after the atomic rollback" +echo "rolled back to $this_version, release deployed, Platform ready" + +step "upgrade $this_version -> $next_version" +deploy "$next" >/dev/null +kubectl -n "$namespace" rollout status deploy/libredb-platform --timeout=300s >/dev/null +case "$(image)" in *":$next_version"*) ;; *) fail "Platform runs $(image) after the upgrade" ;; esac +migrate=$(kubectl -n "$namespace" get jobs -l app.kubernetes.io/component=platform-migrate -o 'jsonpath={.items[*].spec.template.spec.containers[0].image}') +case "$migrate" in *":$next_version"*) ;; *) fail "the migrate job ran $migrate" ;; esac +[ "$(release)" = "$next_version" ] || fail "the installation records $(release) after the upgrade" +[ "$(hash_of libredb-secrets 'seal-keys\.json')" = "$seal" ] || fail "the upgrade changed the seal keys" +[ "$(hash_of libredb-tls 'ca\.crt')" = "$ca" ] || fail "the upgrade changed the internal CA" +[ "$(ready platform.libredb.test /api/health/ready)" = "200" ] || fail "Platform is not ready on $next_version" +[ "$(ready studio.libredb.test /api/ready)" = "200" ] || fail "Studio is not ready on $next_version" +kubectl -n "$namespace" logs deploy/libredb-platform | grep -c '"message":"License active"' >/dev/null || fail "Platform $next_version did not log License active" +echo "Platform and its migrate job run $next_version, the installation records it, seal keys and CA kept" + +step "an upgrade back to $this_version is refused" +if deploy "$chart" > /tmp/downgrade.log 2>&1; then fail "the downgrade to $this_version succeeded"; fi +kubectl -n "$namespace" logs job/libredb-bootstrap 2>/dev/null | grep -c "a downgrade is not supported" >/dev/null \ + || grep -q "downgrade is not supported\|pre-upgrade hooks failed" /tmp/downgrade.log || fail "the downgrade failed for another reason: $(tail -3 /tmp/downgrade.log)" +case "$(image)" in *":$next_version"*) ;; *) fail "Platform runs $(image) after the refused downgrade" ;; esac +[ "$(ready platform.libredb.test /api/health/ready)" = "200" ] || fail "Platform is not ready after the refused downgrade" +echo "refused: $(kubectl -n "$namespace" logs job/libredb-bootstrap 2>/dev/null | grep -o 'this chart runs LibreDB [^:]*' | head -1)" + +step "helm rollback to the $this_version revision is refused" +target=$(helm -n "$namespace" history libredb -o json | node -e ' + let text = ""; process.stdin.on("data", (chunk) => (text += chunk)).on("end", () => { + const deployed = JSON.parse(text).filter((r) => r.app_version === process.argv[1] && r.status !== "failed"); + console.log(deployed.length === 0 ? "" : deployed[deployed.length - 1].revision); + });' "$this_version") +[ -n "$target" ] && [ "$target" != "null" ] || fail "no deployed revision of $this_version to roll back to" +if helm -n "$namespace" rollback libredb "$target" --wait --timeout 10m > /tmp/rollback.log 2>&1; then fail "the rollback to $this_version succeeded"; fi +grep -q "PodFailurePolicy\|pre-rollback hooks failed" /tmp/rollback.log || fail "the rollback failed for another reason: $(tail -3 /tmp/rollback.log)" +kubectl -n "$namespace" logs job/libredb-bootstrap | grep -c "a downgrade is not supported" >/dev/null || fail "the rollback's bootstrap failed for another reason" +case "$(image)" in *":$next_version"*) ;; *) fail "Platform runs $(image) after the refused rollback" ;; esac +[ "$(ready platform.libredb.test /api/health/ready)" = "200" ] || fail "Platform is not ready after the refused rollback" +echo "refused at once; Platform still runs $next_version" + +step "the documented recovery from the refused rollback's pending record" +pending=$(helm -n "$namespace" history libredb -o json | node -e 'let t="";process.stdin.on("data",c=>t+=c).on("end",()=>{const p=JSON.parse(t).filter(r=>r.status==="pending-rollback");console.log(p.length?p[p.length-1].revision:"")})') +[ -n "$pending" ] || fail "Helm left no pending-rollback record, so the recovery step of the docs is stale" +kubectl -n "$namespace" delete secret "sh.helm.release.v1.libredb.v$pending" >/dev/null +deploy "$next" >/dev/null || fail "the upgrade after the recovery failed" +[ "$(ready platform.libredb.test /api/health/ready)" = "200" ] || fail "Platform is not ready after the recovery" +echo "removed the pending record v$pending, and the next upgrade went through" + +printf '\nPASS\n' diff --git a/charts/libredb-platform/tests/e2e/traefik-values.yaml b/charts/libredb-platform/tests/e2e/traefik-values.yaml new file mode 100644 index 0000000..86d7db2 --- /dev/null +++ b/charts/libredb-platform/tests/e2e/traefik-values.yaml @@ -0,0 +1,23 @@ +# The ingress controller of the kind end-to-end test: Traefik on the node ports that kind-config.yaml maps to +# 127.0.0.1:80 and 127.0.0.1:443 of the host. +ingressClass: + enabled: true + isDefaultClass: true +service: + spec: + type: NodePort +ports: + web: + nodePort: 30080 + websecure: + nodePort: 30443 +providers: + kubernetesIngress: + enabled: true + # Publish an address on every Ingress, as a cloud load balancer does: Argo CD counts an Ingress healthy only then. + ingressEndpoint: + ip: 127.0.0.1 + publishedService: + enabled: false + kubernetesCRD: + enabled: false diff --git a/charts/libredb-platform/tests/e2e/values-kind.yaml b/charts/libredb-platform/tests/e2e/values-kind.yaml new file mode 100644 index 0000000..9af7d95 --- /dev/null +++ b/charts/libredb-platform/tests/e2e/values-kind.yaml @@ -0,0 +1,20 @@ +# The values of the end-to-end test: the three hosts under .libredb.test, which CoreDNS resolves to Traefik in the +# cluster and the browser maps to 127.0.0.1. The license comes from run-e2e.sh (--set-file platform.license.value). +hosts: + platform: platform.libredb.test + studio: studio.libredb.test + auth: auth.libredb.test +platform: + organizationName: Acme Test +studio: + config: + oidcIssuer: https://auth.libredb.test/realms/libredb + ingress: + hosts: + - host: studio.libredb.test + paths: + - path: / + pathType: Prefix + tls: + - secretName: libredb-tls-studio + hosts: [studio.libredb.test] diff --git a/charts/libredb-platform/tests/e2e/values-openshift.yaml b/charts/libredb-platform/tests/e2e/values-openshift.yaml new file mode 100644 index 0000000..8bf209f --- /dev/null +++ b/charts/libredb-platform/tests/e2e/values-openshift.yaml @@ -0,0 +1,21 @@ +# The values of the OpenShift test (Red Hat Developer Sandbox, one namespace, restricted-v2 SCC): three hosts under +# the cluster's apps domain, which the router serves; OpenShift turns each Ingress into a Route with the Ingress's TLS +# Secret. LIBREDB_OS_DOMAIN in run-openshift.sh replaces apps.example.com. +hosts: + platform: ldb-platform.apps.example.com + studio: ldb-studio.apps.example.com + auth: ldb-auth.apps.example.com +platform: + organizationName: Acme OpenShift +studio: + config: + oidcIssuer: https://ldb-auth.apps.example.com/realms/libredb + ingress: + hosts: + - host: ldb-studio.apps.example.com + paths: + - path: / + pathType: Prefix + tls: + - secretName: libredb-tls-studio + hosts: [ldb-studio.apps.example.com] diff --git a/charts/libredb-platform/values.yaml b/charts/libredb-platform/values.yaml new file mode 100644 index 0000000..73d6e8e --- /dev/null +++ b/charts/libredb-platform/values.yaml @@ -0,0 +1,280 @@ +# LibreDB Platform on Kubernetes. docs/deploy/install-kubernetes.md is the guide; this file documents every value. +# +# The chart is the installer's Kubernetes mode driven by Helm: it renders these values as a LibreDBInstallation +# (installer/src/config/schema.ts, mode kubernetes), and its jobs run the installer's own code on it. One installation +# per namespace: the objects have fixed names, like the installer's Kubernetes constants. + +global: + compatibility: + openshift: + # -- OpenShift's restricted-v2 SCC: "auto" leaves out the pods' fixed runAsUser, runAsGroup and fsGroup when + # the API server is OpenShift, so the SCC assigns ids from the namespace's range; "force" always leaves them + # out, "disabled" never. The Studio chart reads the same value. + adaptSecurityContext: auto + +installation: + # -- metadata.name of the installation: a lowercase DNS label, which names the internal CA. + name: libredb + +# -- The public host of each app. Each needs DNS that the users' browsers and the pods of this release resolve to the +# ingress controller: Platform and Studio call Keycloak's issuer URL from inside the cluster. +hosts: + platform: "" + studio: "" + # -- Keycloak's host, the host of the issuer https:///realms/. With keycloak.mode external it must be + # the host of keycloak.external.baseUrl. + auth: "" + +platform: + # -- The organization's name, which the first install gives the installation's one tenant. + organizationName: "" + # -- Days Platform keeps query events, from 1 to 36500. null keeps Platform's default, 90. + queryLogRetentionDays: null + # -- The license from LibreDB (required): the token, one line that starts with ldl1. Give it inline with + # `--set-file platform.license.value=acme.license`, or name a Secret that holds it. The license names the + # Platform host, so hosts.platform must be that host. + license: + value: "" + existingSecret: "" + key: license + # -- The seal key rotation of installer/docs/runbooks/seal-key-rotation.md, one helm upgrade per step, as + # bridge.sealKeys of the installation: issueNew adds a live key to Studio's keyring, activeKid (16 hex digits) names + # the key Platform seals to (the oldest live key when empty), and retire lists kids to take out of Studio's keyring, + # which an upgrade refuses while a connection still uses one. The secret store keeps every key. + sealKeys: + issueNew: false + activeKid: "" + retire: [] + # -- During a rotation of Platform's AUTH_SECRET: a Secret that holds the previous value, which Platform keeps + # accepting for the sessions it signed, as platform.authSecretPreviousFile of the installation. + authSecretPrevious: + existingSecret: "" + key: auth-secret-previous + image: + repository: ghcr.io/libredb/libredb-platform + # -- The release's tag and digest, from installer/release/release.config.json. + tag: "1.0.1@sha256:b42e346756b4e39a1ed99eee4990092b139dcdc1e0c561069c12ee1bcc081268" + pullPolicy: IfNotPresent + resources: + requests: + cpu: 100m + memory: 512Mi + limits: + memory: 1Gi + # -- Support attachments under /app/uploads. Disabled keeps them in an emptyDir, which a restart loses. + persistence: + enabled: true + size: 5Gi + storageClass: "" + accessModes: + - ReadWriteOnce + podAnnotations: {} + podLabels: {} + # -- Platform's pods and the jobs that run its image. The chart adds kubernetes.io/arch: amd64 unless this names an + # arch: the Platform image is built for linux/amd64 alone. + nodeSelector: {} + tolerations: [] + affinity: {} + +tls: + # -- generated: the internal CA issues the ingress certificates of the three hosts, as the installer's + # tls.mode generated does; browsers must trust the CA (Secret libredb-tls-platform holds it as ca.crt). + # secrets: the ingress uses TLS Secrets you provide, such as cert-manager's, named in tls.secrets. + mode: generated + secrets: + platform: "" + studio: "" + auth: "" + # -- With tls.mode secrets, the PEM of the CA that issued those certificates (required there), which every pod + # trusts. A public CA's root works too. + caBundle: "" + +ingress: + # -- The Ingress of Platform and of a managed Keycloak. Studio's Ingress is studio.ingress. + enabled: true + className: "" + annotations: + # Platform's sign-in callback sets its session cookie in chunks that hold the Keycloak ID and refresh tokens, + # more response header than ingress-nginx buffers by default (4k), which then answers the callback 502. Other + # controllers ignore the annotation; an nginx-based one of another vendor needs its own setting of 16k. + nginx.ingress.kubernetes.io/proxy-buffer-size: "16k" + +postgresql: + # -- managed: the chart runs PostgreSQL with verified TLS from the internal CA and provisions it, as the installer + # does. external: your server, provisioned once by a DBA with the script the chart writes to Secret + # libredb-postgres-dba-script. + mode: managed + managed: + image: + repository: docker.io/library/postgres + tag: "18.6@sha256:74935e72241653ca55e0414067e6d8763aceb8a810eb51b452253ec3dcfc4336" + pullPolicy: IfNotPresent + persistence: + size: 10Gi + storageClass: "" + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + memory: 1Gi + nodeSelector: {} + tolerations: [] + affinity: {} + external: + # -- The server's DNS name, which its certificate covers: Platform and Studio connect with sslmode=verify-full. + host: "" + port: 5432 + # -- The PEM of the CA that issued the server's certificate. + caBundle: "" + +keycloak: + # -- managed: the chart runs Keycloak behind the installer's edge rules and configures its realm. external: your + # Keycloak, configured by the chart's job (provisioning installer) or by your identity team (provisioning manual). + mode: managed + realm: libredb + # -- The installer's first accounts, created with temporary passwords in Secret libredb-keycloak-config. Enabled + # by default with a managed Keycloak only; provisioning manual creates none. + firstAdmin: + enabled: null + username: libredb-admin + userManager: + enabled: null + username: libredb-user-manager + managed: + image: + repository: quay.io/keycloak/keycloak + tag: "26.8.1@sha256:79c623a22abacf3fe6c00a5fb30f5b7843d717b234033b80995891c06116d244" + pullPolicy: IfNotPresent + # -- The installer's edge in front of Keycloak's public paths: only /realms// and /resources/, and no + # non-canonical path, reach Keycloak (installer/src/render/caddyfile.ts). + edge: + image: + repository: docker.io/library/caddy + tag: "2.11.7@sha256:685ae3d7c34d9b5f8caeddc5d14051db7bbd88ff1f5a2fa0c16bd8ca328fb7af" + pullPolicy: IfNotPresent + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + memory: 128Mi + resources: + requests: + cpu: 200m + memory: 768Mi + limits: + memory: 1536Mi + nodeSelector: {} + tolerations: [] + affinity: {} + external: + # -- The https base URL of your Keycloak, without /realms. + baseUrl: "" + # -- Where the jobs and Platform reach the Admin API, when not at baseUrl. + adminUrl: "" + # -- installer: the chart's job configures the realm with the admin credential below. manual: your identity + # team configures it from `libredb-installer keycloak export` and gives the two client secrets. + provisioning: installer + adminRealm: master + # -- provisioning installer: a Secret with the admin credential, either a client (adminClientId plus the key + # admin-client-secret) or a user (adminUser plus the key admin-password). + adminSecret: "" + adminClientId: "" + adminUser: "" + # -- provisioning manual: a Secret with the keys studio-client-secret and platform-client-secret. + clientSecretsSecret: "" + # -- The PEM of the CA of your Keycloak's certificate, when a private CA issued it. + caBundle: "" + +network: + # -- An http(s) proxy for the apps' outbound HTTPS, written as HTTPS_PROXY with the installer's NO_PROXY. + httpsProxy: "" + +jobs: + # -- The bootstrap hook, the PostgreSQL provisioning, the Keycloak and the migrate jobs. + backoffLimit: 6 + activeDeadlineSeconds: 900 + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + memory: 512Mi + +networkPolicy: + # -- PostgreSQL and Keycloak's Admin API reachable only from this release's pods, like the installer's internal + # network; the ingress reaches Keycloak through its edge alone. Takes effect with a CNI that enforces policies. + enabled: true + +imagePullSecrets: [] + +# -- The Studio chart (oci of https://libredb.org/libredb-studio/, pinned by the release manifest) under governance. +# Repeat hosts.studio as the ingress host and the issuer of hosts.auth here: Helm passes no values down to a +# dependency, so the chart refuses to render when they differ. +studio: + enabled: true + fullnameOverride: libredb-studio + image: + repository: ghcr.io/libredb/libredb-studio + tag: "1.0.0@sha256:0cb0177dd2789534f2dbfbc6673104ec0ce70ee82815f5fe9bec5efe3d329ce3" + authProvider: oidc + governance: + enabled: true + postgresSchema: bridge + secrets: + # -- Written by the bootstrap job; keep this name. + existingSecret: libredb-studio-env + adminEmail: "" + userEmail: "" + config: + # -- Must be https:///realms/. + oidcIssuer: "" + oidcRoleClaim: libredb_roles + oidcAdminRoles: admin + oidcUserRoles: user + oidcPrompt: omit + oidcAutoRedirect: true + oidcLogoutHint: true + sessionTtlSeconds: 28800 + authBootstrap: "off" + storageProvider: postgres + storagePostgresSchema: studio + postgresql: + enabled: false + ingress: + enabled: true + className: "" + annotations: {} + # -- Must hold one host, hosts.studio. + hosts: + - host: "" + paths: + - path: / + pathType: Prefix + # -- tls.mode generated: libredb-tls-studio with hosts.studio. + tls: + - secretName: libredb-tls-studio + hosts: [] + extraEnv: + - name: STORAGE_ENCRYPTION_KEY + valueFrom: + secretKeyRef: + name: libredb-studio-env + key: storage-encryption-key + - name: NODE_EXTRA_CA_CERTS + value: /etc/libredb/ca/ca.crt + # -- The installer's network environment of Studio (NO_PROXY, and HTTPS_PROXY with network.httpsProxy), which the + # bootstrap job writes. + extraEnvFrom: + - secretRef: + name: libredb-studio-network-env + optional: true + extraVolumes: + - name: libredb-ca + configMap: + name: libredb-ca + extraVolumeMounts: + - name: libredb-ca + mountPath: /etc/libredb/ca + readOnly: true diff --git a/docs/deploy/install-docker.md b/docs/deploy/install-docker.md index 017402a..7145df3 100644 --- a/docs/deploy/install-docker.md +++ b/docs/deploy/install-docker.md @@ -4,7 +4,7 @@ This guide installs LibreDB 1.0.1 on one x86_64 Linux host with Docker. The installer, `libredb-installer`, runs on that host as root and deploys PostgreSQL, Keycloak, LibreDB Platform, LibreDB Studio and a Caddy edge as the Docker Compose project `libredb`. The guide takes the shortest path: a PostgreSQL and a Keycloak that the installer runs itself, one DNS name per app, and certificates from the installer's internal CA. The [installer reference](../../installer/README.md) covers the other choices: your own certificates or TLS proxy, an existing PostgreSQL server or Keycloak realm, one host without DNS, and a registry mirror. -Kubernetes support comes in the next release. +To install on Kubernetes instead, use the Helm chart, as the [Kubernetes install guide](install-kubernetes.md) describes. ## What you need diff --git a/docs/deploy/install-kubernetes.md b/docs/deploy/install-kubernetes.md new file mode 100644 index 0000000..e2ab6bc --- /dev/null +++ b/docs/deploy/install-kubernetes.md @@ -0,0 +1,347 @@ +# Install LibreDB on Kubernetes + +This guide installs LibreDB 1.0.1 on a Kubernetes cluster with the Helm chart [`charts/libredb-platform`](../../charts/libredb-platform). +The chart deploys LibreDB Platform, its migrate job and LibreDB Studio under governance, and by default a PostgreSQL server and a Keycloak that it runs itself. +It is the installer's Kubernetes mode driven by Helm: the chart renders its values as an installation configuration of `mode: kubernetes`, and its jobs run the installer's own code on it, so a Helm install gets the same secrets, certificates, environments, PostgreSQL provisioning and Keycloak realm as a [Docker install](install-docker.md) of the same configuration. +The [chart README](../../charts/libredb-platform/README.md) lists every value. + +## What you need + +- Kubernetes 1.28 or newer, with a default StorageClass for the PostgreSQL volume and Platform's uploads. +- An ingress controller with an IngressClass, such as Traefik, HAProxy or ingress-nginx, which terminates TLS for the three hosts and accepts response headers of 16 KB: Platform's sign-in callback sets its session cookie in chunks, more than the 4 KB that nginx buffers by default. + The chart sets `nginx.ingress.kubernetes.io/proxy-buffer-size: "16k"` on its Ingresses for ingress-nginx; another nginx-based controller needs its own setting, or the first sign-in ends in a 502 from the controller. +- Helm 3. +- At least one x86_64 (amd64) node: the Platform image is built for linux/amd64 alone, so the chart schedules Platform and the jobs that run its image with `kubernetes.io/arch: amd64`; Studio, Keycloak and PostgreSQL also run on arm64 nodes. +- Three DNS names, for Platform, Studio and Keycloak, such as `libredb.acme.example`, `studio.acme.example` and `sso.acme.example`, which resolve to the ingress controller from the users' browsers **and from the cluster's pods**: Platform and Studio call Keycloak's issuer URL `https:///realms/libredb` from inside the cluster. +- A license file from LibreDB for the Platform host, as [Request a license](install-docker.md#request-a-license) describes. + +One namespace holds one installation: the chart's objects have fixed names. + +## Write the values + +```sh +cat > libredb-values.yaml <<'EOF' +hosts: + platform: libredb.acme.example + studio: studio.acme.example + auth: sso.acme.example +platform: + organizationName: Acme Corp +studio: + config: + oidcIssuer: https://sso.acme.example/realms/libredb + ingress: + hosts: + - host: studio.acme.example + paths: + - path: / + pathType: Prefix + tls: + - secretName: libredb-tls-studio + hosts: [studio.acme.example] +EOF +``` + +Helm passes no values down to a dependency, so the Studio host and the issuer appear twice; the chart refuses to render when the copies under `studio:` differ from `hosts`. +Set `ingress.className` and `studio.ingress.className` when the cluster has no default IngressClass. + +## Install + +```sh +helm repo add libredb-studio https://libredb.org/libredb-studio/ +helm dependency build charts/libredb-platform +helm install libredb charts/libredb-platform -n libredb --create-namespace \ + -f libredb-values.yaml --set-file platform.license.value=acme.license \ + --wait --wait-for-jobs --timeout 10m +``` + +`helm dependency build` takes the Studio chart that `Chart.lock` pins only from a repository Helm knows, so without the `helm repo add` it stops with `no repository definition for https://libredb.org/libredb-studio/`. +Instead of `--set-file`, `platform.license.existingSecret` names a Secret that holds the token under the key `license`. +The install runs in this order, and `--wait --wait-for-jobs` returns once every step is done: + +1. The bootstrap hook generates the secrets, the seal keys and the internal CA, and writes every Secret and ConfigMap of the installation. +2. PostgreSQL starts with TLS from the internal CA, and the provisioning job runs the installer's provisioning script, which holds SCRAM verifiers only. +3. Keycloak starts, and the keycloak-configure job converges the realm `libredb`, its clients, roles and the two first accounts, as the installer does. +4. The migrate job migrates Platform's database once the PostgreSQL roles exist, and Platform turns ready. +5. Studio turns ready once Platform's bridge exists. + +On a small cluster the whole install takes about a minute once the images are pulled. + +## Check the license + +```sh +kubectl -n libredb logs deploy/libredb-platform | grep -i licen +``` + +Expected: a line with `"message":"License active"`. +A license that does not name `hosts.platform` starts the install in state `invalid`, as [Check the license](install-docker.md#check-the-license) describes. + +## Trust the internal CA + +With the default `tls.mode: generated`, the internal CA issues the certificates of the three hosts, so every browser that opens them must trust it: + +```sh +kubectl -n libredb get secret libredb-tls-platform -o jsonpath='{.data.ca\.crt}' | base64 -d > libredb-ca.crt +``` + +Import it only where the cluster's operators are trusted: the CA can sign a certificate for any name. +With certificates of your own, such as cert-manager's, set `tls.mode: secrets`, name the TLS Secret of each host in `tls.secrets` (and Studio's again as `studio.ingress.tls[0].secretName`), and give the CA that issued them in `tls.caBundle`, which every pod then trusts; a public CA's root works too. + +## Sign in for the first time + +```sh +kubectl -n libredb get secret libredb-keycloak-config -o jsonpath='{.data.first-admin-password}' | base64 -d; echo +``` + +Open the Platform host, choose Continue with SSO and sign in as `libredb-admin` with that temporary password. +Keycloak then asks for a new password and a TOTP device, as the [Docker guide](install-docker.md#sign-in-for-the-first-time) describes. +`libredb-user-manager` manages the realm's users in Keycloak's admin console, which the ingress does not expose: + +```sh +kubectl -n libredb get secret libredb-keycloak-config -o jsonpath='{.data.user-manager-password}' | base64 -d; echo +kubectl -n libredb port-forward svc/keycloak 18080:8080 +``` + +Then open `http://127.0.0.1:18080/admin/libredb/console/`, the admin URL a Docker install uses too. + +## Back up and restore + +The chart has no scheduled backups, so an installation with a managed PostgreSQL is backed up and restored by hand. +A backup is one set, taken together: the Secrets `libredb-secrets` (the seal keys, which open every database credential Platform stores, the role passwords and the client secrets), `libredb-tls` (the internal CA) and `libredb-installer-state`, and a `pg_dump -Fc` of the databases `libredb` and `keycloak`. +The Secrets must come from the same set as the databases, because the databases hold envelopes sealed to those keys. +Take the first set on day one; the set holds every secret of the installation, so store it like the secrets themselves. +With an external PostgreSQL, the site backs up its server, and the set is the three Secrets. + +```sh +( +set -eu +BACKUP="libredb-backup-$(date -u +%Y%m%dT%H%M%SZ)" +mkdir -m 0700 "$BACKUP" +kubectl -n libredb get secret libredb-secrets libredb-tls libredb-installer-state -o yaml > "$BACKUP/secrets.yaml" +for db in libredb keycloak; do + kubectl -n libredb exec libredb-postgres-0 -- sh -c \ + 'PGPASSWORD="$POSTGRES_PASSWORD" pg_dump -h 127.0.0.1 -U postgres -Fc -d "$1"' sh "$db" > "$BACKUP/$db.dump" +done +echo "$BACKUP" +) +``` + +The bootstrap job's Secrets and ConfigMaps carry no Helm ownership, Platform's upload volume carries `helm.sh/resource-policy: keep`, and PostgreSQL's volume claim outlives its StatefulSet by Kubernetes' own rule, so `helm uninstall` leaves them, and a later install under the same release name in the same namespace reuses the seal keys, the CA, the database and the realm. +Deleting the namespace deletes them for good, and then only a backup set brings the installation back. + +To restore a set into a new namespace or a new cluster: + +1. Create the namespace under the same name: `secrets.yaml` names it, and `kubectl create` refuses it in another one. + Put back the Secrets before the install, so the bootstrap job reuses them instead of generating new ones: + + ```sh + kubectl create namespace libredb + kubectl -n libredb create -f "$BACKUP/secrets.yaml" + ``` + +2. Install the chart with the same values and license as before; it provisions empty databases with the restored role passwords and starts every component. +3. Stop the apps and Keycloak, which leaves PostgreSQL running, and put back each database; the archive recreates every object with its owner and its privileges: + + ```sh + kubectl -n libredb scale deployment libredb-platform libredb-studio keycloak --replicas=0 + kubectl -n libredb wait --for=delete pod -l 'app.kubernetes.io/component in (platform,keycloak)' --timeout=180s + kubectl -n libredb wait --for=delete pod -l app.kubernetes.io/name=studio --timeout=180s + for db in libredb keycloak; do + kubectl -n libredb exec -i libredb-postgres-0 -- sh -c \ + 'PGPASSWORD="$POSTGRES_PASSWORD" pg_restore -h 127.0.0.1 -U postgres --clean --if-exists --exit-on-error -d "$1"' sh "$db" \ + < "$BACKUP/$db.dump" + done + kubectl -n libredb scale deployment libredb-platform libredb-studio keycloak --replicas=1 + ``` + +4. Review what the restore undid: every grant and revocation made after the backup point is gone, so a user whose access was withdrawn since then has it again; withdraw it again in Platform. + +The [Docker runbook](../../installer/docs/runbooks/restore.md#a-database-copy-older-than-a-seal-key-retirement) covers a database copy older than a seal key retirement, which applies here alike. + +## An existing PostgreSQL server + +```yaml +postgresql: + mode: external + external: + host: pg.acme.example + port: 5432 + caBundle: | + -----BEGIN CERTIFICATE----- + ... +``` + +The host must be a DNS name its certificate covers: Platform and Studio connect with `sslmode=verify-full`. +The chart holds no admin credential of the server. +Its bootstrap job writes the provisioning script, with SCRAM verifiers only, to Secret `libredb-postgres-dba-script`, and a DBA runs it once, so install without waiting for the jobs, then let the DBA run the script, then wait: + +```sh +helm install libredb charts/libredb-platform -n libredb --create-namespace -f libredb-values.yaml \ + --set-file platform.license.value=acme.license +kubectl -n libredb get secret libredb-postgres-dba-script -o jsonpath='{.data.provision\.sql}' | base64 -d > libredb-provision.sql +psql "host=pg.acme.example port=5432 dbname=postgres user= sslmode=verify-full sslrootcert=" -f libredb-provision.sql +kubectl -n libredb wait --for=condition=complete job -l app.kubernetes.io/component=platform-migrate --timeout=10m +``` + +The migrate job waits until the roles exist; when the DBA runs the script later than the job's deadline of 15 minutes, `helm upgrade` with the same values runs it again. +The [connection budget](platform-runtime.md#connection-budget) applies to the external server. + +## An existing Keycloak + +```yaml +keycloak: + mode: external + realm: libredb + external: + baseUrl: https://sso.acme.example + provisioning: installer + adminSecret: keycloak-admin + adminClientId: libredb-provisioner +``` + +With `provisioning: installer`, the keycloak-configure job converges the realm's LibreDB objects with the admin credential in the Secret that `adminSecret` names, under the key `admin-client-secret` for a client, or with `adminUser` and the key `admin-password` for a user of `adminRealm` (default `master`). +With `provisioning: manual`, your identity team configures the realm from `libredb-installer keycloak export` or [keycloak-manual.md](../../installer/docs/keycloak-manual.md), and `clientSecretsSecret` names a Secret with the keys `studio-client-secret` and `platform-client-secret`. +`hosts.auth` must be the host of `baseUrl`, and `caBundle` takes the CA of the Keycloak's certificate when a private CA issued it. + +## Upgrade + +```sh +helm upgrade libredb charts/libredb-platform -n libredb -f libredb-values.yaml \ + --set-file platform.license.value=acme.license --wait --wait-for-jobs --timeout 10m +``` + +Every upgrade runs the bootstrap hook, the provisioning, keycloak-configure and migrate jobs again, which converge without changing what is already right. +The apps read their Secrets and ConfigMaps when they start, so the bootstrap hook records each workload whose inputs it changed, and the restart hook, which runs once Helm applied the release, restarts it unless its pods already started after the change: Platform for its environment or the CA bundle, Studio for its Secrets or the CA bundle, Keycloak for its own, and PostgreSQL for a renewed leaf. +So a license renewal, a rotated client secret or a new CA reaches the apps with an upgrade, whether the value comes from these values or from a Secret you provide; nothing watches a Secret you provide between upgrades. +A request to restart outlives a failed run, and the next upgrade carries it out. +A downgrade is not supported: the bootstrap job, which runs before a rollback too, refuses an older release than the installation ran, so restore from a [backup](#back-up-and-restore) instead. +A rollback within the same release, such as one that undoes a wrong value, or the one `helm upgrade --atomic` runs after a failure, runs the bootstrap job with the earlier values; the installation counts a release as its own only once its upgrade succeeded. + +Two Helm behaviours to know when something goes back: + +- `helm rollback` to an older release fails at once on the bootstrap job's refusal, the earlier release keeps running, and Helm leaves the record of that rollback `pending-rollback`, which blocks the next operation. Remove that record, the revision `helm history libredb` shows as `pending-rollback`, with `kubectl -n libredb delete secret sh.helm.release.v1.libredb.v`, then continue with `helm upgrade`. +- When an upgrade fails in its bootstrap hook, nothing was applied, and the rollback of `--atomic` may still fail with `no Job with the name ... found`; the earlier release keeps running, and the next `helm upgrade` goes through. + +## Rotate secrets and the seal key + +Each step below is one `helm upgrade` with your values, as [Upgrade](#upgrade) runs it; keep what a step sets in your values file, because the next upgrade without it would undo it. +Back up Secret `libredb-secrets` before a rotation and keep that copy until the new values work, because a rollback cannot restore a secret value. +`PSQL` below is the managed server's superuser, for the checks: + +```sh +PSQL='kubectl -n libredb exec -i libredb-postgres-0 -- sh -c "PGPASSWORD=\$POSTGRES_PASSWORD psql -X -h 127.0.0.1 -U postgres -d libredb"' +``` + +### The seal key + +Platform seals every database credential to Studio's public seal key and can never open or re-seal one, so the rotation adds a key to Studio first, switches Platform to it, has an admin enter the credentials again, and only then retires the old key, as [the runbook of a Docker install](../../installer/docs/runbooks/seal-key-rotation.md) describes; `platform.sealKeys` stands for its `bridge.sealKeys`. + +1. Note the kid that seals today's credentials, the second dot-separated field of each envelope, and the kids every live Studio instance holds: + + ```sh + echo "SELECT instance_id, keyring_kids, canary FROM bridge.studio_instance WHERE last_seen_at > now() - interval '2 minutes'" | eval "$PSQL" + ``` + +2. Set `platform.sealKeys.issueNew: true` and upgrade. + The bootstrap job issues a new key into the secret store and Studio's keyring, and Studio restarts; Platform keeps sealing to the old key. + Run the query of step 1 until every instance lists the new kid and still reports `ok` for the old one. +3. Set `issueNew: false` and `activeKid: ` in the same edit, and upgrade: Platform restarts with the new public key and publishes a canary for it; wait until every instance reports `ok` for the new kid. + Upgrading with `issueNew: true` again issues no second key while the newest key is the active one. +4. Enter again, in the edit form of each connection, which warns about a credential sealed to an earlier key, every credential the old kid still seals; this query lists them, and must come back empty: + + ```sh + echo "SELECT name, split_part(read_secret, '.', 2) AS read_kid, split_part(write_secret, '.', 2) AS write_kid FROM bridge.connections_v1" | eval "$PSQL" + ``` + + Re-entering a credential closes the affected users' Studio provider sessions, so schedule it outside working hours. +5. Keep `activeKid`, add `retire: []`, and upgrade. + Before it changes anything, the bootstrap job asks PostgreSQL, as `libredb_platform`, which connections the old kid still seals, and while one does the upgrade fails at once (`pre-upgrade hooks failed`), its log naming each connection (`kubectl -n libredb logs job/libredb-bootstrap -c bootstrap`), and nothing changes; enter those credentials and upgrade again. + Otherwise the old key leaves Studio's keyring, the secret store keeps it in escrow as `retired` for a restore of an older backup, and the restart job deletes its canary row before it restarts Studio. + Leave the kid in `retire`. + +### Generated secrets + +The bootstrap job generates every secret of the installation once, into Secret `libredb-secrets`, and never replaces one it holds, so a rotation deletes the value there and upgrades; the upgrade generates a new value and every consumer follows in the same run: the Keycloak client, the PostgreSQL role, the apps' Secrets, and the restart job restarts what reads them. + +```sh +kubectl -n libredb patch secret libredb-secrets --type json -p '[{"op":"remove","path":"/data/"}]' +``` + +| Name | Effect of a new value | +| --- | --- | +| `studio-jwt-secret` | Studio's session cookies: every Studio user is signed out, and signs in again without a password while the Keycloak session lives | +| `platform-auth-secret` | Platform's session cookies: give the previous value as below, so the sessions survive | +| `keycloak-client-libredb-studio`, `keycloak-client-libredb-platform` | The clients' secrets: sign-in fails only between the keycloak-configure job and the app's restart | +| `postgres-owner-password`, `postgres-platform-password`, `postgres-studio-password`, `postgres-keycloak-password` | The roles' passwords, which the provisioning job sets; open connections keep working, a new one with the old password fails until its app restarted | + +Platform keeps its sessions through a rotation of `platform-auth-secret` when the previous value stays for one session lifetime, 8 days: + +```sh +(umask 077 && kubectl -n libredb get secret libredb-secrets -o 'jsonpath={.data.platform-auth-secret}' | base64 -d > auth-previous) +kubectl -n libredb create secret generic libredb-auth-previous --from-file=auth-secret-previous=auth-previous && rm auth-previous +kubectl -n libredb patch secret libredb-secrets --type json -p '[{"op":"remove","path":"/data/platform-auth-secret"}]' +``` + +Upgrade with `platform.authSecretPrevious.existingSecret: libredb-auth-previous`; after 8 days, upgrade without it and delete the Secret. +To end every Platform session at once, skip the previous value. + +`postgres-superuser-password` (the managed server reads it only when it creates its data directory), `postgres-backup-password` and `studio-storage-encryption-key` (Studio's stored data is encrypted with it) are not rotated this way. + +## Differences from a Docker install + +- The ingress controller replaces the installer's Caddy edge for Platform and Studio; a managed Keycloak keeps the installer's edge as a sidecar, so only `/realms//` and `/resources/` reach it. +- The managed Keycloak keeps its bootstrap admin client in realm `master`: Keycloak would recreate it from its Deployment's environment on the next start. Its secret is in Secrets `libredb-secrets`, `libredb-keycloak-env` and `libredb-keycloak-config` and in Keycloak's environment; Keycloak's port 8080, which serves the Admin API in-cluster over plain HTTP, accepts only this release's pods while `networkPolicy.enabled` is true (the default) and the cluster's CNI enforces network policies. +- There is no renewal timer: any `helm upgrade` re-issues a leaf of the internal CA that has fewer than 30 days left and restarts PostgreSQL for its own; the ingress controller picks up a renewed ingress certificate by itself. The leaves are valid for 825 days, so run an upgrade at least once in that time. +- Secrets and the seal key are rotated with values and upgrades, as [Rotate secrets and the seal key](#rotate-secrets-and-the-seal-key) describes, not with the installer's `apply`. +- This release runs one replica of each component. + +## Security notes + +- Every pod runs under the restricted Pod Security Standard, so the namespace may enforce it (`pod-security.kubernetes.io/enforce=restricted`); the managed Keycloak's edge adds only the capability `NET_BIND_SERVICE`, which its binary carries. +- The bootstrap hook's ServiceAccount may create Secrets and ConfigMaps in the namespace, which Kubernetes cannot limit by name, and read and replace only the installation's own objects; it exists only while the hook runs, and a failed hook leaves it until the next run replaces it. +- The containers have memory limits and no CPU limit, so a busy pod is never throttled; set `resources.limits.cpu` of each component where a policy requires one. +- `networkPolicy.enabled` guards PostgreSQL and Keycloak's Admin API; Platform and Studio take traffic from the ingress controller, whose namespace differs per cluster, so a policy for them is yours to add. +- `network.httpsProxy` must hold no credentials: it is part of the installation document, a ConfigMap. +- A license given inline lands in Helm's release Secrets; with GitOps, keep it in a Secret of your own and name it in `platform.license.existingSecret`. + +## Argo CD + +The chart works as an Argo CD application from a Git checkout or a packaged chart: + +- Argo CD runs the bootstrap job as a PreSync hook, the provisioning, keycloak-configure and migrate jobs as Sync hooks, which it replaces on every sync, and the restart job as a PostSync hook, so each sync converges as a `helm upgrade` does. +- The objects the jobs write carry the label `install.libredb.org/installation`, never a tracking label, and the annotations `argocd.argoproj.io/sync-options: Prune=false,Delete=false` and `argocd.argoproj.io/compare-options: IgnoreExtraneous`, so a prune never deletes the seal keys. +- Give the license as `platform.license.existingSecret`. +- Argo CD counts an Ingress healthy only once the ingress controller publishes an address on it, as a cloud load balancer's controller does; a controller behind a NodePort needs its published address set, such as Traefik's `providers.kubernetesIngress.ingressEndpoint.ip`, or the app stays Progressing. + +## OpenShift + +The chart installs on OpenShift 4 with the default `restricted-v2` SCC and namespace-scoped rights only, as in a Red Hat Developer Sandbox: + +- With `global.compatibility.openshift.adaptSecurityContext: auto`, the default, the chart sees `security.openshift.io/v1` and leaves out the pods' fixed `runAsUser`, `runAsGroup` and `fsGroup`, so the SCC assigns them from the namespace's range; the Studio chart reads the same value. `force` does the same where Helm cannot ask the API server, such as `helm template` for GitOps, and `disabled` keeps the fixed ids, for an SCC that allows them. +- OpenShift turns each Ingress into a Route that terminates TLS at the router with the Ingress's Secret, so `tls.mode: generated` works unchanged; the hosts are names under the cluster's apps domain, or names your DNS points at the router. +- The project template may add NetworkPolicies of its own, such as an `allow-same-namespace` policy, and policies add up: there, every pod of the namespace reaches PostgreSQL and Keycloak's Admin API, so give the installation a namespace of its own. + +## An install without network access + +`helm dependency build` fetches the pinned Studio chart from `https://libredb.org/libredb-studio/`. +For a site without that access, run `helm package --dependency-update charts/libredb-platform` on a machine that has it and install the archive, which holds the dependency, so Helm on the site needs no repository. +The images come from the registries in `values.yaml`; copy the five of them into your mirror with their digests, for example with `docker buildx imagetools create -t /: ` or `skopeo copy --all`, and point the chart at it, keeping the tags, which pin each image by digest: + +```sh +helm install libredb libredb-platform-.tgz -n libredb -f values.yaml --set-file platform.license.value=acme.license \ + --set platform.image.repository=/libredb/libredb-platform \ + --set postgresql.managed.image.repository=/library/postgres \ + --set keycloak.managed.image.repository=/keycloak/keycloak \ + --set keycloak.managed.edge.image.repository=/library/caddy \ + --set studio.image.repository=/libredb/libredb-studio +``` + +## End-to-end test + +[`charts/libredb-platform/tests/e2e/run-e2e.sh`](../../charts/libredb-platform/tests/e2e/run-e2e.sh) installs the chart on a local kind cluster into a namespace that enforces the restricted Pod Security Standard, checks the endpoints, the network policies, the license and Studio's seal canary, runs the browser suite of [`installer/e2e/kubernetes`](../../installer/e2e/kubernetes), upgrades, uninstalls and installs again; it needs a license for `platform.libredb.test`. +[`run-external.sh`](../../charts/libredb-platform/tests/e2e/run-external.sh) then installs against a PostgreSQL and a Keycloak of a simulated customer, with the customer's TLS Secrets and a license Secret; it needs a license for `platform-ext.libredb.test`. +On the installation run-e2e.sh leaves running (`LIBREDB_E2E_KEEP=1`), [`run-lifecycle.sh`](../../charts/libredb-platform/tests/e2e/run-lifecycle.sh) renews leaves that expire in 10 days and walks a license through expiring, expired and active, [`run-upgrade.sh`](../../charts/libredb-platform/tests/e2e/run-upgrade.sh) upgrades to the chart of a later release, with a failed `--atomic` attempt first, and checks that going back is refused, and [`run-argocd.sh`](../../charts/libredb-platform/tests/e2e/run-argocd.sh) installs Argo CD with label tracking and syncs the packaged chart with automated prune. +`LIBREDB_E2E_EXISTING_CLUSTER=1` runs run-e2e.sh against the cluster of `KUBECONFIG`, such as from a container on kind's network. +[`run-restore.sh`](../../charts/libredb-platform/tests/e2e/run-restore.sh) takes the backup set above, deletes the namespace with its volumes and restores the set, then signs the first admin in with the password of the backup and has a user read a table through a credential sealed before it; [`run-offline.sh`](../../charts/libredb-platform/tests/e2e/run-offline.sh) installs the packaged archive with no Helm repository and no way out, every image from a mirror. +[`run-rotation.sh`](../../charts/libredb-platform/tests/e2e/run-rotation.sh) walks [Rotate secrets and the seal key](#rotate-secrets-and-the-seal-key) with a granted user querying throughout; [`run-resilience.sh`](../../charts/libredb-platform/tests/e2e/run-resilience.sh) kills PostgreSQL, then Platform, Studio and Keycloak at once, then restarts the node, and checks after each that the endpoints come back on their own and no data was lost; [`run-soak.sh`](../../charts/libredb-platform/tests/e2e/run-soak.sh) leaves one installation running for hours and records availability, memory, restarts and growth. +[`run-openshift.sh`](../../charts/libredb-platform/tests/e2e/run-openshift.sh) runs the same checks and the browser suite in the namespace of the current `oc login`, with the hosts `ldb--.`, and checks that `restricted-v2` admits every pod with a uid from the namespace's range. diff --git a/docs/deploy/platform-runtime.md b/docs/deploy/platform-runtime.md index 458af7f..c99292c 100644 --- a/docs/deploy/platform-runtime.md +++ b/docs/deploy/platform-runtime.md @@ -68,7 +68,7 @@ A self-hosted install leaves these variables unset: ## Migrate container -The `migrate` command runs as a one-shot container (a Docker Compose service that exits, or a Kubernetes Job once Kubernetes support arrives in the next release) before every start of a new image. +The `migrate` command runs as a one-shot container (a Docker Compose service that exits, or a Kubernetes Job of the Helm chart, [install-kubernetes.md](install-kubernetes.md)) before every start of a new image. It receives exactly these variables, and the serve container never receives `MIGRATE_DATABASE_URL`: `serve` refuses to start when it is set. | Variable | Used by | Value | diff --git a/installer/README.md b/installer/README.md index 1f918e7..8d29e72 100644 --- a/installer/README.md +++ b/installer/README.md @@ -67,7 +67,7 @@ The Docker Engine minimum and that refused range keep the ports published to `12 The refused range lives in the preflight module instead of the release manifest, because a minimum cannot exclude it and still admit 28.0.x and 28.1.x, and its message names the advisory GHSA-x4rx-4gw3-53p4 and asks for 28.3.3 or newer, which fixes it. The [Docker install guide](../docs/deploy/install-docker.md) lists every prerequisite of the host. Run every command below as root. -This build applies only `mode: docker`: `preflight`, `plan`, `apply`, `verify` and `db-script` refuse any other mode with exit 2, and Kubernetes mode follows in the next release. +This build applies only `mode: docker`: `preflight`, `plan`, `apply`, `verify` and `db-script` refuse any other mode with exit 2. On Kubernetes, the Helm chart `charts/libredb-platform` runs the installer's Kubernetes-mode logic in its jobs (`src/kubernetes/`, bundled by `bun run build:helm-jobs`), as [install-kubernetes.md](../docs/deploy/install-kubernetes.md) describes. This build applies every `tls.mode` (`generated`, `provided` and `external`) and `postgres.mode` `managed` or `external`. Keycloak is either managed, or the customer's own Keycloak with a realm the installer reuses (see External Keycloak below). @@ -150,14 +150,30 @@ The installer never renews a provided certificate: replace the files and run `ap `tls.mode: external` puts the edge behind the customer's TLS proxy. The edge serves plain HTTP on `tls.external.httpPort` (default 8080), one site per public hostname, and the upstream proxy terminates TLS on 443 of each public hostname and forwards to that port with `X-Forwarded-Proto` and `X-Forwarded-Host`. +The proxy must accept response headers of 16 KB: Platform's sign-in callback sets its session cookie in chunks, which hold the Keycloak ID and refresh tokens, beside the state and PKCE cookies it clears, more than the 4 KB that nginx buffers by default on x86_64, so an nginx with its defaults answers that callback 502 (`upstream sent too big header`) while every check of `verify` passes. +With nginx, such as the proxy of CapRover, set `proxy_buffer_size 16k;` and `proxy_buffers 8 16k;` in the location that forwards to the edge; Traefik, as Dokploy runs it, needs no setting. `tls.external.trustedProxies` is required and has no default: it lists the upstream proxy's addresses as the edge sees them, as CIDR ranges such as `192.0.2.10/32`, and the edge passes `X-Forwarded-For`, `X-Forwarded-Proto` and `X-Forwarded-Host` on only from them, so `validate` refuses `tls.mode: external` without it with exit 2. The edge sees a proxy on another host at that host's own address when no NAT lies between them, and a proxy container on the network `libredb_edge` at its address there, while a proxy on the Docker host that connects through `127.0.0.1` arrives from the gateway of `libredb_edge`, the address that containers of other networks arrive from as well when they connect to the published port. +Docker picks that gateway when the first `apply` creates the network, such as `172.19.0.1` on a host whose bridge networks hold `172.17.0.0/16` and `172.18.0.0/16`, as the default bridge and the `docker_gwbridge` of a Swarm such as Dokploy's or CapRover's do; `docker network inspect libredb_edge -f '{{(index .IPAM.Config 0).Gateway}}'` prints it, and when it differs from `trustedProxies`, `verify` fails `studio-oidc-chain` with a `redirect_uri` of `http://`, so correct the value and run `apply` again. `private_ranges` is accepted when written out, but it trusts every client on a private network. The edge's HTTP port must be reachable only from the upstream proxy, because a client that reaches it directly bypasses the proxy's TLS: restrict it with a host firewall rule, which with Docker's iptables backend belongs in the `DOCKER-USER` chain, because Docker forwards the traffic of a published port past the `INPUT` chain. +Match the host's external interface in that rule, such as `iptables -I DOCKER-USER -i eth0 -p tcp -m conntrack --ctorigdstport 8080 -j DROP` for a proxy on the Docker host: Docker passes the traffic between containers through `DOCKER-USER` as well, so a rule on the port alone also drops Platform's and Studio's calls to Keycloak on its port 8080, and `verify` then fails `platform-admin-api`, `platform-signin-redirect`, `studio-oidc-chain` and `oidc-discovery-in-network`. The edge then holds no hostname alias, so the containers reach the public issuer and app URLs through the upstream proxy: every public hostname must resolve from inside the network `libredb_edge` to the proxy, and the proxy must present a certificate for it that validates against `tls.caBundleFile`, which is required. `preflight` checks both from a one-shot container of the Platform image on `libredb_edge`, or on Docker's default bridge before the first `apply` has created that network, and refuses with exit 3 naming each host that does not resolve or present a valid chain. `verify`'s `edge-tls` then connects to each public hostname on 443 by its name, through the upstream proxy. +#### Beside Dokploy or CapRover + +Dokploy and CapRover hold ports 80 and 443 of their host, where `preflight` refuses the default `tls.mode: generated` with exit 3 naming their proxy container, so LibreDB installs on such a host in `tls.mode: external` behind their proxy: + +- Dokploy: a file under `/etc/dokploy/traefik/dynamic` with an HTTP router for the three hostnames on entry point `websecure`, `tls: {}`, the site's certificate in a `tls.certificates` entry, and a service to `http://:8080`. +- CapRover: an app with the three hostnames as custom domains whose custom nginx configuration listens on 443 with the site's certificate, which CapRover's own HTTPS gives only from Let's Encrypt, proxies to `http://:8080` and sets `proxy_buffer_size 16k;` and `proxy_buffers 8 16k;`. +- `trustedProxies` holds the gateway of `libredb_edge`, which both proxies arrive from. + +Both clean up Docker on their own, which removes the containers of a LibreDB service that is stopped at that moment, such as Studio after the kill switch of [break-glass.md](docs/runbooks/break-glass.md) or the apps during a [restore](docs/runbooks/restore.md): Dokploy's daily Docker cleanup, on by default, runs `docker container prune`, `docker image prune --all` and `docker system prune --all`, and CapRover runs `docker container prune` after every deploy of one of its apps. +`docker compose up -d` and `apply` create such a container again, with its volume. +Dokploy's cleanup of unused volumes (`docker volume prune --all`), which its daily cleanup leaves out, also deletes the volume of a removed container, such as `libredb_platform-uploads` with every support attachment, so never run it while a LibreDB service is stopped. + The managed PostgreSQL leaf comes from the internal CA in every TLS mode. `tls export-ca [-o [--force]]` writes the internal CA certificate, without its key, for the trust stores of database clients and, in `tls.mode: generated`, of browsers: diff --git a/installer/docs/runbooks/break-glass.md b/installer/docs/runbooks/break-glass.md index baa4898..1871269 100644 --- a/installer/docs/runbooks/break-glass.md +++ b/installer/docs/runbooks/break-glass.md @@ -59,9 +59,11 @@ Studio's public URL answers 502 from the edge while it is stopped. Start it again when the cause is dealt with; a full `apply` starts it as well: ```sh -docker compose -p libredb -f /opt/libredb/compose.yaml start studio +docker compose -p libredb -f /opt/libredb/compose.yaml up -d studio ``` +`up -d` also creates the container again when something removed the stopped one, as `docker container prune` does, which a PaaS on the same host may run on its own, such as Dokploy's daily Docker cleanup or CapRover after every deploy; `start` then starts nothing and still exits 0. + ## PostgreSQL is down Studio keeps serving from its last governance snapshot until that snapshot is older than `GOVERNANCE_MAX_STALENESS_MS` (120000 ms by default), and then stops all data access by design, because it can no longer see revocations. diff --git a/installer/e2e/helpers/studio.ts b/installer/e2e/helpers/studio.ts index 351c11a..7c25c83 100644 --- a/installer/e2e/helpers/studio.ts +++ b/installer/e2e/helpers/studio.ts @@ -13,8 +13,8 @@ import { type PasswordUser, } from "./platform"; -/** Studio's URL in test/fixtures/configs/docker.yaml. */ -export const STUDIO_URL = "https://studio.libredb.test"; +/** Studio's URL in test/fixtures/configs/docker.yaml; LIBREDB_E2E_STUDIO_URL points the Helm chart's suite elsewhere. */ +export const STUDIO_URL = process.env.LIBREDB_E2E_STUDIO_URL ?? "https://studio.libredb.test"; /** The Studio container of the Compose project libredb. */ export const STUDIO_CONTAINER = "libredb-studio-1"; diff --git a/installer/e2e/kubernetes/helm-install.k8s.ts b/installer/e2e/kubernetes/helm-install.k8s.ts new file mode 100644 index 0000000..00c2bbb --- /dev/null +++ b/installer/e2e/kubernetes/helm-install.k8s.ts @@ -0,0 +1,217 @@ +// The browser suite of a Helm install: on the installation run-e2e.sh made on kind, the first admin signs in for the +// first time (new password and TOTP), sees the license, connects a target PostgreSQL and grants a Keycloak user read +// access; the user reads the table in Studio and cannot write to it; the grant's revocation reaches Studio within +// seconds; and Platform's audit log shows the user's statement. Platform and Studio never call each other, so every +// effect crossed the bridge schema of the shared PostgreSQL. The UI steps are the installer's own browser helpers +// (installer/e2e/helpers), so a Helm install is held to what a Docker install is held to. Each step saves a +// screenshot under LIBREDB_E2E_SCREENSHOTS. +import { mkdirSync } from "node:fs"; +import { join } from "node:path"; +import { expect, test, type BrowserContext, type Page } from "@playwright/test"; +import { + createConnection, + deleteConnection, + FIRST_ADMIN, + firstLoginToPlatform, + grantAccess, + grantRemoved, + openConnection, + revokeGrant, + signedInFirstAdmin, + signInFirstAdminToPlatform, + type FirstLogin, +} from "../helpers/platform"; +import { openStudioConsole, signInToStudioAs, STUDIO_URL, studioRequest, type StudioAnswer } from "../helpers/studio"; +import { + createKeycloakUser, + deleteKeycloakUser, + kubectl, + PLATFORM_URL, + psqlAsSuperuser, + removeTargetDatabase, + secretValue, + startTargetDatabase, + stopKeycloakForward, + targetRowCount, + type KeycloakUser, + type TargetDatabase, +} from "./k8s"; + +test.describe.configure({ mode: "serial" }); + +const SHOTS = process.env.LIBREDB_E2E_SCREENSHOTS ?? join("test-results", "helm-screenshots"); +const REVOCATION_LIMIT_MS = 15_000; + +const firstAdmin: FirstLogin = { + username: FIRST_ADMIN, + temporaryPassword: secretValue("libredb-keycloak-config", "first-admin-password"), + newPassword: `K8s-${Date.now()}-Admin-Pass!`, + email: "admin@e2e.libredb.test", + firstName: "Helm", + lastName: "Admin", +}; + +let target: TargetDatabase; +let user: KeycloakUser; +let adminContext: BrowserContext; +let userContext: BrowserContext; +let platform: Page; +let studio: Page; +let connection: { id: string; name: string } | undefined; + +async function shot(page: Page, name: string): Promise { + mkdirSync(SHOTS, { recursive: true }); + await page.screenshot({ path: join(SHOTS, `${name}.png`), fullPage: true }); +} + +function studioId(): string { + if (connection === undefined) throw new Error("no connection yet"); + return `seed:${connection.id}`; +} + +function query(page: Page, sql: string): Promise { + return studioRequest(page, "/api/db/query", { body: { connectionId: studioId(), sql } }); +} + +async function managedIds(page: Page): Promise { + const answer = await studioRequest(page, "/api/connections/managed"); + expect(answer.status, JSON.stringify(answer.body)).toBe(200); + return (answer.body as { connections: { id: string }[] }).connections.map((managed) => managed.id); +} + +test.beforeAll(async ({ browser }) => { + const image = kubectl(["get", "statefulset", "libredb-postgres", "-o", "jsonpath={.spec.template.spec.containers[0].image}"]); + target = startTargetDatabase(image); + user = await createKeycloakUser(`analyst-${Date.now().toString(36)}`, "user"); + adminContext = await browser.newContext(); + userContext = await browser.newContext(); + platform = await adminContext.newPage(); + studio = await userContext.newPage(); +}); + +test.afterAll(async () => { + if (connection !== undefined) { + try { + // The admin's session from the first test is still open. + await deleteConnection(platform, connection); + } catch (error) { + console.log(`cleanup: the connection could not be deleted: ${String(error)}`); + } + } + if (user !== undefined) await deleteKeycloakUser(user.id); + removeTargetDatabase(); + stopKeycloakForward(); + await adminContext?.close(); + await userContext?.close(); +}); + +test("the first admin signs in to Platform for the first time with a new password and TOTP", async () => { + // A rerun against the same installation signs in with the password and TOTP device the first run set. + if (signedInFirstAdmin() === undefined) { + const { completed } = await firstLoginToPlatform(platform, firstAdmin); + expect(completed).toContain("password"); + expect(completed).toContain("totp"); + } else { + await signInFirstAdminToPlatform(platform, firstAdmin); + } + await expect(platform).toHaveURL((url) => url.origin === PLATFORM_URL && url.pathname !== "/login"); + await platform.waitForLoadState("networkidle"); + await shot(platform, "01-platform-after-first-login"); + // A callback that fails at the ingress, such as a 502 for its response headers, also leaves /login behind, so the + // session itself is the proof, asked from the page as the Docker suite asks it. + const session = await platform.evaluate( + async () => (await fetch("/api/auth/session")).json() as Promise<{ user?: { email?: string } } | null>, + ); + expect(session?.user?.email).toBe(firstAdmin.email); +}); + +test("the license page shows the license the chart installed as active", async () => { + await platform.goto("/admin/license"); + // The license id the script read from the license file it installed. + const licenseId = process.env.LIBREDB_E2E_LICENSE_ID; + if (licenseId !== undefined && licenseId !== "") await expect(platform.locator("body")).toContainText(licenseId); + await expect(platform.locator("body")).toContainText(/active/i); + await shot(platform, "02-platform-license"); +}); + +test("the admin connects the target PostgreSQL, sealing its credentials to Studio's key", async () => { + const name = `orders-${Date.now().toString(36)}`; + await createConnection(platform, { + name, + host: target.host, + port: target.port, + database: target.database, + read: target.read, + write: target.write, + }); + const id = await openConnection(platform, name); + connection = { id, name }; + // Platform stores only lsb1 envelopes: neither password is in its tables. + const stored = psqlAsSuperuser("libredb", `SELECT "readSecret" || ' ' || coalesce("writeSecret", '') FROM platform.database_connections WHERE id = '${id}'`); + expect(stored).toMatch(/^lsb1\./); + expect(stored).not.toContain(target.read.password); + expect(stored).not.toContain(target.write.password); + await shot(platform, "03-platform-connection"); +}); + +test("the admin grants a Keycloak user read access", async () => { + await grantAccess(platform, user.username, "read"); + await shot(platform, "04-platform-grant-read"); +}); + +test("the user signs in to Studio, sees the connection and reads the table, but cannot write", async () => { + await signInToStudioAs(studio, user); + await openStudioConsole(studio); + await expect.poll(() => managedIds(studio), { timeout: 30_000 }).toContain(studioId()); + const read = await query(studio, `SELECT item, qty FROM ${target.table} ORDER BY id`); + expect(read.status, JSON.stringify(read.body)).toBe(200); + expect(JSON.stringify(read.body)).toContain("keyboard"); + const before = targetRowCount(); + const write = await query(studio, `INSERT INTO ${target.table} (item, qty) VALUES ('blocked', 1)`); + expect(write.status, JSON.stringify(write.body)).not.toBe(200); + expect(targetRowCount()).toBe(before); + await studio.goto(STUDIO_URL); + await studio.waitForLoadState("networkidle"); + await expect(studio.locator("body")).toContainText(connection?.name ?? ""); + await shot(studio, "05-studio-user-sees-connection"); + // The same read through Studio's editor, as the user runs it. + const editor = studio.locator(".monaco-editor").first(); + await editor.click(); + await studio.keyboard.type(`SELECT item, qty FROM ${target.table} ORDER BY id`); + await studio.getByRole("button", { name: "RUN", exact: true }).click(); + await expect(studio.locator("body")).toContainText("monitor", { timeout: 30_000 }); + await shot(studio, "05b-studio-user-query-result"); +}); + +test("revoking the grant in Platform takes the connection away from Studio within seconds", async () => { + await openConnection(platform, connection?.name ?? ""); + const { answeredAt } = await revokeGrant(platform, user.username); + await grantRemoved(platform, user.username); + await openStudioConsole(studio); + let latency = -1; + for (;;) { + const answer = await query(studio, "SELECT 1"); + if (answer.status === 404) { + latency = Date.now() - answeredAt; + break; + } + if (Date.now() - answeredAt > REVOCATION_LIMIT_MS) throw new Error(`Studio still answered ${answer.status} after the revocation`); + await new Promise((resolve) => setTimeout(resolve, 100)); + } + console.log(`revocation latency: ${latency} ms`); + test.info().annotations.push({ type: "revocation latency", description: `${latency} ms` }); + await shot(platform, "06-platform-grant-revoked"); +}); + +test("Platform's audit log shows the user's statement", async () => { + await expect + .poll( + async () => { + await platform.goto("/audit"); + return platform.locator("body").innerText(); + }, + { timeout: 60_000 }, + ) + .toContain(user.username); + await shot(platform, "07-platform-audit-log"); +}); diff --git a/installer/e2e/kubernetes/k8s.ts b/installer/e2e/kubernetes/k8s.ts new file mode 100644 index 0000000..6683219 --- /dev/null +++ b/installer/e2e/kubernetes/k8s.ts @@ -0,0 +1,232 @@ +// The Kubernetes side of the browser specs: kubectl against the kind cluster, the target database as a pod in the +// installation's namespace, and Keycloak users created through the Admin API with the managed Keycloak's bootstrap +// admin client, which a port-forward to Service keycloak reaches. The client secret is read from Secret +// libredb-keycloak-config and stays in this process. +import { execFileSync, spawn, type ChildProcess } from "node:child_process"; +import { randomBytes } from "node:crypto"; + +export const NAMESPACE = process.env.LIBREDB_E2E_NAMESPACE ?? "libredb"; +const KUBECTL = process.env.KUBECTL ?? "kubectl"; +const REALM = "libredb"; +export const PLATFORM_URL = process.env.LIBREDB_E2E_PLATFORM_URL ?? "https://platform.libredb.test"; +// LIBREDB_E2E_OPENSHIFT=1: the namespace's SCC assigns the pods' uids, and rejects a fixed one outside its range. +const FIXED_UIDS = process.env.LIBREDB_E2E_OPENSHIFT === "1" ? {} : { runAsUser: 999, runAsGroup: 999, fsGroup: 999 }; + +export function kubectl(args: string[], input?: string): string { + return execFileSync(KUBECTL, ["-n", NAMESPACE, ...args], { + encoding: "utf8", + input, + stdio: ["pipe", "pipe", "pipe"], + }).trim(); +} + +export function secretValue(name: string, key: string): string { + const encoded = kubectl(["get", "secret", name, "-o", `jsonpath={.data.${key.replaceAll(".", "\\.")}}`]); + return Buffer.from(encoded, "base64").toString("utf8"); +} + +/** psql as the managed server's superuser in the pod libredb-postgres-0, with unaligned tuples-only output. */ +export function psqlAsSuperuser(database: string, sql: string): string { + return kubectl([ + "exec", + "libredb-postgres-0", + "--", + "sh", + "-c", + `PGPASSWORD="$POSTGRES_PASSWORD" psql -X -q -A -t -v ON_ERROR_STOP=1 -h 127.0.0.1 -U postgres -d ${database} -c "$0"`, + sql, + ]); +} + +function secret(): string { + return randomBytes(18).toString("base64url"); +} + +export interface Credential { + user: string; + password: string; +} + +export interface TargetDatabase { + host: string; + port: number; + database: string; + table: string; + read: Credential; + write: Credential; +} + +const TARGET = "e2e-target"; +// LIBREDB_E2E_TARGET_NAMESPACE keeps the target database out of the installation's namespace, as a customer's own +// database is, so a test can delete that namespace and restore the installation from a backup. +const TARGET_NAMESPACE = process.env.LIBREDB_E2E_TARGET_NAMESPACE ?? NAMESPACE; + +/** kubectl in the target database's namespace: a later -n wins over kubectl()'s. */ +function targetKubectl(args: string[], input?: string): string { + return kubectl(["-n", TARGET_NAMESPACE, ...args], input); +} + +/** Starts a PostgreSQL pod and Service e2e-target with a read role and a write role on one table. */ +export function startTargetDatabase(image: string): TargetDatabase { + removeTargetDatabase(); + const superuser = secret(); + targetKubectl( + ["apply", "-f", "-"], + JSON.stringify({ + apiVersion: "v1", + kind: "List", + items: [ + { + apiVersion: "v1", + kind: "Pod", + metadata: { name: TARGET, labels: { app: TARGET } }, + // Restricted, as the installation's namespace enforces. + spec: { + securityContext: { runAsNonRoot: true, ...FIXED_UIDS, seccompProfile: { type: "RuntimeDefault" } }, + containers: [ + { + name: "postgres", + image, + env: [ + { name: "POSTGRES_PASSWORD", value: superuser }, + { name: "POSTGRES_DB", value: "interop" }, + ], + securityContext: { allowPrivilegeEscalation: false, capabilities: { drop: ["ALL"] } }, + readinessProbe: { exec: { command: ["pg_isready", "-h", "127.0.0.1", "-U", "postgres"] }, periodSeconds: 2 }, + volumeMounts: [ + { name: "data", mountPath: "/var/lib/postgresql" }, + { name: "run", mountPath: "/var/run/postgresql" }, + ], + }, + ], + volumes: [ + { name: "data", emptyDir: {} }, + { name: "run", emptyDir: {} }, + ], + }, + }, + { + apiVersion: "v1", + kind: "Service", + metadata: { name: TARGET }, + spec: { selector: { app: TARGET }, ports: [{ port: 5432 }] }, + }, + ], + }), + ); + targetKubectl(["wait", "--for=condition=Ready", `pod/${TARGET}`, "--timeout=180s"]); + const read = { user: "e2e_reader", password: secret() }; + const write = { user: "e2e_writer", password: secret() }; + const sql = [ + "CREATE SCHEMA shop;", + "CREATE TABLE shop.orders (id serial PRIMARY KEY, item text NOT NULL, qty int NOT NULL);", + "INSERT INTO shop.orders (item, qty) VALUES ('keyboard', 2), ('monitor', 1), ('cable', 5);", + `CREATE ROLE ${read.user} LOGIN PASSWORD '${read.password}';`, + `CREATE ROLE ${write.user} LOGIN PASSWORD '${write.password}';`, + `GRANT USAGE ON SCHEMA shop TO ${read.user}, ${write.user};`, + `GRANT SELECT ON shop.orders TO ${read.user};`, + `GRANT SELECT, INSERT, UPDATE, DELETE ON shop.orders TO ${write.user};`, + `GRANT USAGE ON SEQUENCE shop.orders_id_seq TO ${write.user};`, + ].join("\n"); + targetKubectl(["exec", "-i", TARGET, "--", "psql", "-X", "-q", "-v", "ON_ERROR_STOP=1", "-U", "postgres", "-d", "interop"], sql); + return { host: `${TARGET}.${TARGET_NAMESPACE}.svc.cluster.local`, port: 5432, database: "interop", table: "shop.orders", read, write }; +} + +export function removeTargetDatabase(): void { + try { + targetKubectl(["delete", "pod,service", TARGET, "--ignore-not-found", "--wait=true"]); + } catch { + // Nothing to remove. + } +} + +export function targetRowCount(): number { + return Number(targetKubectl(["exec", TARGET, "--", "psql", "-X", "-A", "-t", "-U", "postgres", "-d", "interop", "-c", "SELECT count(*) FROM shop.orders"])); +} + +let forward: ChildProcess | undefined; +// Not 18080, the admin port of a Docker install, which a host may already use. +const FORWARD_PORT = Number(process.env.LIBREDB_E2E_KEYCLOAK_PORT ?? "28080"); + +/** A port-forward to Service keycloak on 127.0.0.1, which reaches the Admin API of a managed Keycloak. */ +async function keycloakAdminUrl(): Promise { + if (forward === undefined) { + forward = spawn(KUBECTL, ["-n", NAMESPACE, "port-forward", "svc/keycloak", `${FORWARD_PORT}:8080`], { stdio: "ignore" }); + for (let attempt = 0; attempt < 50; attempt += 1) { + try { + await fetch(`http://127.0.0.1:${FORWARD_PORT}/realms/${REALM}`); + break; + } catch { + await new Promise((resolve) => setTimeout(resolve, 200)); + } + } + } + return `http://127.0.0.1:${FORWARD_PORT}`; +} + +export function stopKeycloakForward(): void { + forward?.kill(); + forward = undefined; +} + +async function adminToken(base: string): Promise { + const response = await fetch(`${base}/realms/master/protocol/openid-connect/token`, { + method: "POST", + headers: { "content-type": "application/x-www-form-urlencoded" }, + body: new URLSearchParams({ + grant_type: "client_credentials", + client_id: "libredb-bootstrap", + client_secret: secretValue("libredb-keycloak-config", "keycloak-bootstrap-client-secret"), + }), + }); + if (!response.ok) throw new Error(`Keycloak refused the bootstrap client: ${response.status}`); + return ((await response.json()) as { access_token: string }).access_token; +} + +export interface KeycloakUser { + id: string; + username: string; + email: string; + password: string; +} + +/** Creates a user of the realm with a password and the client role libredb:, as the user manager would. */ +export async function createKeycloakUser(username: string, role: "admin" | "user"): Promise { + const base = await keycloakAdminUrl(); + const token = await adminToken(base); + const headers = { authorization: `Bearer ${token}`, "content-type": "application/json" }; + const realm = `${base}/admin/realms/${REALM}`; + const password = `${secret()}Aa1!`; + const email = `${username}@e2e.libredb.test`; + const created = await fetch(`${realm}/users`, { + method: "POST", + headers, + body: JSON.stringify({ + username, + email, + emailVerified: true, + firstName: username, + lastName: "E2E", + enabled: true, + credentials: [{ type: "password", value: password, temporary: false }], + }), + }); + if (created.status !== 201) throw new Error(`creating ${username} answered ${created.status}: ${await created.text()}`); + const id = (created.headers.get("location") ?? "").split("/").pop() as string; + const clients = (await (await fetch(`${realm}/clients?clientId=libredb`, { headers })).json()) as { id: string }[]; + const client = clients[0] as { id: string }; + const roleRep = await (await fetch(`${realm}/clients/${client.id}/roles/${role}`, { headers })).json(); + const mapped = await fetch(`${realm}/users/${id}/role-mappings/clients/${client.id}`, { + method: "POST", + headers, + body: JSON.stringify([roleRep]), + }); + if (mapped.status !== 204) throw new Error(`granting libredb:${role} to ${username} answered ${mapped.status}`); + return { id, username, email, password }; +} + +export async function deleteKeycloakUser(id: string): Promise { + const base = await keycloakAdminUrl(); + const token = await adminToken(base); + await fetch(`${base}/admin/realms/${REALM}/users/${id}`, { method: "DELETE", headers: { authorization: `Bearer ${token}` } }); +} diff --git a/installer/e2e/kubernetes/playwright.config.ts b/installer/e2e/kubernetes/playwright.config.ts new file mode 100644 index 0000000..3acd2a4 --- /dev/null +++ b/installer/e2e/kubernetes/playwright.config.ts @@ -0,0 +1,30 @@ +// The browser specs of the Helm chart, run against an installation on a kind cluster that run-e2e.sh made with the +// hostnames *.libredb.test, which Traefik serves on 127.0.0.1:443: +// cd installer && bunx playwright test --config e2e/kubernetes/playwright.config.ts +// The browser maps every *.libredb.test name to 127.0.0.1 and trusts nothing it is not told to: the ingress serves +// leaves of the installation's internal CA, which run-e2e.sh checks with curl against the CA, so the browser accepts +// them as the installer's own browser specs do. +// On a cluster whose hosts resolve publicly, such as OpenShift's apps domain, LIBREDB_E2E_PLATFORM_URL and +// LIBREDB_E2E_STUDIO_URL name the two apps and the browser resolves them as it would anywhere. +import { defineConfig, devices } from "@playwright/test"; +import { PLATFORM_URL } from "./k8s"; + +export default defineConfig({ + testDir: ".", + // Not *.spec.ts, so the Docker suite of e2e/playwright.config.ts never picks this one up. + testMatch: "*.k8s.ts", + workers: 1, + fullyParallel: false, + retries: 0, + timeout: 4 * 60_000, + reporter: [["list"]], + outputDir: process.env.LIBREDB_E2E_OUTPUT_DIR ?? "test-results", + use: { + ...devices["Desktop Chrome"], + baseURL: PLATFORM_URL, + ignoreHTTPSErrors: true, + trace: "retain-on-failure", + screenshot: "only-on-failure", + launchOptions: { args: process.env.LIBREDB_E2E_PLATFORM_URL ? [] : ["--host-resolver-rules=MAP *.libredb.test 127.0.0.1"] }, + }, +}); diff --git a/installer/e2e/kubernetes/restore.k8s.ts b/installer/e2e/kubernetes/restore.k8s.ts new file mode 100644 index 0000000..1bdf953 --- /dev/null +++ b/installer/e2e/kubernetes/restore.k8s.ts @@ -0,0 +1,116 @@ +// The browser side of run-restore.sh and run-resilience.sh, in two phases that LIBREDB_E2E_RESTORE_PHASE picks, around +// what the script does to the installation in between: a backup, the loss of the namespace and a restore, or killed +// pods and a restarted node. seed: the first admin signs in for the first time, connects the target PostgreSQL, which +// lives in a namespace of its own, and grants a Keycloak user read access, and the user reads the table in Studio; the +// phase writes what it made to LIBREDB_E2E_RESTORE_STATE. verify: the first admin signs in with the password and TOTP +// device of the seed, Platform still holds the connection with the envelope of the seed, and the user still reads the +// table in Studio, so Studio opened the credential sealed in the seed. Without LIBREDB_E2E_RESTORE_PHASE, as in +// run-e2e.sh, the suite skips. +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { expect, test, type BrowserContext, type Page } from "@playwright/test"; +import { createConnection, FIRST_ADMIN, grantAccess, openConnection, signInFirstAdminToPlatform, type FirstLogin } from "../helpers/platform"; +import { openStudioConsole, signInToStudioAs, STUDIO_URL, studioRequest } from "../helpers/studio"; +import { createKeycloakUser, kubectl, psqlAsSuperuser, secretValue, startTargetDatabase, stopKeycloakForward, type KeycloakUser, type TargetDatabase } from "./k8s"; + +const PHASE = process.env.LIBREDB_E2E_RESTORE_PHASE; +const STATE = process.env.LIBREDB_E2E_RESTORE_STATE ?? join("test-results", "restore-state.json"); +const SHOTS = process.env.LIBREDB_E2E_SCREENSHOTS ?? join("test-results", "restore-screenshots"); + +interface RestoreState { + target: TargetDatabase; + user: KeycloakUser; + connection: { id: string; name: string }; + envelope: string; +} + +test.describe.configure({ mode: "serial" }); +test.skip(PHASE !== "seed" && PHASE !== "verify", "run by run-restore.sh and run-resilience.sh only"); + +let adminContext: BrowserContext; +let userContext: BrowserContext; +let platform: Page; +let studio: Page; + +async function shot(page: Page, name: string): Promise { + mkdirSync(SHOTS, { recursive: true }); + await page.screenshot({ path: join(SHOTS, `${name}.png`), fullPage: true }); +} + +function firstAdmin(): FirstLogin { + return { + username: FIRST_ADMIN, + temporaryPassword: secretValue("libredb-keycloak-config", "first-admin-password"), + newPassword: `Restore-${Date.now()}-Admin-Pass!`, + email: "admin@e2e.libredb.test", + firstName: "Restore", + lastName: "Admin", + }; +} + +function envelopeOf(id: string): string { + return psqlAsSuperuser("libredb", `SELECT "readSecret" FROM platform.database_connections WHERE id = '${id}'`); +} + +async function userReadsTable(state: RestoreState): Promise { + await signInToStudioAs(studio, state.user); + await openStudioConsole(studio); + const connectionId = `seed:${state.connection.id}`; + await expect + .poll(async () => { + const answer = await studioRequest(studio, "/api/connections/managed"); + return answer.status === 200 ? (answer.body as { connections: { id: string }[] }).connections.map((managed) => managed.id) : []; + }, { timeout: 60_000 }) + .toContain(connectionId); + const read = await studioRequest(studio, "/api/db/query", { + body: { connectionId, sql: `SELECT item, qty FROM ${state.target.table} ORDER BY id` }, + }); + expect(read.status, JSON.stringify(read.body)).toBe(200); + expect(JSON.stringify(read.body)).toContain("keyboard"); +} + +test.beforeAll(async ({ browser }) => { + adminContext = await browser.newContext(); + userContext = await browser.newContext(); + platform = await adminContext.newPage(); + studio = await userContext.newPage(); +}); + +test.afterAll(async () => { + stopKeycloakForward(); + await adminContext?.close(); + await userContext?.close(); +}); + +test("seed: the first admin connects the target database and grants a user, who reads the table in Studio", async () => { + test.skip(PHASE !== "seed"); + if (existsSync(STATE)) throw new Error(`${STATE} exists: the seed phase runs once per backup`); + const image = kubectl(["get", "statefulset", "libredb-postgres", "-o", "jsonpath={.spec.template.spec.containers[0].image}"]); + const target = startTargetDatabase(image); + const user = await createKeycloakUser(`restored-${Date.now().toString(36)}`, "user"); + await signInFirstAdminToPlatform(platform, firstAdmin()); + const name = `restore-${Date.now().toString(36)}`; + await createConnection(platform, { name, host: target.host, port: target.port, database: target.database, read: target.read, write: target.write }); + const id = await openConnection(platform, name); + await grantAccess(platform, user.username, "read"); + const state: RestoreState = { target, user, connection: { id, name }, envelope: envelopeOf(id) }; + expect(state.envelope).toMatch(/^lsb1\./); + await userReadsTable(state); + writeFileSync(STATE, JSON.stringify(state), { mode: 0o600 }); + await shot(platform, "seed-platform-connection"); +}); + +test("verify: the installation signs the first admin in and Studio opens the credential sealed in the seed", async () => { + test.skip(PHASE !== "verify"); + const state = JSON.parse(readFileSync(STATE, "utf8")) as RestoreState; + // The envelope is the backup's, byte for byte: the restore did not seal it again. + expect(envelopeOf(state.connection.id)).toBe(state.envelope); + await signInFirstAdminToPlatform(platform, firstAdmin()); + expect(await openConnection(platform, state.connection.name)).toBe(state.connection.id); + await shot(platform, "verify-platform-connection"); + await userReadsTable(state); + await studio.goto(STUDIO_URL); + await studio.waitForLoadState("networkidle"); + await expect(studio.locator("body")).toContainText(state.connection.name); + await shot(studio, "verify-studio-connection"); +}); diff --git a/installer/e2e/kubernetes/rotation.k8s.ts b/installer/e2e/kubernetes/rotation.k8s.ts new file mode 100644 index 0000000..8f2cc5c --- /dev/null +++ b/installer/e2e/kubernetes/rotation.k8s.ts @@ -0,0 +1,303 @@ +// The rotations of "Rotate secrets and the seal key" in docs/deploy/install-kubernetes.md, on the installation +// run-rotation.sh made on kind, each step a helm upgrade as the guide gives it. The seal key drill follows the runbook +// of a Docker install (installer/e2e/runbooks.spec.ts): a new key reaches every Studio heartbeat, Platform switches to +// it, a retirement while a credential still uses the old key is refused at once and changes nothing, the admin enters +// the credentials again, and the retirement then takes the old key out of Studio's keyring and its canary out of +// Platform's table, while the granted user keeps querying. Then Studio's JWT_SECRET, Platform's AUTH_SECRET with its +// previous value, Studio's Keycloak client secret and libredb_platform's password are rotated by deleting each from +// Secret libredb-secrets and upgrading. Without LIBREDB_E2E_ROTATION=1, as in run-e2e.sh, the suite skips. +import { execFileSync, spawnSync } from "node:child_process"; +import { expect, test, type BrowserContext, type Page } from "@playwright/test"; +import { sealKeyId } from "../../src/secrets/seal-keys"; +import { + createConnection, + FIRST_ADMIN, + grantAccess, + openConnection, + signInFirstAdminToPlatform, + type Credential, +} from "../helpers/platform"; +import { openStudioConsole, signInToStudioAs, STUDIO_URL, studioRequest, type StudioAnswer } from "../helpers/studio"; +import { + createKeycloakUser, + kubectl, + NAMESPACE, + PLATFORM_URL, + psqlAsSuperuser, + secretValue, + startTargetDatabase, + stopKeycloakForward, + type KeycloakUser, + type TargetDatabase, +} from "./k8s"; + +const ENABLED = process.env.LIBREDB_E2E_ROTATION === "1"; +const CHART = process.env.LIBREDB_E2E_CHART ?? ""; +const VALUES = process.env.LIBREDB_E2E_VALUES ?? ""; +const LICENSE = process.env.LIBREDB_E2E_LICENSE ?? ""; +const HEARTBEAT_WAIT_MS = 180_000; + +test.describe.configure({ mode: "serial" }); +test.skip(!ENABLED, "run by run-rotation.sh only"); + +let target: TargetDatabase; +let user: KeycloakUser; +let adminContext: BrowserContext; +let userContext: BrowserContext; +let platform: Page; +let studio: Page; +let connection: { id: string; name: string }; +/** The --set flags of platform.sealKeys the installation runs with, which every later upgrade repeats. */ +let sealSets: string[] = []; + +/** helm upgrade of the installation with the values of run-rotation.sh and `sets`; returns helm's exit status. */ +function upgrade(sets: string[]): { status: number; output: string } { + const args = ["upgrade", "libredb", CHART, "-n", NAMESPACE, "-f", VALUES, "--set-file", `platform.license.value=${LICENSE}`]; + for (const set of [...sealSets, ...sets]) args.push("--set", set); + const run = spawnSync("helm", [...args, "--wait", "--wait-for-jobs", "--timeout", "10m"], { encoding: "utf8" }); + return { status: run.status ?? -1, output: `${run.stdout}${run.stderr}` }; +} + +/** The pods of a selector, terminating ones included. */ +function podCount(selector: string): number { + return kubectl(["get", "pods", "-l", selector, "-o", "name"]).split("\n").filter((line) => line !== "").length; +} + +/** Whether the ingress answers `url` within 5 seconds with `status`, three times in a row. */ +function settled(url: string, status: string): boolean { + const host = new URL(url).host; + for (let attempt = 0; attempt < 3; attempt += 1) { + const run = spawnSync("curl", ["-sk", "--max-time", "5", "--resolve", `${host}:443:127.0.0.1`, "-o", "/dev/null", "-w", "%{http_code}", url], { encoding: "utf8" }); + if (run.stdout !== status) return false; + } + return true; +} + +/** + * An upgrade that succeeds, once the restarted apps run, their old pods are gone and the ingress stopped sending + * requests to them. The Studio chart gives its pod no preStop delay, so for a few seconds after a restart the ingress + * can still send a request to the pod that just stopped, which waits 30 seconds and ends in a 504. + */ +async function upgraded(sets: string[] = []): Promise { + const run = upgrade(sets); + expect(run.status, run.output).toBe(0); + for (const deploy of ["libredb-platform", "libredb-studio"]) kubectl(["rollout", "status", `deploy/${deploy}`, "--timeout=300s"]); + for (const selector of ["app.kubernetes.io/component=platform", "app.kubernetes.io/name=studio"]) { + await expect.poll(() => podCount(selector), { timeout: 180_000, intervals: [1_000] }).toBe(1); + } + await expect.poll(() => settled(`${STUDIO_URL}/login`, "307") && settled(`${PLATFORM_URL}/api/health/ready`, "200"), { timeout: 180_000, intervals: [1_000] }).toBe(true); +} + +interface Heartbeat { + kids: string[]; + canary: Record; +} + +/** The live heartbeats of bridge.studio_instance, as the runbook's query reads them. */ +function heartbeats(): Heartbeat[] { + return JSON.parse( + psqlAsSuperuser( + "libredb", + `SELECT coalesce(json_agg(json_build_object('kids', keyring_kids, 'canary', canary) ORDER BY instance_id), '[]') + FROM bridge.studio_instance WHERE last_seen_at > now() - interval '2 minutes'`, + ), + ) as Heartbeat[]; +} + +function envelopeKids(): { read: string; write: string } { + return JSON.parse( + psqlAsSuperuser( + "libredb", + `SELECT json_build_object('read', split_part(read_secret, '.', 2), 'write', split_part(write_secret, '.', 2)) + FROM bridge.connections_v1 WHERE connection_id = '${connection.id}'`, + ), + ) as { read: string; write: string }; +} + +function canaryKids(): string[] { + return JSON.parse(psqlAsSuperuser("libredb", "SELECT coalesce(json_agg(kid ORDER BY kid), '[]') FROM bridge.seal_canary_v1")) as string[]; +} + +/** The kid of BRIDGE_SEAL_PUBLIC_KEY in the running Platform pod. */ +function platformKid(): string { + return sealKeyId(kubectl(["exec", "deploy/libredb-platform", "--", "printenv", "BRIDGE_SEAL_PUBLIC_KEY"])); +} + +function query(): Promise { + return studioRequest(studio, "/api/db/query", { body: { connectionId: `seed:${connection.id}`, sql: "SELECT 1" } }); +} + +async function userStillQueries(): Promise { + await openStudioConsole(studio); + await expect.poll(async () => (await query()).status, { timeout: 90_000, intervals: [1_000] }).toBe(200); +} + +/** Removes one value from the secret store, as the guide's rotation does, so the next upgrade generates it again. */ +function deleteStored(name: string): string { + const value = secretValue("libredb-secrets", name); + expect(value, name).not.toBe(""); + kubectl(["patch", "secret", "libredb-secrets", "--type", "json", "-p", JSON.stringify([{ op: "remove", path: `/data/${name}` }])]); + return value; +} + +test.beforeAll(async ({ browser }) => { + const image = kubectl(["get", "statefulset", "libredb-postgres", "-o", "jsonpath={.spec.template.spec.containers[0].image}"]); + target = startTargetDatabase(image); + user = await createKeycloakUser(`rotation-${Date.now().toString(36)}`, "user"); + adminContext = await browser.newContext(); + userContext = await browser.newContext(); + platform = await adminContext.newPage(); + studio = await userContext.newPage(); + await signInFirstAdminToPlatform(platform, { + username: FIRST_ADMIN, + temporaryPassword: secretValue("libredb-keycloak-config", "first-admin-password"), + newPassword: `Rotation-${Date.now()}-Admin-Pass!`, + email: "admin@e2e.libredb.test", + firstName: "Rotation", + lastName: "Admin", + }); + const name = `rotation-${Date.now().toString(36)}`; + await createConnection(platform, { name, host: target.host, port: target.port, database: target.database, read: target.read, write: target.write }); + connection = { id: await openConnection(platform, name), name }; + await grantAccess(platform, user.username, "read"); + await signInToStudioAs(studio, user); + await userStillQueries(); +}); + +test.afterAll(async () => { + stopKeycloakForward(); + await adminContext?.close(); + await userContext?.close(); +}); + +test("seal key rotation: issue, switch, a refused early retirement, re-entry, retirement", async () => { + test.setTimeout(40 * 60_000); + const oldKid = envelopeKids().read; + expect(envelopeKids()).toEqual({ read: oldKid, write: oldKid }); + expect(platformKid()).toBe(oldKid); + + // 1. Add the new key to Studio. + sealSets = ["platform.sealKeys.issueNew=true"]; + await upgraded(); + let newKid = ""; + await expect + .poll( + () => { + const live = heartbeats(); + newKid = live[0]?.kids.find((kid) => kid !== oldKid) ?? ""; + return newKid !== "" && live.every((row) => row.kids.includes(oldKid) && row.kids.includes(newKid) && row.canary[oldKid] === "ok"); + }, + { timeout: HEARTBEAT_WAIT_MS, intervals: [2_000] }, + ) + .toBe(true); + expect(platformKid()).toBe(oldKid); + // The same values again issue no second key. + await upgraded(); + expect(heartbeats().every((row) => row.kids.length === 2)).toBe(true); + await userStillQueries(); + + // 2. Switch Platform to the new key; its canary for the new kid turns ok in every heartbeat. + sealSets = ["platform.sealKeys.issueNew=false", `platform.sealKeys.activeKid=${newKid}`]; + await upgraded(); + expect(platformKid()).toBe(newKid); + await expect + .poll(() => heartbeats().every((row) => row.canary[newKid] === "ok" && row.canary[oldKid] === "ok"), { + timeout: HEARTBEAT_WAIT_MS, + intervals: [2_000], + }) + .toBe(true); + await userStillQueries(); + + // Retiring before the re-entry: the bootstrap job refuses at once, naming the connection, and changes nothing. + const storeBefore = kubectl(["get", "secret", "libredb-secrets", "-o", "jsonpath={.data.seal-keys\\.json}"]); + const started = Date.now(); + const refused = upgrade([`platform.sealKeys.retire[0]=${oldKid}`]); + expect(refused.status, refused.output).not.toBe(0); + expect(Date.now() - started).toBeLessThan(5 * 60_000); + const log = kubectl(["logs", "job/libredb-bootstrap", "-c", "bootstrap"]); + expect(log).toContain(`${connection.name} (${connection.id})`); + expect(kubectl(["get", "secret", "libredb-secrets", "-o", "jsonpath={.data.seal-keys\\.json}"])).toBe(storeBefore); + expect(canaryKids()).toContain(oldKid); + await userStillQueries(); + + // 3. Platform warns about the credentials sealed to the old key; the admin enters them again. + await platform.goto(`/connections/${connection.id}`); + await expect(platform.getByRole("alert").filter({ hasText: "sealed to an earlier Studio key" })).toContainText(oldKid); + const readGroup = platform.getByRole("group", { name: "Read credential" }); + await readGroup.getByLabel("User", { exact: true }).fill(target.read.user); + await readGroup.getByLabel("Password", { exact: true }).fill(target.read.password); + await platform.getByLabel(/I confirm that the read credential can only read/).check(); + await platform.getByLabel("Write credential", { exact: true }).selectOption("set"); + const writeGroup = platform.getByRole("group", { name: "New write credential" }); + await writeGroup.getByLabel("User", { exact: true }).fill((target.write as Credential).user); + await writeGroup.getByLabel("Password", { exact: true }).fill((target.write as Credential).password); + await platform.getByRole("button", { name: "Save changes" }).click(); + await expect.poll(() => envelopeKids(), { timeout: 30_000, intervals: [500] }).toEqual({ read: newKid, write: newKid }); + await userStillQueries(); + + // 4. Retire the old key: out of every keyring, its canary row gone, and the user still queries. + sealSets = ["platform.sealKeys.issueNew=false", `platform.sealKeys.activeKid=${newKid}`, `platform.sealKeys.retire[0]=${oldKid}`]; + await upgraded(); + expect(canaryKids()).not.toContain(oldKid); + expect(canaryKids()).toContain(newKid); + await expect + .poll( + () => heartbeats().map((row) => ({ old: row.kids.includes(oldKid) || oldKid in row.canary, current: row.canary[newKid] })), + { timeout: HEARTBEAT_WAIT_MS, intervals: [2_000] }, + ) + .toEqual([{ old: false, current: "ok" }]); + const stored = JSON.parse(secretValue("libredb-secrets", "seal-keys.json")) as { kid: string; state: string }[]; + expect(stored.map((key) => [key.kid, key.state])).toEqual([ + [oldKid, "retired"], + [newKid, "live"], + ]); + expect(platformKid()).toBe(newKid); + await userStillQueries(); +}); + +test("Studio's JWT_SECRET: a new value ends every Studio session, and Keycloak's session signs the user in again", async () => { + test.setTimeout(15 * 60_000); + const old = deleteStored("studio-jwt-secret"); + await upgraded(); + expect(secretValue("libredb-studio-env", "jwt-secret")).not.toBe(old); + await openStudioConsole(studio); + await expect.poll(async () => (await query()).status, { timeout: 60_000 }).toBe(401); + // The Keycloak session is still alive, so the sign-in completes without the password form, as the runbook says. + await studio.goto(`${STUDIO_URL}/login`); + await expect(studio).toHaveURL((url) => url.origin === STUDIO_URL && url.pathname !== "/login", { timeout: 30_000 }); + await expect(studio.locator("#password")).toHaveCount(0); + await userStillQueries(); +}); + +test("Platform's AUTH_SECRET with its previous value: the admin's session survives the rotation", async () => { + test.setTimeout(15 * 60_000); + const old = deleteStored("platform-auth-secret"); + kubectl(["create", "secret", "generic", "libredb-auth-previous", `--from-literal=auth-secret-previous=${old}`]); + await upgraded(["platform.authSecretPrevious.existingSecret=libredb-auth-previous"]); + const current = kubectl(["exec", "deploy/libredb-platform", "--", "printenv", "AUTH_SECRET"]); + expect(current).not.toBe(old); + expect(kubectl(["exec", "deploy/libredb-platform", "--", "printenv", "AUTH_SECRET_PREVIOUS"])).toBe(old); + await platform.goto(`/connections/${connection.id}`); + await expect(platform).toHaveURL((url) => url.origin === PLATFORM_URL && url.pathname === `/connections/${connection.id}`); + // After the session lifetime, the previous value goes. + await upgraded(); + expect(spawnSync("kubectl", ["-n", NAMESPACE, "exec", "deploy/libredb-platform", "--", "printenv", "AUTH_SECRET_PREVIOUS"]).status).not.toBe(0); + kubectl(["delete", "secret", "libredb-auth-previous"]); +}); + +test("Studio's Keycloak client secret and libredb_platform's password: new values reach Keycloak, PostgreSQL and the apps", async () => { + test.setTimeout(15 * 60_000); + const client = deleteStored("keycloak-client-libredb-studio"); + const role = deleteStored("postgres-platform-password"); + await upgraded(); + expect(secretValue("libredb-studio-env", "oidc-client-secret")).not.toBe(client); + expect(kubectl(["exec", "deploy/libredb-platform", "--", "printenv", "DATABASE_URL"])).not.toContain(role); + // A new Studio sign-in goes through Keycloak with the new client secret. + await studio.context().clearCookies(); + await signInToStudioAs(studio, user); + await userStillQueries(); + // Platform reads its database with the new password. + await platform.goto("/audit"); + await expect(platform).toHaveURL((url) => url.pathname === "/audit"); + expect(execFileSync("curl", ["-sk", "--resolve", `${new URL(PLATFORM_URL).host}:443:127.0.0.1`, "-o", "/dev/null", "-w", "%{http_code}", `${PLATFORM_URL}/api/health/ready`], { encoding: "utf8" })).toBe("200"); +}); diff --git a/installer/e2e/runbooks.spec.ts b/installer/e2e/runbooks.spec.ts index 5ca47c0..35b8d6c 100644 --- a/installer/e2e/runbooks.spec.ts +++ b/installer/e2e/runbooks.spec.ts @@ -461,8 +461,11 @@ test("break-glass: the kill switch stops Studio, start brings it back, and statu expectSuccess(runBlock(runbook, 3, "stop studio")); expect(await ready()).toBe(502); + // A PaaS on the host, such as Dokploy or CapRover, may prune the stopped container; the start block brings Studio + // back all the same. + asRootOk("docker container prune --force --filter label=com.docker.compose.project=libredb --filter label=com.docker.compose.service=studio"); - expectSuccess(runBlock(runbook, 4, "start studio")); + expectSuccess(runBlock(runbook, 4, "up -d studio")); await expect.poll(ready, { timeout: 120_000, intervals: [2_000] }).toBe(200); const status = expectSuccess(runBlock(runbook, 5, "status")); diff --git a/installer/package.json b/installer/package.json index a40f8fa..4a7a8e6 100644 --- a/installer/package.json +++ b/installer/package.json @@ -15,7 +15,8 @@ "embed": "bun scripts/embed-release.ts", "release:check": "bun scripts/check-release-compat.ts", "release:record": "bun scripts/record-release.ts", - "build": "bun scripts/build.ts" + "build": "bun scripts/build.ts", + "build:helm-jobs": "bun scripts/build-helm-jobs.ts" }, "devDependencies": { "@playwright/test": "1.58.2", diff --git a/installer/scripts/build-helm-jobs.ts b/installer/scripts/build-helm-jobs.ts new file mode 100644 index 0000000..c50ffae --- /dev/null +++ b/installer/scripts/build-helm-jobs.ts @@ -0,0 +1,48 @@ +// Bundles the Helm chart's jobs (src/cli/kubernetes-job.ts) into one CommonJS file for the Platform image's node, +// gzipped as charts/libredb-platform/files/libredb-jobs.cjs.gz, which the chart mounts from a ConfigMap: compressed, so +// the Helm release record stays far below the 1 MiB of a Secret. The bundle is generated: +// test/unit/kubernetes/bundle.test.ts rebuilds it and fails when the committed file differs, so the chart always runs +// the installer's current code. Run `bun run build:helm-jobs` after changing anything the jobs import. +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { gunzipSync, gzipSync } from "node:zlib"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +const root = join(import.meta.dir, ".."); +export const ENTRYPOINT = join(root, "src/cli/kubernetes-job.ts"); +export const CHART_BUNDLE = join(root, "..", "charts", "libredb-platform", "files", "libredb-jobs.cjs.gz"); + +const BANNER = + "// Generated by installer/scripts/build-helm-jobs.ts from installer/src/cli/kubernetes-job.ts; do not edit.\n"; + +/** Builds the bundle and returns its content. */ +export function buildHelmJobs(): string { + const dir = mkdtempSync(join(tmpdir(), "libredb-helm-jobs-")); + try { + const outfile = join(dir, "libredb-jobs.cjs"); + const result = Bun.spawnSync( + ["bun", "build", "--target=node", "--format=cjs", "--minify", ENTRYPOINT, "--outfile", outfile], + { cwd: root, stdout: "pipe", stderr: "pipe" }, + ); + if (result.exitCode !== 0) throw new Error(`bun build failed:\n${result.stderr.toString()}`); + return BANNER + readFileSync(outfile, "utf8"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +/** The bundle as the chart ships it: gzip at level 9, whose header carries no file name and no time. */ +export function compressBundle(content: string): Buffer { + return gzipSync(Buffer.from(content, "utf8"), { level: 9 }); +} + +/** The committed bundle, decompressed. */ +export function committedBundle(): string { + return gunzipSync(readFileSync(CHART_BUNDLE)).toString("utf8"); +} + +if (import.meta.main) { + const compressed = compressBundle(buildHelmJobs()); + writeFileSync(CHART_BUNDLE, compressed); + console.log(`wrote ${CHART_BUNDLE} (${compressed.length} bytes)`); +} diff --git a/installer/src/adapters/kubernetes-api.ts b/installer/src/adapters/kubernetes-api.ts new file mode 100644 index 0000000..d0707dc --- /dev/null +++ b/installer/src/adapters/kubernetes-api.ts @@ -0,0 +1,222 @@ +// The Kubernetes API of the chart's jobs: the namespace's Secrets and ConfigMaps through the in-cluster service +// account. The job's container trusts the cluster CA through NODE_EXTRA_CA_CERTS, so plain fetch reaches the API +// server. A replace carries the resourceVersion the job read, so two jobs never overwrite each other's write, and +// keeps the labels and annotations others gave the object. Every object the jobs write is marked so that neither +// Helm nor Argo CD prunes it: it holds the installation's secrets, which must outlive the release. +import type { KubernetesObjects, ObjectData } from "../kubernetes/bootstrap"; +import type { Workload } from "../kubernetes/names"; + +/** Argo CD keeps an object with these annotations through a prune and leaves it out of the app's sync status. */ +export const KEEP_ANNOTATIONS = { + "argocd.argoproj.io/sync-options": "Prune=false,Delete=false", + "argocd.argoproj.io/compare-options": "IgnoreExtraneous", +} as const; + +/** The label by which Helm-aware tools track an object as part of a release; the jobs' objects never carry it. */ +export const TRACKING_LABEL = "app.kubernetes.io/instance"; + +/** The pod template annotation a restart sets, as `kubectl rollout restart` does with its own. */ +export const RESTARTED_AT = "install.libredb.org/restarted-at"; + +const TIMEOUT_MS = 30_000; + +/** The files every pod with a service account token reads. */ +export const SERVICE_ACCOUNT_DIR = "/var/run/secrets/kubernetes.io/serviceaccount"; + +export class KubernetesApiError extends Error { + constructor(method: string, path: string, status: number, reason: string) { + super(`Kubernetes API ${method} ${path} answered ${status}${reason === "" ? "" : `: ${reason}`}`); + this.name = "KubernetesApiError"; + } +} + +export interface KubernetesApiOptions { + /** https://: of the API server. */ + server: string; + token: string; + namespace: string; + /** The labels every object the job writes carries. */ + labels: Readonly>; + fetch?: typeof fetch; + log?: (line: string) => void; +} + +interface ObjectMeta { + name: string; + resourceVersion?: string; + labels?: Record; + annotations?: Record; +} + +function encode(data: ObjectData): Record { + return Object.fromEntries(Object.entries(data).map(([key, value]) => [key, Buffer.from(value, "utf8").toString("base64")])); +} + +function decode(data: Record | undefined): ObjectData { + return Object.fromEntries(Object.entries(data ?? {}).map(([key, value]) => [key, Buffer.from(value, "base64").toString("utf8")])); +} + +async function reasonOf(response: Response): Promise { + try { + const body = (await response.json()) as { message?: unknown }; + return typeof body.message === "string" ? body.message : ""; + } catch { + return ""; + } +} + +export class KubernetesApi implements KubernetesObjects { + readonly #options: KubernetesApiOptions; + readonly #versions = new Map(); + readonly #metadata = new Map(); + readonly #types = new Map(); + + constructor(options: KubernetesApiOptions) { + this.#options = options; + } + + /** The API of the pod's own service account, from the standard environment and files. */ + static inCluster( + env: Readonly>, + readFile: (path: string) => string | undefined, + labels: Readonly>, + log?: (line: string) => void, + ): KubernetesApi { + const host = env.KUBERNETES_SERVICE_HOST; + const port = env.KUBERNETES_SERVICE_PORT ?? "443"; + const token = readFile(`${SERVICE_ACCOUNT_DIR}/token`)?.trim(); + const namespace = readFile(`${SERVICE_ACCOUNT_DIR}/namespace`)?.trim(); + if (host === undefined || token === undefined || namespace === undefined) { + throw new Error("not running in a Kubernetes pod with a service account token: KUBERNETES_SERVICE_HOST or the token is missing"); + } + const server = host.includes(":") ? `https://[${host}]:${port}` : `https://${host}:${port}`; + return new KubernetesApi({ server, token, namespace, labels, ...(log === undefined ? {} : { log }) }); + } + + get namespace(): string { + return this.#options.namespace; + } + + async #request(method: string, path: string, body?: unknown, contentType = "application/json"): Promise { + const response = await (this.#options.fetch ?? fetch)(`${this.#options.server}${path}`, { + method, + headers: { + authorization: `Bearer ${this.#options.token}`, + accept: "application/json", + ...(body === undefined ? {} : { "content-type": contentType }), + }, + ...(body === undefined ? {} : { body: JSON.stringify(body) }), + signal: AbortSignal.timeout(TIMEOUT_MS), + }); + this.#options.log?.(`kubernetes ${method} ${path} ${response.status}`); + return response; + } + + #path(kind: "secrets" | "configmaps", name?: string): string { + const base = `/api/v1/namespaces/${encodeURIComponent(this.#options.namespace)}/${kind}`; + return name === undefined ? base : `${base}/${encodeURIComponent(name)}`; + } + + async #get( + kind: "secrets" | "configmaps", + name: string, + ): Promise<{ metadata: ObjectMeta; data?: Record; type?: string } | undefined> { + const path = this.#path(kind, name); + const response = await this.#request("GET", path); + if (response.status === 404) { + await response.body?.cancel(); + this.#versions.delete(`${kind}/${name}`); + return undefined; + } + if (!response.ok) throw new KubernetesApiError("GET", path, response.status, await reasonOf(response)); + const object = (await response.json()) as { metadata: ObjectMeta; data?: Record; type?: string }; + if (object.metadata.resourceVersion !== undefined) this.#versions.set(`${kind}/${name}`, object.metadata.resourceVersion); + this.#metadata.set(`${kind}/${name}`, object.metadata); + if (object.type !== undefined) this.#types.set(`${kind}/${name}`, object.type); + return object; + } + + async #put(kind: "secrets" | "configmaps", name: string, body: Record): Promise { + const key = `${kind}/${name}`; + if (!this.#versions.has(key)) await this.#get(kind, name); + const version = this.#versions.get(key); + const existingType = this.#types.get(key); + if (version !== undefined && body.type !== undefined && existingType !== undefined && existingType !== body.type) { + throw new KubernetesApiError( + "PUT", + this.#path(kind, name), + 422, + `Secret ${name} exists with type ${existingType}, not ${String(body.type)}; it is in the way of the installation, so delete or rename it`, + ); + } + const previous = this.#metadata.get(key); + // An object an earlier build wrote may carry Helm's instance label, which a tracking tool would prune by. + const { [TRACKING_LABEL]: _tracking, ...kept } = previous?.labels ?? {}; + const metadata: ObjectMeta = { + name, + labels: { ...kept, ...this.#options.labels }, + annotations: { ...previous?.annotations, ...KEEP_ANNOTATIONS }, + }; + const object = { ...body, metadata: version === undefined ? metadata : { ...metadata, resourceVersion: version } }; + const path = version === undefined ? this.#path(kind) : this.#path(kind, name); + const method = version === undefined ? "POST" : "PUT"; + const response = await this.#request(method, path, object); + if (!response.ok) throw new KubernetesApiError(method, path, response.status, await reasonOf(response)); + const written = (await response.json()) as { metadata: ObjectMeta; type?: string }; + if (written.metadata.resourceVersion !== undefined) this.#versions.set(key, written.metadata.resourceVersion); + this.#metadata.set(key, written.metadata); + if (written.type !== undefined) this.#types.set(key, written.type); + } + + async getSecret(name: string): Promise { + const object = await this.#get("secrets", name); + return object === undefined ? undefined : decode(object.data); + } + + async getConfigMap(name: string): Promise { + return (await this.#get("configmaps", name))?.data; + } + + async restart(workload: Workload, at: Date): Promise<"restarted" | "absent"> { + const path = `/apis/apps/v1/namespaces/${encodeURIComponent(this.#options.namespace)}/${workload.kind}/${encodeURIComponent(workload.name)}`; + const patch = { spec: { template: { metadata: { annotations: { [RESTARTED_AT]: at.toISOString() } } } } }; + const response = await this.#request("PATCH", path, patch, "application/merge-patch+json"); + if (response.status === 404) { + await response.body?.cancel(); + return "absent"; + } + if (!response.ok) throw new KubernetesApiError("PATCH", path, response.status, await reasonOf(response)); + await response.body?.cancel(); + return "restarted"; + } + + async podsCreated(workload: Workload): Promise { + const namespace = encodeURIComponent(this.#options.namespace); + const path = `/apis/apps/v1/namespaces/${namespace}/${workload.kind}/${encodeURIComponent(workload.name)}`; + const response = await this.#request("GET", path); + if (response.status === 404) { + await response.body?.cancel(); + return undefined; + } + if (!response.ok) throw new KubernetesApiError("GET", path, response.status, await reasonOf(response)); + const object = (await response.json()) as { spec: { selector: { matchLabels: Record } } }; + const selector = Object.entries(object.spec.selector.matchLabels).map(([key, value]) => `${key}=${value}`).join(","); + const podsPath = `/api/v1/namespaces/${namespace}/pods?labelSelector=${encodeURIComponent(selector)}`; + const pods = await this.#request("GET", podsPath); + if (!pods.ok) throw new KubernetesApiError("GET", podsPath, pods.status, await reasonOf(pods)); + const list = (await pods.json()) as { items: { metadata: { creationTimestamp: string } }[] }; + return list.items.map((pod) => new Date(pod.metadata.creationTimestamp)); + } + + async putSecret(name: string, data: ObjectData, type: "Opaque" | "kubernetes.io/tls" = "Opaque"): Promise { + await this.#put("secrets", name, { apiVersion: "v1", kind: "Secret", type, data: encode(data) }); + } + + async putConfigMap(name: string, data: ObjectData, binaryData?: Readonly>): Promise { + const binary = + binaryData === undefined + ? {} + : { binaryData: Object.fromEntries(Object.entries(binaryData).map(([key, value]) => [key, Buffer.from(value).toString("base64")])) }; + await this.#put("configmaps", name, { apiVersion: "v1", kind: "ConfigMap", data, ...binary }); + } +} diff --git a/installer/src/cli/kubernetes-job.ts b/installer/src/cli/kubernetes-job.ts new file mode 100644 index 0000000..6596ebd --- /dev/null +++ b/installer/src/cli/kubernetes-job.ts @@ -0,0 +1,114 @@ +// The entry point of the Helm chart's jobs, bundled by scripts/build-helm-jobs.ts into +// charts/libredb-platform/files/libredb-jobs.cjs and run by the Platform image's node: +// +// node libredb-jobs.cjs bootstrap the pre-install and pre-upgrade hook: secrets, seal keys, TLS, envs +// node libredb-jobs.cjs keycloak the keycloak-configure job +// node libredb-jobs.cjs restart the post-install and post-upgrade hook: the restarts the bootstrap asked for +// +// The installation document is the chart's rendering of the Helm values as a LibreDBInstallation of mode kubernetes, +// validated by the installer's own schema. Every line the job prints passes through the redactor. +import { readFileSync, writeFileSync } from "node:fs"; +import { KubernetesApi } from "../adapters/kubernetes-api"; +import { ConfigError, parseInstallation } from "../config/schema"; +import { ExitCode } from "./exit-codes"; +import { runBootstrap, runRestarts } from "../kubernetes/bootstrap"; +import { JOBS_BUNDLE_KEY } from "../kubernetes/names"; +import { runKeycloakJob } from "../kubernetes/keycloak"; +import { Redactor } from "../log/redact"; +import { parseReleaseManifest } from "../release/manifest"; +import { applyKeycloak, observeKeycloak } from "../steps/keycloak-configure"; +import manifestDocument from "../../release/release.config.json" with { type: "json" }; + +/** Where the chart mounts its ConfigMap with installation.json and chart.json. */ +export const CHART_CONFIG_DIR = "/etc/libredb/chart"; + +/** Where the chart mounts the gzipped bundle this file was decompressed from. */ +export const JOBS_DIR = "/etc/libredb/jobs"; + +function readOptional(path: string): string | undefined { + try { + return readFileSync(path, "utf8"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined; + throw error; + } +} + +function readJson(path: string): unknown { + const text = readOptional(path); + if (text === undefined) throw new Error(`${path} is missing: the chart mounts it from its ConfigMap`); + return JSON.parse(text); +} + +/** The error's message with its causes, such as the connection error behind `fetch failed`. */ +export function describeError(error: unknown): string { + const parts: string[] = []; + for (let current: unknown = error; current !== undefined && parts.length < 4; current = (current as { cause?: unknown }).cause) { + parts.push(current instanceof Error ? current.message : String(current)); + if (!(current instanceof Error)) break; + } + return parts.join(": "); +} + +/** + * The job's exit code: the installer's code of a refusal that a retry cannot change (an invalid installation, a failed + * precondition, an unsafe change), on which the chart's podFailurePolicy fails the Job at once, and 1 otherwise. + */ +export function jobExitCode(error: unknown): number { + if (error instanceof ConfigError) return ExitCode.InvalidUsage; + const code = (error as { exitCode?: unknown }).exitCode; + const final: readonly number[] = [ExitCode.InvalidUsage, ExitCode.PreflightFailed, ExitCode.UnsafeChangeRefused]; + return typeof code === "number" && final.includes(code) ? code : ExitCode.RuntimeFailure; +} + +async function main(command: string | undefined): Promise { + const redactor = new Redactor(); + const log = (line: string) => console.log(redactor.redact(line)); + try { + const chart = readJson(`${CHART_CONFIG_DIR}/chart.json`) as { studioSecretName?: unknown; labels?: unknown }; + const labels = (chart.labels ?? {}) as Record; + const manifest = parseReleaseManifest(manifestDocument); + if (command === "restart") { + const objects = KubernetesApi.inCluster(process.env, readOptional, labels); + await runRestarts(objects, manifest.version, () => new Date(), log); + return; + } + const config = parseInstallation(readJson(`${CHART_CONFIG_DIR}/installation.json`)); + if (typeof chart.studioSecretName !== "string" || chart.studioSecretName === "") { + throw new Error(`${CHART_CONFIG_DIR}/chart.json names no studioSecretName`); + } + const objects = KubernetesApi.inCluster(process.env, readOptional, labels); + if (command === "bootstrap") { + await runBootstrap( + { config, manifest, studioSecretName: chart.studioSecretName, jobsBundle: readFileSync(`${JOBS_DIR}/${JOBS_BUNDLE_KEY}`) }, + { objects, readFile: readOptional, env: process.env, now: () => new Date(), log, redactor }, + ); + } else if (command === "keycloak") { + await runKeycloakJob( + config, + manifest, + { + objects, + readFile: readOptional, + writeFile: (path, content) => writeFileSync(path, content, { mode: 0o600 }), + env: process.env, + tmpDir: "/tmp", + owner: { uid: process.getuid?.() ?? 0, gid: process.getgid?.() ?? 0 }, + log, + redactor, + }, + { observe: observeKeycloak, apply: applyKeycloak }, + ); + } else { + console.error("usage: libredb-jobs.cjs bootstrap|keycloak|restart"); + process.exitCode = 64; + return; + } + } catch (error) { + const message = error instanceof ConfigError ? `invalid installation:\n${error.message}` : describeError(error); + console.error(redactor.redact(`error: ${message}`)); + process.exitCode = jobExitCode(error); + } +} + +void main(process.argv[2]); diff --git a/installer/src/kubernetes/bootstrap.ts b/installer/src/kubernetes/bootstrap.ts new file mode 100644 index 0000000..36bce4f --- /dev/null +++ b/installer/src/kubernetes/bootstrap.ts @@ -0,0 +1,612 @@ +// The bootstrap job of the Helm chart charts/libredb-platform: the Kubernetes counterpart of the installer's secrets, +// tls and env steps. It runs as a pre-install and pre-upgrade hook before any app starts, with the installation's +// configuration as the chart rendered it, and writes every Secret and ConfigMap the chart's workloads mount. Every +// value comes from the installer's own modules, so a Helm install gets the same URLs, environments, certificates, +// scripts and keyring as a Docker install of the same configuration. +// A generated secret, a seal key or the internal CA is created once and never replaced; the derived objects are +// written again on every run. +import type { LibreDBInstallation } from "../config/schema"; +import { readSecretReference } from "../config/secret-ref"; +import { APP_SECRET_NAMES, dockerSecretNames, POSTGRES_SUPERUSER_SECRET, siteSecretReferences } from "../docker/secrets"; +import { CLIENT_SECRET_NAMES } from "../keycloak/desired"; +import type { Redactor } from "../log/redact"; +import { computeDesired, type DesiredInstallation } from "../model/desired"; +import { readAuthSecretPrevious, readPlatformLicense, renderPlatformEnv, renderPlatformMigrateEnv } from "../model/platform-env"; +import { renderStudioEnv } from "../model/studio-env"; +import type { ReleaseManifest } from "../release/manifest"; +import { renderCaddyfile } from "../render/caddyfile"; +import { renderPgHba } from "../render/compose"; +import { generateSecret } from "../secrets/generate"; +import { renderKeyring } from "../secrets/keyring"; +import { activeSealKey, parseEnvelopeUses, planSealRotation, sealKidsToRetire, type EnvelopeUse } from "../secrets/seal-rotation"; +import { generateSealKeyPair } from "../secrets/seal-keys"; +import type { SealKeyRecord } from "../secrets/store"; +import { renderDbaScript, renderProvisionScript, ROLE_SPECS, roleVerifiers, expectedRoles } from "../sql/provision"; +import { checkImmutable, immutableFieldsOf } from "../state/immutable"; +import { emptyState, parseState, type InstallerState } from "../state/store"; +import { buildCaBundle, caBundleFiles, postgresCaBundleFile, type CaBundleInput } from "../tls/bundle"; +import { createCa, type KeyedCertificate } from "../tls/ca"; +import { issueLeaf } from "../tls/leaf"; +import { desiredTls, planTls, type ObservedTls } from "../tls/plan"; +import { ExitCode } from "../cli/exit-codes"; +import { compareReleaseVersions } from "../release/compat"; +import { + BOOTSTRAP_CLIENT, + JOBS_BUNDLE_KEY, + K8S_NAMES, + KEYCLOAK_EDGE_PORT, + STORE_KEYS, + STUDIO_SECRET_KEYS, + WORKLOADS, + type Workload, +} from "./names"; + +/** + * A run the configuration or the installation's history refuses, which a retry cannot fix. It carries the installer's + * exit code, on which the chart's jobs fail at once instead of retrying (podFailurePolicy). + */ +export class RefusedError extends Error { + readonly exitCode: ExitCode; + + constructor(message: string, exitCode: ExitCode) { + super(message); + this.name = "RefusedError"; + this.exitCode = exitCode; + } +} + +/** Secret data by key, as plain text; the adapter encodes and decodes base64. */ +export type ObjectData = Record; + +/** The namespace's Secrets and ConfigMaps, which the adapter reaches through the Kubernetes API. */ +export interface KubernetesObjects { + getSecret(name: string): Promise; + getConfigMap(name: string): Promise; + /** Restarts a Deployment or StatefulSet by a pod template annotation, or reports it absent. */ + restart(workload: Workload, at: Date): Promise<"restarted" | "absent">; + /** When each pod of a workload was created, or undefined when the workload does not exist. */ + podsCreated(workload: Workload): Promise; + /** Creates the Secret or replaces its data. */ + putSecret(name: string, data: ObjectData, type?: "Opaque" | "kubernetes.io/tls"): Promise; + /** Creates the ConfigMap or replaces its data and its binary data. */ + putConfigMap(name: string, data: ObjectData, binaryData?: Readonly>): Promise; +} + +export interface BootstrapInput { + config: LibreDBInstallation; + manifest: ReleaseManifest; + /** The Secret the Studio chart reads as secrets.existingSecret. */ + studioSecretName: string; + /** The jobs' gzipped bundle as the hook mounts it, which the keycloak-configure job runs from ConfigMap libredb-jobs. */ + jobsBundle?: Uint8Array; +} + +export interface BootstrapIO { + objects: KubernetesObjects; + /** The content of a mounted input file, or undefined when it does not exist. */ + readFile(path: string): string | undefined; + env: Readonly>; + now(): Date; + log(line: string): void; + redactor: Redactor; +} + +/** The state Secret's key. */ +export const STATE_KEY = "state.json"; + +/** + * The state Secret's key of the Platform release the installation last ran, which a downgrade may not go below. The + * restart job records it once an install or upgrade succeeded, so the rollback that `helm upgrade --atomic` runs after + * a failed upgrade goes back to a release the installation still counts as its own. + */ +export const RELEASE_KEY = "release"; + +/** + * The state Secret's key of the restarts the bootstrap job asks for: the workloads whose inputs it changed and when. + * The restart job runs after Helm applied the release, so a workload restarts once, with the release's pod template, + * and a restart a failed run did not finish stays asked for. + */ +export const PENDING_RESTARTS_KEY = "pending-restarts"; + +export interface PendingRestarts { + /** When the inputs last changed: a pod created after it already read them. */ + since: string; + workloads: Workload[]; +} + +/** The pending restarts in the state Secret, or none. */ +export function pendingRestartsOf(data: ObjectData | undefined): PendingRestarts | undefined { + const text = data?.[PENDING_RESTARTS_KEY]; + return text === undefined || text === "" ? undefined : (JSON.parse(text) as PendingRestarts); +} + +/** Adds workloads to the pending restarts, with the time of this change. */ +export function addPendingRestarts(previous: PendingRestarts | undefined, workloads: readonly Workload[], at: Date): PendingRestarts { + const all = new Map(); + for (const workload of [...(previous?.workloads ?? []), ...workloads]) all.set(`${workload.kind}/${workload.name}`, workload); + return { since: at.toISOString(), workloads: [...all.values()] }; +} + +/** Loads the installer state from its Secret, or an empty state for a new installation. */ +export async function loadState(objects: KubernetesObjects): Promise { + const data = await objects.getSecret(K8S_NAMES.state); + const text = data?.[STATE_KEY]; + return text === undefined ? emptyState() : parseState(text, `Secret ${K8S_NAMES.state}`); +} + +/** Saves the state, keeping the other keys of its Secret, such as the release. */ +export async function saveState(objects: KubernetesObjects, state: InstallerState, extra: ObjectData = {}): Promise { + const current = (await objects.getSecret(K8S_NAMES.state)) ?? {}; + await objects.putSecret(K8S_NAMES.state, { ...current, ...extra, [STATE_KEY]: `${JSON.stringify(state, null, 2)}\n` }); +} + +/** + * Refuses to run an older release than the installation last ran: its migrations and its seal and realm logic would + * meet the newer schema and objects. A downgrade, a `helm rollback` included, restores from a backup instead. + */ +export async function checkRelease(objects: KubernetesObjects, version: string): Promise { + const recorded = (await objects.getSecret(K8S_NAMES.state))?.[RELEASE_KEY]; + if (recorded !== undefined && compareReleaseVersions(version, recorded) < 0) { + throw new RefusedError( + `this chart runs LibreDB ${version}, and the installation already ran ${recorded}: a downgrade is not supported, ` + + "restore the installation from a backup instead", + ExitCode.UnsafeChangeRefused, + ); + } +} + +/** + * Records the immutable fields on the first run and refuses a later run that changes one, such as postgres.mode or + * keycloak.realm, with the installer's UnsafeChangeError. + */ +export function checkInstallation(state: InstallerState, config: LibreDBInstallation): InstallerState { + const desired = immutableFieldsOf(config); + if (state.installation !== undefined) checkImmutable(state.installation, desired); + return { ...state, installation: desired }; +} + +/** The names of the generated secrets: the installer's set of the configuration plus the chart's own. */ +export function generatedNames(config: LibreDBInstallation): string[] { + const names = dockerSecretNames(config); + if (config.keycloak.mode === "managed") names.push(STORE_KEYS.keycloakBootstrapSecret); + if (config.keycloak.firstAdmin.enabled) names.push(STORE_KEYS.firstAdminPassword); + if (config.keycloak.userManager.enabled) names.push(STORE_KEYS.userManagerPassword); + return names; +} + +/** + * Where the chart's seal-envelopes init container writes, for a run whose bridge.sealKeys.retire names a kid, psql's + * output of envelopeUsesScript: the connections in bridge.connections_v1 whose envelopes those kids sealed. + */ +export const SEAL_ENVELOPES_FILE = "/tmp/seal-envelopes.out"; + +/** + * The seal keys after bridge.sealKeys, as the installer's secrets step rotates them: each live kid of retire becomes + * retired, which keeps it in the store and out of Studio's keyring, and issueNew adds one live key while the active key + * is the newest live one, so the same values issue no second key. A retirement is refused with exit 6 while an + * envelope still uses the kid; the chart's restart job deletes a retired kid's canary row once the release applied. + */ +export function rotateSealKeys( + keys: readonly SealKeyRecord[], + config: LibreDBInstallation, + readFile: (path: string) => string | undefined, +): { keys: SealKeyRecord[]; done: string[] } { + const retiring = sealKidsToRetire(keys, config.bridge.sealKeys); + let envelopes: EnvelopeUse[] = []; + if (retiring.length > 0) { + const uses = readFile(SEAL_ENVELOPES_FILE); + if (uses === undefined) { + throw new Error( + `retiring seal key ${retiring.join(", ")} needs the connections each still seals, which the chart's ` + + `seal-envelopes init container writes to ${SEAL_ENVELOPES_FILE}, and it wrote none`, + ); + } + envelopes = parseEnvelopeUses(uses); + } + let rotated = [...keys]; + const done: string[] = []; + for (const action of planSealRotation(keys, config.bridge.sealKeys, envelopes)) { + if (action.kind === "retire-seal-key") { + rotated = rotated.map((key) => (key.kid === action.kid ? { ...key, state: "retired" } : key)); + done.push(`retired seal key ${action.kid}: out of the Studio keyring, kept in the secret store`); + } else if (action.kind === "issue-seal-key") { + const issued: SealKeyRecord = { ...generateSealKeyPair(), state: "live" }; + rotated.push(issued); + done.push(`issued seal key ${issued.kid} into the Studio keyring`); + } + } + return { keys: rotated, done }; +} + +/** Parses the stored seal keys; a new store gets its first live key. */ +export function sealKeysOf(stored: string | undefined): { keys: SealKeyRecord[]; issued: boolean } { + if (stored === undefined) return { keys: [{ ...generateSealKeyPair(), state: "live" }], issued: true }; + const parsed: unknown = JSON.parse(stored); + if (!Array.isArray(parsed) || parsed.length === 0) { + throw new Error(`${STORE_KEYS.sealKeys} of Secret ${K8S_NAMES.secretStore} holds no seal key; restore it from a backup`); + } + return { keys: parsed as SealKeyRecord[], issued: false }; +} + +/** + * The secret store after this run: every missing generated secret drawn once, the site-supplied client secrets of + * keycloak.external.provisioning manual taken from their references, and the seal keys. Never replaces a stored value + * that the run generated before. + */ +export function convergeStore( + config: LibreDBInstallation, + stored: ObjectData, + io: Pick, +): { store: ObjectData; changed: string[]; sealKeys: SealKeyRecord[]; rotated: string[] } { + const store: ObjectData = { ...stored }; + const changed: string[] = []; + for (const name of generatedNames(config)) { + if (store[name] !== undefined && store[name] !== "") continue; + store[name] = generateSecret(); + changed.push(name); + } + for (const site of siteSecretReferences(config)) { + const lookup = readSecretReference(site.reference, { env: io.env, readFile: io.readFile }); + if ("missing" in lookup) throw new Error(`${site.reference.path}: ${lookup.missing}`); + if (store[site.name] === lookup.value) continue; + store[site.name] = lookup.value; + changed.push(site.name); + } + const seal = sealKeysOf(store[STORE_KEYS.sealKeys]); + const rotation = rotateSealKeys(seal.keys, config, io.readFile); + if (seal.issued || rotation.done.length > 0) { + store[STORE_KEYS.sealKeys] = JSON.stringify(rotation.keys); + changed.push(STORE_KEYS.sealKeys); + } + for (const [key, value] of Object.entries(store)) { + if (key !== STORE_KEYS.sealKeys && value !== "") io.redactor.register(value); + } + for (const key of rotation.keys) io.redactor.register(key.privateKey); + return { store, changed, sealKeys: rotation.keys, rotated: rotation.done }; +} + +const CA_CERT = "ca.crt"; +const CA_KEY = "ca.key"; +const leafCert = (name: string) => `${name}.crt`; +const leafKey = (name: string) => `${name}.key`; + +/** The TLS material in Secret libredb-tls as the installer's tls plan observes it. */ +export function observedTls(data: ObjectData, leafNames: readonly string[]): ObservedTls { + const pair = (cert: string, key: string): Partial => ({ + ...(data[cert] === undefined ? {} : { certPem: data[cert] }), + ...(data[key] === undefined ? {} : { keyPem: data[key] }), + }); + return { + ca: pair(CA_CERT, CA_KEY), + leaves: Object.fromEntries(leafNames.map((name) => [name, pair(leafCert(name), leafKey(name))])), + }; +} + +/** + * Creates the internal CA when it is missing and issues every leaf the installer's tls plan issues: missing, + * unreadable, not covering its hosts, not chaining to the CA, or with fewer than 30 days left. A present CA is never + * replaced. + */ +export async function convergeTls( + config: LibreDBInstallation, + stored: ObjectData, + now: Date, + log: (line: string) => void, +): Promise<{ data: ObjectData; changed: boolean }> { + const desired = desiredTls(config); + const plan = planTls(observedTls(stored, desired.leaves.map((leaf) => leaf.name)), desired, now); + const data: ObjectData = { ...stored }; + let changed = false; + if (plan.ca === "create") { + const ca = await createCa({ commonName: desired.caCommonName, now }); + data[CA_CERT] = ca.certPem; + data[CA_KEY] = ca.keyPem; + changed = true; + log(`tls: created the internal CA "${desired.caCommonName}"`); + } + const ca = { certPem: data[CA_CERT] as string, keyPem: data[CA_KEY] as string }; + for (const leaf of plan.leaves) { + if (leaf.action === "keep") continue; + const issued = await issueLeaf(ca, leaf.hosts, now); + data[leafCert(leaf.name)] = issued.certPem; + data[leafKey(leaf.name)] = issued.keyPem; + changed = true; + log(`tls: issued the ${leaf.name} leaf for ${leaf.hosts.join(", ")} (${leaf.reasons.join("; ")})`); + } + return { data, changed }; +} + +/** The CA bundle of every app container, built as the installer builds it from the configured caBundleFile inputs. */ +export function caBundleOf(config: LibreDBInstallation, caPem: string, readFile: (path: string) => string | undefined): string { + const read = (path: string): CaBundleInput => { + const pem = readFile(path); + if (pem === undefined) throw new Error(`the CA bundle file ${path} does not exist`); + return { source: path, pem }; + }; + const postgresFile = postgresCaBundleFile(config); + const inputs = caBundleFiles(config) + .filter((path) => path !== postgresFile) + .map(read); + return buildCaBundle(caPem, inputs, postgresFile === undefined ? [] : [read(postgresFile)]); +} + +/** The role passwords from the store, by app role. */ +function rolePasswords(config: LibreDBInstallation, store: ObjectData): Partial> { + const roles = expectedRoles({ managedKeycloak: config.keycloak.mode === "managed", backupRole: true }); + return Object.fromEntries(roles.map((role) => [role, required(store, ROLE_SPECS[role].secret)])); +} + +function required(store: ObjectData, name: string): string { + const value = store[name]; + if (value === undefined || value === "") throw new Error(`Secret ${K8S_NAMES.secretStore} holds no ${name}`); + return value; +} + +/** Whether two objects hold the same keys and values. */ +export function sameData(a: ObjectData, b: ObjectData): boolean { + const keys = Object.keys(a); + return keys.length === Object.keys(b).length && keys.every((key) => a[key] === b[key]); +} + +/** Drops the variables an env renderer left unset. */ +function defined(env: Readonly>): ObjectData { + return Object.fromEntries(Object.entries(env).filter((entry): entry is [string, string] => entry[1] !== undefined)); +} + +/** + * The Caddyfile of a managed Keycloak's edge: the installer's auth site as it renders behind an upstream TLS proxy, here + * the ingress controller, so only /realms// and /resources/ reach Keycloak, and a non-canonical path does not. + */ +export function keycloakEdgeCaddyfile(config: LibreDBInstallation, manifest: ReleaseManifest): string { + const behindIngress: LibreDBInstallation = { + ...config, + tls: { mode: "external", caBundleFile: "/dev/null", external: { httpPort: KEYCLOAK_EDGE_PORT, trustedProxies: ["private_ranges"] } }, + }; + const desired = computeDesired(behindIngress, manifest); + return renderCaddyfile({ ...desired, edge: { ...desired.edge, sites: desired.edge.sites.filter((site) => site.endpoint === "auth") } }); +} + +/** A derived object with the workloads that read it at start, which a change of its content restarts. */ +export interface RenderedObjects { + secrets: { name: string; data: ObjectData; type?: "kubernetes.io/tls"; readers?: Workload[] }[]; + configMaps: { name: string; data: ObjectData; readers?: Workload[] }[]; +} + +/** The keys of Studio's environment that the Studio chart has no value for, as the installer renders them. */ +export const STUDIO_NETWORK_KEYS = ["NO_PROXY", "no_proxy", "HTTPS_PROXY", "NODE_USE_ENV_PROXY"] as const; + +/** Every derived object of the run, from the store, the TLS material and the configuration. */ +export function renderObjects( + input: BootstrapInput, + desired: DesiredInstallation, + store: ObjectData, + sealKeys: readonly SealKeyRecord[], + tls: ObjectData, + io: Pick, +): RenderedObjects { + const { config } = input; + const postgres = { host: desired.postgres.host, port: desired.postgres.port }; + const passwords = rolePasswords(config, store); + const secrets: RenderedObjects["secrets"] = []; + const configMaps: RenderedObjects["configMaps"] = []; + + configMaps.push({ + name: K8S_NAMES.caBundle, + data: { [CA_CERT]: caBundleOf(config, tls[CA_CERT] as string, io.readFile) }, + readers: [WORKLOADS.platform, WORKLOADS.studio, WORKLOADS.keycloak], + }); + + const authSecretPrevious = readAuthSecretPrevious(config.platform, io.readFile); + if (authSecretPrevious !== undefined) io.redactor.register(authSecretPrevious); + const platformEnv = renderPlatformEnv(desired, { + postgres, + secrets: { + authSecret: required(store, APP_SECRET_NAMES.platformAuthSecret), + ...(authSecretPrevious === undefined ? {} : { authSecretPrevious }), + license: readPlatformLicense(config.platform, io.readFile, io.redactor), + oidcClientSecret: required(store, CLIENT_SECRET_NAMES.platform), + databasePassword: passwords.platform as string, + }, + sealPublicKey: activeSealKey(sealKeys, config.bridge.sealKeys.activeKid).publicKey, + }); + secrets.push({ name: K8S_NAMES.platformEnv, data: defined(platformEnv), readers: [WORKLOADS.platform] }); + + const migrateEnv = renderPlatformMigrateEnv(desired, { + postgres, + secrets: { databasePassword: passwords.platform as string, ownerPassword: passwords.owner as string }, + }); + secrets.push({ name: K8S_NAMES.migrateEnv, data: defined(migrateEnv) }); + + const studioEnv = renderStudioEnv(desired, { + postgres, + secrets: { + oidcClientSecret: required(store, CLIENT_SECRET_NAMES.studio), + jwtSecret: required(store, APP_SECRET_NAMES.studioJwtSecret), + storageEncryptionKey: required(store, APP_SECRET_NAMES.studioStorageEncryptionKey), + databasePassword: passwords.studio as string, + }, + }); + secrets.push({ + name: input.studioSecretName, + data: { + [STUDIO_SECRET_KEYS.jwtSecret]: studioEnv.JWT_SECRET as string, + [STUDIO_SECRET_KEYS.oidcClientId]: studioEnv.OIDC_CLIENT_ID as string, + [STUDIO_SECRET_KEYS.oidcClientSecret]: studioEnv.OIDC_CLIENT_SECRET as string, + [STUDIO_SECRET_KEYS.storagePostgresUrl]: studioEnv.STORAGE_POSTGRES_URL as string, + [STUDIO_SECRET_KEYS.governancePostgresUrl]: studioEnv.GOVERNANCE_POSTGRES_URL as string, + [STUDIO_SECRET_KEYS.governancePrivateKeys]: renderKeyring(sealKeys), + [STUDIO_SECRET_KEYS.storageEncryptionKey]: studioEnv.STORAGE_ENCRYPTION_KEY as string, + }, + readers: [WORKLOADS.studio], + }); + secrets.push({ + name: K8S_NAMES.studioNetworkEnv, + data: defined(Object.fromEntries(STUDIO_NETWORK_KEYS.map((key) => [key, studioEnv[key]]))), + readers: [WORKLOADS.studio], + }); + + const managedKeycloak = config.keycloak.mode === "managed"; + if (config.postgres.mode === "managed") { + secrets.push({ name: K8S_NAMES.postgresSuperuser, data: { POSTGRES_PASSWORD: required(store, POSTGRES_SUPERUSER_SECRET) } }); + secrets.push({ + name: K8S_NAMES.postgresTls, + data: { "server.crt": tls[leafCert("postgres")] as string, "server.key": tls[leafKey("postgres")] as string }, + // PostgreSQL reads its leaf at start; a renewed leaf restarts it, which the installer does with a reload. + readers: [WORKLOADS.postgres], + }); + configMaps.push({ name: K8S_NAMES.postgresConfig, data: { "pg_hba.conf": renderPgHba() }, readers: [WORKLOADS.postgres] }); + const roles = expectedRoles({ managedKeycloak, backupRole: true }); + secrets.push({ + name: K8S_NAMES.postgresProvision, + data: { + "provision.sql": renderProvisionScript({ + verifiers: roleVerifiers(passwords), + managedKeycloak, + resetPasswords: roles.filter((role) => role !== "backup"), + }), + }, + }); + } else if (config.postgres.external.provisioning === "manual") { + secrets.push({ + name: K8S_NAMES.postgresDbaScript, + data: { "provision.sql": renderDbaScript({ passwords, managedKeycloak, server: postgres }) }, + }); + } + + // Only the leaves the configuration still wants: after a switch to tls.mode secrets the store keeps the old ones. + const wanted = new Set(desiredTls(config).leaves.map((leaf) => leaf.name)); + for (const endpoint of ["platform", "studio", "auth"] as const) { + const cert = tls[leafCert(endpoint)]; + const key = tls[leafKey(endpoint)]; + if (!wanted.has(endpoint) || cert === undefined || key === undefined) continue; + secrets.push({ + name: K8S_NAMES.ingressTls(endpoint), + type: "kubernetes.io/tls", + data: { "tls.crt": cert, "tls.key": key, [CA_CERT]: tls[CA_CERT] as string }, + }); + } + + if (managedKeycloak) { + configMaps.push({ + name: K8S_NAMES.keycloakEdge, + data: { Caddyfile: keycloakEdgeCaddyfile(config, input.manifest) }, + readers: [WORKLOADS.keycloak], + }); + secrets.push({ + name: K8S_NAMES.keycloakEnv, + data: { + KC_DB_PASSWORD: passwords.keycloak as string, + KC_BOOTSTRAP_ADMIN_CLIENT_ID: BOOTSTRAP_CLIENT, + KC_BOOTSTRAP_ADMIN_CLIENT_SECRET: required(store, STORE_KEYS.keycloakBootstrapSecret), + }, + readers: [WORKLOADS.keycloak], + }); + } + const keycloakConfig: ObjectData = { + [CLIENT_SECRET_NAMES.studio]: required(store, CLIENT_SECRET_NAMES.studio), + [CLIENT_SECRET_NAMES.platform]: required(store, CLIENT_SECRET_NAMES.platform), + }; + if (managedKeycloak) keycloakConfig[STORE_KEYS.keycloakBootstrapSecret] = required(store, STORE_KEYS.keycloakBootstrapSecret); + for (const key of [STORE_KEYS.firstAdminPassword, STORE_KEYS.userManagerPassword]) { + if (store[key] !== undefined) keycloakConfig[key] = store[key]; + } + secrets.push({ name: K8S_NAMES.keycloakConfig, data: keycloakConfig }); + + return { secrets, configMaps }; +} + +/** Runs the bootstrap job. */ +export async function runBootstrap(input: BootstrapInput, io: BootstrapIO): Promise { + const { config, manifest } = input; + if (config.postgres.mode === "external" && config.postgres.external.provisioning !== "manual") { + throw new RefusedError( + "the Helm chart supports an external PostgreSQL server only with provisioning manual: a DBA runs the script in " + + `Secret ${K8S_NAMES.postgresDbaScript}, so no admin credential of the server enters the cluster`, + ExitCode.InvalidUsage, + ); + } + await checkRelease(io.objects, manifest.version); + const state = checkInstallation(await loadState(io.objects), config); + + const converged = convergeStore(config, (await io.objects.getSecret(K8S_NAMES.secretStore)) ?? {}, io); + for (const line of converged.rotated) io.log(`seal keys: ${line}`); + if (converged.changed.length > 0) { + await io.objects.putSecret(K8S_NAMES.secretStore, converged.store); + io.log(`secrets: stored ${converged.changed.join(", ")} in Secret ${K8S_NAMES.secretStore}`); + } + + const tls = await convergeTls(config, (await io.objects.getSecret(K8S_NAMES.tls)) ?? {}, io.now(), io.log); + if (tls.changed) await io.objects.putSecret(K8S_NAMES.tls, tls.data); + + const desired = computeDesired(config, manifest); + const rendered = renderObjects(input, desired, converged.store, converged.sealKeys, tls.data, io); + if (input.jobsBundle !== undefined) { + await io.objects.putConfigMap(K8S_NAMES.jobsBundle, {}, { [JOBS_BUNDLE_KEY]: input.jobsBundle }); + io.log(`wrote ConfigMap ${K8S_NAMES.jobsBundle}`); + } + // A workload reads its Secrets and ConfigMaps when it starts, so one whose input changed must restart. The restarts + // are recorded before any object is written, so a run that fails halfway still leaves them asked for, and the + // restart job carries them out after Helm applied the release. + const restart = new Map(); + const changed = (before: ObjectData | undefined, after: ObjectData, readers: Workload[] | undefined) => { + if (readers === undefined || (before !== undefined && sameData(before, after))) return; + for (const reader of readers) restart.set(`${reader.kind}/${reader.name}`, reader); + }; + for (const configMap of rendered.configMaps) changed(await io.objects.getConfigMap(configMap.name), configMap.data, configMap.readers); + for (const secret of rendered.secrets) changed(await io.objects.getSecret(secret.name), secret.data, secret.readers); + const stateData = (await io.objects.getSecret(K8S_NAMES.state)) ?? {}; + const pending = pendingRestartsOf(stateData); + if (restart.size > 0) { + const asked = addPendingRestarts(pending, [...restart.values()], io.now()); + await io.objects.putSecret(K8S_NAMES.state, { ...stateData, [PENDING_RESTARTS_KEY]: JSON.stringify(asked) }); + io.log(`asked to restart ${asked.workloads.map((workload) => `${workload.kind}/${workload.name}`).join(", ")} once the release is applied`); + } + for (const configMap of rendered.configMaps) { + await io.objects.putConfigMap(configMap.name, configMap.data); + io.log(`wrote ConfigMap ${configMap.name}`); + } + for (const secret of rendered.secrets) { + await io.objects.putSecret(secret.name, secret.data, secret.type); + io.log(`wrote Secret ${secret.name}`); + } + await saveState(io.objects, state); + io.log("bootstrap done"); +} + +/** + * The restart job, a post-install and post-upgrade hook: restarts each workload the bootstrap job asked for, unless all + * its pods were created after the change, as a rollout of the release's own pod template or a first install makes + * them; a workload that does not exist is dropped. The request is cleared only once every restart was done, and the + * release is then recorded as the one the installation runs. + */ +export async function runRestarts( + objects: KubernetesObjects, + version: string, + now: () => Date, + log: (line: string) => void, +): Promise { + const stateData = (await objects.getSecret(K8S_NAMES.state)) ?? {}; + const pending = pendingRestartsOf(stateData); + if (pending === undefined) { + log("no restart asked for"); + await objects.putSecret(K8S_NAMES.state, { ...stateData, [RELEASE_KEY]: version }); + log(`the installation runs LibreDB ${version}`); + return; + } + const since = new Date(pending.since).getTime(); + for (const workload of pending.workloads) { + const name = `${workload.kind}/${workload.name}`; + const created = await objects.podsCreated(workload); + if (created === undefined) { + log(`${name} does not exist`); + } else if (created.length > 0 && created.every((at) => at.getTime() >= since)) { + log(`${name} started after its inputs changed`); + } else { + await objects.restart(workload, now()); + log(`restarted ${name}: an input it reads at start changed`); + } + } + const { [PENDING_RESTARTS_KEY]: _done, ...rest } = (await objects.getSecret(K8S_NAMES.state)) ?? {}; + await objects.putSecret(K8S_NAMES.state, { ...rest, [RELEASE_KEY]: version }); + log(`the installation runs LibreDB ${version}`); +} diff --git a/installer/src/kubernetes/keycloak.ts b/installer/src/kubernetes/keycloak.ts new file mode 100644 index 0000000..f364b67 --- /dev/null +++ b/installer/src/kubernetes/keycloak.ts @@ -0,0 +1,213 @@ +// The keycloak-configure job of the Helm chart: the Kubernetes counterpart of the installer's keycloak-configure step, +// which it runs unchanged. The realm, its clients, client scope, roles and accounts come from desiredKeycloak, the +// actions from planKeycloak, and every Admin REST request from the same request builder, so a Helm install converges +// the same objects as a Docker install. The installer state, which records the accounts it created, lives in a Secret. +// +// A managed Keycloak is reached in-cluster with the bootstrap admin client of realm master, whose secret only the +// generated Secrets hold. Unlike the Docker install, the chart keeps that client: Keycloak's environment is part of a +// Helm-managed Deployment, and Keycloak would recreate the client from it on the next start. +import type { AdminCredentials } from "../adapters/keycloak-admin"; +import { KeycloakAdmin } from "../adapters/keycloak-admin"; +import type { LibreDBInstallation } from "../config/schema"; +import { desiredKeycloak, managedRealm, CLIENT_SECRET_NAMES, type Account } from "../keycloak/desired"; +import { lookupAdminCredentials } from "../keycloak/external"; +import { planKeycloak, type KeycloakAction } from "../keycloak/planner"; +import { MANAGED_ATTRIBUTE } from "../keycloak/representations"; +import type { Redactor } from "../log/redact"; +import { computeDesired } from "../model/desired"; +import type { ReleaseManifest } from "../release/manifest"; +import type { InstallerState } from "../state/store"; +import { ExitCode } from "../cli/exit-codes"; +import { loadState, RefusedError, saveState, type KubernetesObjects } from "./bootstrap"; +import { BOOTSTRAP_CLIENT, K8S_NAMES, KEYCLOAK_SERVICE, STORE_KEYS } from "./names"; + +/** The in-cluster Admin API of a managed Keycloak: its Service on the HTTP port. */ +export const MANAGED_KEYCLOAK_URL = `http://${KEYCLOAK_SERVICE}:8080`; + +/** Where the chart mounts Secret libredb-keycloak-config into the job. */ +export const KEYCLOAK_CONFIG_DIR = "/etc/libredb/keycloak-config"; + +/** The installer step's functions, injected so the pure module does not import the Docker step. */ +export interface ConvergeFunctions { + observe(target: ConvergeTarget, state: InstallerState): Promise[1]>; + apply(actions: readonly KeycloakAction[], target: ConvergeTarget, state: InstallerState): Promise; +} + +/** The ConfigureTarget of the installer's keycloak-configure step. */ +export interface ConvergeTarget { + baseUrl: string; + realm: string; + credentials: AdminCredentials; + desired: ReturnType; + passwordFiles: Readonly>; + owner: { uid: number; gid: number }; + redactor: Redactor; + log: (line: string) => void; + fetch?: typeof fetch; +} + +export interface KeycloakJobIO { + objects: KubernetesObjects; + readFile(path: string): string | undefined; + /** Writes a 0600 file of the job's own user, for the step's temporary password files. */ + writeFile(path: string, content: string): void; + env: Readonly>; + /** A directory the job may write to. */ + tmpDir: string; + owner: { uid: number; gid: number }; + log(line: string): void; + redactor: Redactor; + fetch?: typeof fetch; + sleep?: (ms: number) => Promise; + now?: () => number; +} + +function configValue(io: Pick, key: string): string { + const value = io.readFile(`${KEYCLOAK_CONFIG_DIR}/${key}`); + if (value === undefined || value === "") throw new Error(`Secret ${K8S_NAMES.keycloakConfig} holds no ${key}`); + io.redactor.register(value); + return value; +} + +/** Where the job reaches the Admin API and as whom. */ +export function adminTarget( + config: LibreDBInstallation, + io: Pick, +): { baseUrl: string; credentials: AdminCredentials } { + if (config.keycloak.mode === "managed") { + return { + baseUrl: MANAGED_KEYCLOAK_URL, + credentials: { realm: "master", clientId: BOOTSTRAP_CLIENT, clientSecret: configValue(io, STORE_KEYS.keycloakBootstrapSecret) }, + }; + } + const { external } = config.keycloak; + if (external.provisioning === "manual") { + throw new RefusedError( + "keycloak.external.provisioning is manual: the site configures the realm, and the chart runs no keycloak job", + ExitCode.InvalidUsage, + ); + } + const lookup = lookupAdminCredentials(external, { env: io.env, readFile: io.readFile }); + if ("missing" in lookup) throw new RefusedError(lookup.missing, ExitCode.InvalidUsage); + io.redactor.register(lookup.secret); + return { baseUrl: external.adminUrl ?? external.baseUrl, credentials: lookup.credentials }; +} + +/** Creates the realm of a managed Keycloak with the settings the converge keeps, as the installer's handover does. */ +export async function ensureManagedRealm(admin: Pick, realm: string): Promise<"created" | "exists"> { + const found = await admin.find<{ attributes?: Record }>(`/admin/realms/${encodeURIComponent(realm)}`); + if (found === undefined) { + await admin.write("POST", "/admin/realms", managedRealm(realm), [201]); + return "created"; + } + if (found.attributes?.[MANAGED_ATTRIBUTE] !== "true") { + throw new RefusedError( + `realm ${realm} exists without the attribute ${MANAGED_ATTRIBUTE}=true that the installer gives the realm it ` + + "creates, so the chart does not take it over; install with keycloak.mode external instead", + ExitCode.UnsafeChangeRefused, + ); + } + return "exists"; +} + +/** How long the job waits for Keycloak to answer before it fails, and how often it tries. */ +export const KEYCLOAK_WAIT = { timeoutMs: 10 * 60_000, intervalMs: 3_000 } as const; + +/** + * Waits until Keycloak answers HTTP at `baseUrl`, whatever the status, so a job that starts beside a Keycloak that is + * still starting waits instead of failing; it fails with the last connection error once `timeoutMs` passed. + */ +export async function waitForKeycloak( + baseUrl: string, + options: { + fetch: typeof fetch; + log: (line: string) => void; + sleep: (ms: number) => Promise; + now: () => number; + timeoutMs?: number; + intervalMs?: number; + }, +): Promise { + const { timeoutMs = KEYCLOAK_WAIT.timeoutMs, intervalMs = KEYCLOAK_WAIT.intervalMs } = options; + const deadline = options.now() + timeoutMs; + let waiting = false; + for (;;) { + try { + const response = await options.fetch(`${baseUrl}/realms/master`, { signal: AbortSignal.timeout(intervalMs * 3) }); + await response.body?.cancel(); + if (waiting) options.log(`keycloak: ${baseUrl} answers`); + return; + } catch (error) { + if (options.now() >= deadline) { + throw new Error(`Keycloak at ${baseUrl} did not answer within ${timeoutMs / 1000} s: ${(error as Error).message}`); + } + if (!waiting) options.log(`keycloak: waiting for ${baseUrl} to answer`); + waiting = true; + await options.sleep(intervalMs); + } + } +} + +/** Runs the keycloak-configure job. */ +export async function runKeycloakJob( + config: LibreDBInstallation, + manifest: ReleaseManifest, + io: KeycloakJobIO, + converge: ConvergeFunctions, +): Promise { + const desired = computeDesired(config, manifest); + const { baseUrl, credentials } = adminTarget(config, io); + await waitForKeycloak(baseUrl, { + fetch: io.fetch ?? fetch, + log: io.log, + sleep: io.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))), + now: io.now ?? Date.now, + }); + if (config.keycloak.mode === "managed") { + const admin = new KeycloakAdmin({ baseUrl, credentials, log: io.log, ...(io.fetch === undefined ? {} : { fetch: io.fetch }) }); + const realm = await ensureManagedRealm(admin, desired.realm); + io.log(`keycloak: realm ${desired.realm} ${realm}`); + } + const passwordFiles: Record = { + "first-admin": `${io.tmpDir}/first-admin.txt`, + "user-manager": `${io.tmpDir}/user-manager.txt`, + }; + // The temporary passwords were generated by the bootstrap job; the step reads a stored password file before it + // generates one, so the accounts get exactly these. + const accounts: [Account, boolean, string][] = [ + ["first-admin", config.keycloak.firstAdmin.enabled, STORE_KEYS.firstAdminPassword], + ["user-manager", config.keycloak.userManager.enabled, STORE_KEYS.userManagerPassword], + ]; + for (const [account, enabled, key] of accounts) { + if (enabled) io.writeFile(passwordFiles[account], `${configValue(io, key)}\n`); + } + const target: ConvergeTarget = { + baseUrl, + realm: desired.realm, + credentials, + desired: desiredKeycloak({ + installation: desired, + keycloak: config.keycloak, + clientSecrets: { + studio: configValue(io, CLIENT_SECRET_NAMES.studio), + platform: configValue(io, CLIENT_SECRET_NAMES.platform), + }, + passwordLocations: { + firstAdmin: `Secret ${K8S_NAMES.keycloakConfig} key ${STORE_KEYS.firstAdminPassword}`, + userManager: `Secret ${K8S_NAMES.keycloakConfig} key ${STORE_KEYS.userManagerPassword}`, + }, + }), + passwordFiles, + owner: io.owner, + redactor: io.redactor, + log: io.log, + ...(io.fetch === undefined ? {} : { fetch: io.fetch }), + }; + const state = await loadState(io.objects); + const observed = await converge.observe(target, state); + const actions = planKeycloak(target.desired, observed); + for (const action of actions) io.log(`keycloak: ${action.summary}`); + await converge.apply(actions, target, state); + await saveState(io.objects, state); + io.log(actions.length === 0 ? "keycloak: realm already converged" : `keycloak: applied ${actions.length} actions`); +} diff --git a/installer/src/kubernetes/names.ts b/installer/src/kubernetes/names.ts new file mode 100644 index 0000000..efd9caf --- /dev/null +++ b/installer/src/kubernetes/names.ts @@ -0,0 +1,91 @@ +// The names of the Kubernetes objects of an installation that the Helm chart charts/libredb-platform deploys. The +// chart's templates name the same objects; a chart test renders them and compares, so the two never drift. +// Every name is fixed, like the installer's Kubernetes constants (Service libredb-postgres), so a namespace holds one +// installation. +import { POSTGRES_KUBERNETES_SERVICE } from "../tls/plan"; + +/** The Service of a managed Keycloak; Platform's KEYCLOAK_ADMIN_URL of a managed Keycloak is http://keycloak:8080. */ +export const KEYCLOAK_SERVICE = "keycloak"; + +export const K8S_NAMES = { + /** Generated secrets and seal keys, never overwritten once generated. */ + secretStore: "libredb-secrets", + /** The installer state (installer/src/state/store.ts), which holds no secret value. */ + state: "libredb-installer-state", + /** The internal CA and the leaves it issued, as PEM. */ + tls: "libredb-tls", + /** The CA bundle every app container mounts at /etc/libredb/ca/ca.crt. */ + caBundle: "libredb-ca", + /** Platform's runtime environment, for `serve` only. */ + platformEnv: "libredb-platform-env", + /** The migrate job's environment, the only place MIGRATE_DATABASE_URL lives. */ + migrateEnv: "libredb-platform-migrate-env", + /** The managed PostgreSQL server: its Service, which the leaf covers, and its objects. */ + postgres: POSTGRES_KUBERNETES_SERVICE, + /** The superuser password of the managed PostgreSQL server, as POSTGRES_PASSWORD. */ + postgresSuperuser: "libredb-postgres-superuser", + /** The managed server's leaf and key. */ + postgresTls: "libredb-postgres-tls", + /** pg_hba.conf of the managed server. */ + postgresConfig: "libredb-postgres-config", + /** The provisioning script of a managed server, with SCRAM verifiers only. */ + postgresProvision: "libredb-postgres-provision", + /** The script a DBA runs on an external server with provisioning manual, with SCRAM verifiers only. */ + postgresDbaScript: "libredb-postgres-dba-script", + /** A managed Keycloak's database password and bootstrap admin client. */ + keycloakEnv: "libredb-keycloak-env", + /** The Caddyfile of a managed Keycloak's edge, the installer's auth site behind the ingress. */ + keycloakEdge: "libredb-keycloak-edge", + /** Studio's network environment of the installer (NO_PROXY, and HTTPS_PROXY with a proxy), read through studio.extraEnvFrom. */ + studioNetworkEnv: "libredb-studio-network-env", + /** The jobs' gzipped bundle, which the bootstrap job copies from its hook ConfigMap for the keycloak-configure job. */ + jobsBundle: "libredb-jobs", + /** What the keycloak-configure job needs: the client secrets and the temporary passwords. */ + keycloakConfig: "libredb-keycloak-config", + /** The ingress leaves of tls.mode generated, one kubernetes.io/tls Secret per endpoint. */ + ingressTls: (endpoint: "platform" | "studio" | "auth") => `libredb-tls-${endpoint}`, +} as const; + +/** The plain HTTP port of a managed Keycloak's edge, which the ingress forwards hosts.auth to. */ +export const KEYCLOAK_EDGE_PORT = 8081; + +/** The bootstrap admin client of a managed Keycloak in realm master, as the installer's handover names it. */ +export const BOOTSTRAP_CLIENT = "libredb-bootstrap"; + +/** The secret store's names besides the installer's generated secrets. */ +export const STORE_KEYS = { + sealKeys: "seal-keys.json", + keycloakBootstrapSecret: "keycloak-bootstrap-client-secret", + firstAdminPassword: "first-admin-password", + userManagerPassword: "user-manager-password", +} as const; + +/** The workloads the bootstrap job restarts when an object they read at start changed. */ +export const WORKLOADS = { + platform: { kind: "deployments", name: "libredb-platform" }, + studio: { kind: "deployments", name: "libredb-studio" }, + keycloak: { kind: "deployments", name: KEYCLOAK_SERVICE }, + postgres: { kind: "statefulsets", name: POSTGRES_KUBERNETES_SERVICE }, +} as const; + +export type Workload = (typeof WORKLOADS)[keyof typeof WORKLOADS]; + +/** The label of every object the jobs write, which names the installation: never a Helm or Argo CD tracking label. */ +export const INSTALLATION_LABEL = "install.libredb.org/installation"; + +/** The key of the gzipped bundle in its ConfigMaps. */ +export const JOBS_BUNDLE_KEY = "libredb-jobs.cjs.gz"; + +/** Where the chart mounts the operator's inputs into the bootstrap job. */ +export const INPUT_DIR = "/etc/libredb/inputs"; + +/** The keys of the Studio Secret, which the Studio chart reads as secrets.existingSecret. */ +export const STUDIO_SECRET_KEYS = { + jwtSecret: "jwt-secret", + oidcClientId: "oidc-client-id", + oidcClientSecret: "oidc-client-secret", + storagePostgresUrl: "storage-postgres-url", + governancePostgresUrl: "governance-postgres-url", + governancePrivateKeys: "governance-private-keys", + storageEncryptionKey: "storage-encryption-key", +} as const; diff --git a/installer/src/verify/oidc-chain.ts b/installer/src/verify/oidc-chain.ts index 337f192..c4afe2f 100644 --- a/installer/src/verify/oidc-chain.ts +++ b/installer/src/verify/oidc-chain.ts @@ -67,6 +67,12 @@ const chainResult = z }) .strict(); +/** Why Studio sends its callback with http: where the public URL is https:. */ +const PLAIN_HTTP_HINT = + "Studio saw plain HTTP, so the edge did not pass X-Forwarded-Proto on; in tls.mode external, " + + "tls.external.trustedProxies must hold the address the upstream proxy arrives from, " + + "which for a proxy on the Docker host is the gateway of the network libredb_edge"; + /** What a redirect to /login?error= of Studio's OIDC login route means. */ function loginErrorMeaning(code: string, issuer: string): string { if (code === "oidc_config") return "Studio's OIDC configuration is invalid"; @@ -120,9 +126,9 @@ export const studioOidcChainCheck: Check = { } const redirectUri = location.searchParams.get("redirect_uri"); if (redirectUri !== oidc.studioRedirectUri) { - return failed( - `the authorization request sends redirect_uri ${redirectUri}, and Studio's callback is ${oidc.studioRedirectUri}`, - ); + const cause = `the authorization request sends redirect_uri ${redirectUri}, and Studio's callback is ${oidc.studioRedirectUri}`; + // Studio builds its callback from X-Forwarded-Proto, which the edge drops from a proxy it does not trust. + return failed(redirectUri === oidc.studioRedirectUri.replace(/^https:/, "http:") ? `${cause}: ${PLAIN_HTTP_HINT}` : cause); } const prompt = location.searchParams.get("prompt"); if (prompt !== null) { diff --git a/installer/test/unit/adapters/kubernetes-api.test.ts b/installer/test/unit/adapters/kubernetes-api.test.ts new file mode 100644 index 0000000..8c1462b --- /dev/null +++ b/installer/test/unit/adapters/kubernetes-api.test.ts @@ -0,0 +1,166 @@ +import { describe, expect, test } from "bun:test"; +import { KEEP_ANNOTATIONS, KubernetesApi, KubernetesApiError, RESTARTED_AT, SERVICE_ACCOUNT_DIR } from "../../../src/adapters/kubernetes-api"; + +interface Call { + method: string; + url: string; + body?: { + metadata: { name: string; resourceVersion?: string; labels?: Record; annotations?: Record }; + data?: Record; + type?: string; + }; +} + +/** An API server with one namespace's Secrets, answering as kube-apiserver does. */ +function apiServer(initial: Record; version: string; type?: string; labels?: Record }> = {}) { + const objects = new Map(Object.entries(initial)); + const calls: Call[] = []; + let version = 100; + const fetcher = (async (input: string | URL | Request, init?: RequestInit) => { + const url = String(input); + const method = init?.method ?? "GET"; + const body = init?.body === undefined ? undefined : (JSON.parse(String(init.body)) as Call["body"]); + calls.push({ method, url, ...(body === undefined ? {} : { body }) }); + const name = url.split("/").pop() as string; + if (method === "GET") { + const found = objects.get(name); + if (found === undefined) return Response.json({ message: "not found" }, { status: 404 }); + return Response.json({ metadata: { name, resourceVersion: found.version, labels: found.labels }, data: found.data, type: found.type }); + } + const target = body?.metadata.name as string; + if (method === "PUT" && objects.get(target)?.version !== body?.metadata.resourceVersion) { + return Response.json({ message: "the object has been modified" }, { status: 409 }); + } + version += 1; + objects.set(target, { data: body?.data ?? {}, version: String(version) }); + return Response.json({ metadata: { name: target, resourceVersion: String(version) } }, { status: method === "POST" ? 201 : 200 }); + }) as typeof fetch; + return { fetcher, calls, objects }; +} + +const b64 = (text: string) => Buffer.from(text).toString("base64"); + +function api(fetcher: typeof fetch) { + return new KubernetesApi({ server: "https://10.0.0.1:443", token: "t", namespace: "libredb", labels: { "app.kubernetes.io/part-of": "libredb" }, fetch: fetcher }); +} + +describe("KubernetesApi", () => { + test("reads a Secret decoded, and undefined for a missing one", async () => { + const server = apiServer({ present: { data: { key: b64("value") }, version: "7" } }); + const kube = api(server.fetcher); + expect(await kube.getSecret("present")).toEqual({ key: "value" }); + expect(await kube.getSecret("missing")).toBeUndefined(); + expect(server.calls[0]?.url).toBe("https://10.0.0.1:443/api/v1/namespaces/libredb/secrets/present"); + }); + + test("creates a missing Secret with the job's labels, and replaces it with the version it read", async () => { + const server = apiServer(); + const kube = api(server.fetcher); + await kube.putSecret("store", { a: "1" }); + await kube.putSecret("store", { a: "2" }, "Opaque"); + const writes = server.calls.filter((call) => call.method !== "GET"); + expect(writes.map((call) => call.method)).toEqual(["POST", "PUT"]); + expect(writes[0]?.body?.metadata).toEqual({ name: "store", labels: { "app.kubernetes.io/part-of": "libredb" }, annotations: { ...KEEP_ANNOTATIONS } }); + expect(writes[1]?.body?.metadata.resourceVersion).toBe("101"); + expect(Buffer.from(server.objects.get("store")?.data.a as string, "base64").toString()).toBe("2"); + }); + + test("a replace after another writer changed the object fails instead of overwriting it", async () => { + const server = apiServer({ store: { data: {}, version: "1" } }); + const kube = api(server.fetcher); + await kube.getSecret("store"); + server.objects.set("store", { data: {}, version: "2" }); + await expect(kube.putSecret("store", { a: "1" })).rejects.toThrow(KubernetesApiError); + }); + + test("writes a ConfigMap's data as plain text", async () => { + const server = apiServer(); + await api(server.fetcher).putConfigMap("libredb-ca", { "ca.crt": "pem" }); + const write = server.calls.find((call) => call.method === "POST"); + expect(write?.url).toBe("https://10.0.0.1:443/api/v1/namespaces/libredb/configmaps"); + expect(write?.body?.data).toEqual({ "ca.crt": "pem" }); + }); + + test("a failed read names the status and the server's reason", async () => { + const fetcher = (async () => Response.json({ message: "forbidden by RBAC" }, { status: 403 })) as unknown as typeof fetch; + await expect(api(fetcher).getSecret("x")).rejects.toThrow("answered 403: forbidden by RBAC"); + }); + + test("inCluster reads the service account and the API server's address", () => { + const files: Record = { [`${SERVICE_ACCOUNT_DIR}/token`]: "tok\n", [`${SERVICE_ACCOUNT_DIR}/namespace`]: "libredb\n" }; + const kube = KubernetesApi.inCluster({ KUBERNETES_SERVICE_HOST: "fd00::1", KUBERNETES_SERVICE_PORT: "6443" }, (path) => files[path], {}); + expect(kube.namespace).toBe("libredb"); + expect(() => KubernetesApi.inCluster({}, () => undefined, {})).toThrow("not running in a Kubernetes pod"); + }); +}); + +describe("KubernetesApi beyond Secrets", () => { + test("a replace keeps the labels others gave the object, except Helm's tracking label", async () => { + const server = apiServer({ store: { data: {}, version: "3", labels: { team: "db", "app.kubernetes.io/instance": "libredb" } } }); + await api(server.fetcher).putSecret("store", { a: "1" }); + const put = server.calls.find((call) => call.method === "PUT"); + expect(put?.body?.metadata.labels).toEqual({ team: "db", "app.kubernetes.io/part-of": "libredb" }); + }); + + test("a Secret of another type in the way fails naming it instead of a bare 422", async () => { + const server = apiServer({ "libredb-tls-platform": { data: {}, version: "1", type: "Opaque" } }); + await expect(api(server.fetcher).putSecret("libredb-tls-platform", { a: "1" }, "kubernetes.io/tls")).rejects.toThrow( + "exists with type Opaque, not kubernetes.io/tls", + ); + }); + + test("reads a ConfigMap's data, and undefined for a missing one", async () => { + const fetcher = (async (input: string | URL | Request) => + String(input).endsWith("/present") ? Response.json({ metadata: { name: "present" }, data: { k: "v" } }) : Response.json({}, { status: 404 })) as unknown as typeof fetch; + expect(await api(fetcher).getConfigMap("present")).toEqual({ k: "v" }); + expect(await api(fetcher).getConfigMap("missing")).toBeUndefined(); + }); + + test("restarts a workload with a merge patch of its pod template, and reports one that does not exist", async () => { + const calls: { url: string; method?: string; type?: string | null; body?: string }[] = []; + const fetcher = (async (input: string | URL | Request, init?: RequestInit) => { + calls.push({ url: String(input), method: init?.method, type: new Headers(init?.headers).get("content-type"), body: String(init?.body) }); + return String(input).includes("/statefulsets/") ? Response.json({}, { status: 404 }) : Response.json({}); + }) as unknown as typeof fetch; + const at = new Date("2026-10-20T00:00:00Z"); + expect(await api(fetcher).restart({ kind: "deployments", name: "libredb-platform" }, at)).toBe("restarted"); + expect(await api(fetcher).restart({ kind: "statefulsets", name: "libredb-postgres" }, at)).toBe("absent"); + expect(calls[0]).toEqual({ + url: "https://10.0.0.1:443/apis/apps/v1/namespaces/libredb/deployments/libredb-platform", + method: "PATCH", + type: "application/merge-patch+json", + body: JSON.stringify({ spec: { template: { metadata: { annotations: { [RESTARTED_AT]: at.toISOString() } } } } }), + }); + const refused = (async () => Response.json({ message: "forbidden" }, { status: 403 })) as unknown as typeof fetch; + await expect(api(refused).restart({ kind: "deployments", name: "keycloak" }, at)).rejects.toThrow("answered 403: forbidden"); + }); +}); + +describe("KubernetesApi.podsCreated", () => { + test("lists the creation times of a workload's pods by its selector, and undefined for a missing workload", async () => { + const urls: string[] = []; + const fetcher = (async (input: string | URL | Request) => { + const url = String(input); + urls.push(url); + if (url.endsWith("/deployments/keycloak")) return Response.json({}, { status: 404 }); + if (url.includes("/deployments/")) return Response.json({ spec: { selector: { matchLabels: { app: "platform", tier: "web" } } } }); + return Response.json({ items: [{ metadata: { creationTimestamp: "2026-10-20T00:00:00Z" } }] }); + }) as unknown as typeof fetch; + const created = await api(fetcher).podsCreated({ kind: "deployments", name: "libredb-platform" }); + expect(created).toEqual([new Date("2026-10-20T00:00:00Z")]); + expect(urls[1]).toBe("https://10.0.0.1:443/api/v1/namespaces/libredb/pods?labelSelector=app%3Dplatform%2Ctier%3Dweb"); + expect(await api(fetcher).podsCreated({ kind: "deployments", name: "keycloak" })).toBeUndefined(); + }); + + test("a refused read names the status", async () => { + let calls = 0; + const fetcher = (async () => { + calls += 1; + return calls === 1 ? Response.json({ message: "no" }, { status: 403 }) : Response.json({}); + }) as unknown as typeof fetch; + await expect(api(fetcher).podsCreated({ kind: "deployments", name: "libredb-studio" })).rejects.toThrow("answered 403"); + const pods = (async (input: string | URL | Request) => + String(input).includes("/pods") ? Response.json({ message: "no pods" }, { status: 403 }) : Response.json({ spec: { selector: { matchLabels: { a: "b" } } } })) as unknown as typeof fetch; + await expect(api(pods).podsCreated({ kind: "deployments", name: "libredb-studio" })).rejects.toThrow("no pods"); + }); +}); diff --git a/installer/test/unit/kubernetes/bootstrap.test.ts b/installer/test/unit/kubernetes/bootstrap.test.ts new file mode 100644 index 0000000..52cb0ef --- /dev/null +++ b/installer/test/unit/kubernetes/bootstrap.test.ts @@ -0,0 +1,478 @@ +import { describe, expect, test } from "bun:test"; +import { PLATFORM_ENV_KEYS, PLATFORM_MIGRATE_ENV_KEYS } from "../../../src/contracts/platform"; +import { APP_SECRET_NAMES, POSTGRES_SUPERUSER_SECRET } from "../../../src/docker/secrets"; +import { CLIENT_SECRET_NAMES } from "../../../src/keycloak/desired"; +import { + addPendingRestarts, + caBundleOf, + checkRelease, + pendingRestartsOf, + runRestarts, + RELEASE_KEY, + sameData, + checkInstallation, + convergeStore, + convergeTls, + generatedNames, + keycloakEdgeCaddyfile, + loadState, + observedTls, + runBootstrap, + saveState, + rotateSealKeys, + SEAL_ENVELOPES_FILE, + sealKeysOf, + STATE_KEY, +} from "../../../src/kubernetes/bootstrap"; +import { INPUT_DIR, K8S_NAMES, STORE_KEYS, STUDIO_SECRET_KEYS, WORKLOADS } from "../../../src/kubernetes/names"; +import { Redactor } from "../../../src/log/redact"; +import { emptyState } from "../../../src/state/store"; +import { UnsafeChangeError } from "../../../src/state/immutable"; +import { createCa } from "../../../src/tls/ca"; +import { manifestFixture } from "../release/fixture"; +import { EXTERNAL_POSTGRES, files, installation, INSTALLER_KEYCLOAK, io, LICENSE, MANUAL_KEYCLOAK, MemoryObjects } from "./fixture"; + +const STUDIO_SECRET = "libredb-studio-env"; + +async function bootstrap(objects: MemoryObjects, config = installation(), readFile = files(), now?: Date) { + const run = io(objects, readFile, now); + await runBootstrap({ config, manifest: manifestFixture(), studioSecretName: STUDIO_SECRET }, run.io); + return run.lines; +} + +describe("the bootstrap job with a managed PostgreSQL and Keycloak", () => { + test("writes every Secret and ConfigMap the chart's workloads mount", async () => { + const objects = new MemoryObjects(); + const lines = await bootstrap(objects); + expect([...objects.secrets.keys()].sort()).toEqual( + [ + K8S_NAMES.secretStore, + K8S_NAMES.state, + K8S_NAMES.tls, + K8S_NAMES.platformEnv, + K8S_NAMES.migrateEnv, + STUDIO_SECRET, + K8S_NAMES.studioNetworkEnv, + K8S_NAMES.postgresSuperuser, + K8S_NAMES.postgresTls, + K8S_NAMES.postgresProvision, + K8S_NAMES.keycloakEnv, + K8S_NAMES.keycloakConfig, + K8S_NAMES.ingressTls("platform"), + K8S_NAMES.ingressTls("studio"), + K8S_NAMES.ingressTls("auth"), + ].sort(), + ); + expect([...objects.configMaps.keys()].sort()).toEqual( + [K8S_NAMES.caBundle, K8S_NAMES.postgresConfig, K8S_NAMES.keycloakEdge].sort(), + ); + expect(objects.secrets.get(K8S_NAMES.ingressTls("platform"))?.type).toBe("kubernetes.io/tls"); + expect(lines.at(-1)).toBe("bootstrap done"); + expect(lines.join("\n")).not.toContain(objects.secret(K8S_NAMES.secretStore)[APP_SECRET_NAMES.platformAuthSecret] as string); + }); + + test("Platform's environment holds exactly the installer's env set, never MIGRATE_DATABASE_URL", async () => { + const objects = new MemoryObjects(); + await bootstrap(objects); + const env = objects.secret(K8S_NAMES.platformEnv); + for (const key of Object.keys(env)) expect(PLATFORM_ENV_KEYS as readonly string[]).toContain(key); + expect(env.MIGRATE_DATABASE_URL).toBeUndefined(); + expect(env.PLATFORM_LICENSE).toBe(LICENSE); + expect(env.AUTH_URL).toBe("https://platform.libredb.test"); + expect(env.AUTH_KEYCLOAK_ISSUER).toBe("https://auth.libredb.test/realms/libredb"); + expect(env.KEYCLOAK_ADMIN_URL).toBe("http://keycloak:8080"); + expect(env.DATABASE_URL).toStartWith("postgresql://libredb_platform:"); + expect(env.DATABASE_URL).toContain("@libredb-postgres:5432/libredb?schema=platform&sslmode=verify-full"); + const migrate = objects.secret(K8S_NAMES.migrateEnv); + for (const key of Object.keys(migrate)) expect(PLATFORM_MIGRATE_ENV_KEYS as readonly string[]).toContain(key); + expect(migrate.MIGRATE_DATABASE_URL).toStartWith("postgresql://libredb_owner:"); + }); + + test("Studio's Secret holds the keys of the Studio chart, a keyring of the live seal key and the studio role's URL", async () => { + const objects = new MemoryObjects(); + await bootstrap(objects); + const studio = objects.secret(STUDIO_SECRET); + expect(Object.keys(studio).sort()).toEqual(Object.values(STUDIO_SECRET_KEYS).sort()); + expect(studio[STUDIO_SECRET_KEYS.oidcClientId]).toBe("libredb-studio"); + expect(studio[STUDIO_SECRET_KEYS.governancePostgresUrl]).toStartWith("postgresql://libredb_studio:"); + const keyring = JSON.parse(studio[STUDIO_SECRET_KEYS.governancePrivateKeys] as string) as { version: number; keys: string[] }; + const seal = JSON.parse(objects.secret(K8S_NAMES.secretStore)[STORE_KEYS.sealKeys] as string) as { privateKey: string; publicKey: string }[]; + expect(keyring).toEqual({ version: 1, keys: [seal[0]?.privateKey as string] }); + // Platform holds the public half only. + expect(objects.secret(K8S_NAMES.platformEnv).BRIDGE_SEAL_PUBLIC_KEY).toBe(seal[0]?.publicKey as string); + expect(JSON.stringify(objects.secret(K8S_NAMES.platformEnv))).not.toContain(seal[0]?.privateKey as string); + }); + + test("the managed server gets the installer's pg_hba.conf, a leaf for its Service names and a script with verifiers only", async () => { + const objects = new MemoryObjects(); + await bootstrap(objects); + expect(objects.configMaps.get(K8S_NAMES.postgresConfig)?.["pg_hba.conf"]).toContain("hostssl all all 0.0.0.0/0 scram-sha-256"); + const store = objects.secret(K8S_NAMES.secretStore); + const script = objects.secret(K8S_NAMES.postgresProvision)["provision.sql"] as string; + expect(script).toContain("SCRAM-SHA-256$"); + expect(script).not.toContain(store["postgres-platform-password"] as string); + expect(objects.secret(K8S_NAMES.postgresSuperuser).POSTGRES_PASSWORD).toBe(store[POSTGRES_SUPERUSER_SECRET] as string); + expect(objects.secret(K8S_NAMES.postgresTls)["server.crt"]).toContain("BEGIN CERTIFICATE"); + }); + + test("the managed Keycloak gets its database password, the bootstrap client and the installer's auth edge", async () => { + const objects = new MemoryObjects(); + await bootstrap(objects); + const store = objects.secret(K8S_NAMES.secretStore); + expect(objects.secret(K8S_NAMES.keycloakEnv)).toEqual({ + KC_DB_PASSWORD: store["postgres-keycloak-password"] as string, + KC_BOOTSTRAP_ADMIN_CLIENT_ID: "libredb-bootstrap", + KC_BOOTSTRAP_ADMIN_CLIENT_SECRET: store[STORE_KEYS.keycloakBootstrapSecret] as string, + }); + const config = objects.secret(K8S_NAMES.keycloakConfig); + expect(Object.keys(config).sort()).toEqual( + [CLIENT_SECRET_NAMES.studio, CLIENT_SECRET_NAMES.platform, STORE_KEYS.keycloakBootstrapSecret, STORE_KEYS.firstAdminPassword, STORE_KEYS.userManagerPassword].sort(), + ); + const caddyfile = objects.configMaps.get(K8S_NAMES.keycloakEdge)?.Caddyfile as string; + expect(caddyfile).toContain("http://auth.libredb.test:8081 {"); + expect(caddyfile).toContain("@keycloak path /realms/libredb/* /resources/*"); + expect(caddyfile).not.toContain("platform.libredb.test"); + }); + + test("a second run keeps every generated secret, the seal keys and the internal CA, and rewrites only derived objects", async () => { + const objects = new MemoryObjects(); + await bootstrap(objects); + const store = objects.secret(K8S_NAMES.secretStore); + const tls = objects.secret(K8S_NAMES.tls); + objects.writes.length = 0; + const lines = await bootstrap(objects); + expect(objects.secret(K8S_NAMES.secretStore)).toEqual(store); + expect(objects.secret(K8S_NAMES.tls)).toEqual(tls); + expect(objects.writes).not.toContain(`secret/${K8S_NAMES.secretStore}`); + expect(objects.writes).not.toContain(`secret/${K8S_NAMES.tls}`); + expect(lines.join("\n")).not.toContain("secrets: stored"); + }); + + test("refuses a run that changes an immutable field, such as postgres.mode", async () => { + const objects = new MemoryObjects(); + await bootstrap(objects); + const external = installation(EXTERNAL_POSTGRES); + await expect(bootstrap(objects, external, files({ [`${INPUT_DIR}/ca/postgres-ca.crt`]: (await createCa({ commonName: "corp", now: new Date() })).certPem }))).rejects.toThrow(UnsafeChangeError); + }); + + test("an AUTH_SECRET rotation passes the previous secret to Platform", async () => { + const objects = new MemoryObjects(); + const previousFile = `${INPUT_DIR}/auth-secret-previous`; + const config = installation({ platform: { organizationName: "Acme", licenseFile: `${INPUT_DIR}/license/license`, authSecretPreviousFile: previousFile } }); + await bootstrap(objects, config, files({ [previousFile]: "the-previous-auth-secret-of-32-characters" })); + expect(objects.secret(K8S_NAMES.platformEnv).AUTH_SECRET_PREVIOUS).toBe("the-previous-auth-secret-of-32-characters"); + }); + + test("refuses a missing license file", async () => { + const objects = new MemoryObjects(); + await expect(bootstrap(objects, installation(), () => undefined)).rejects.toThrow("platform.licenseFile"); + }); +}); + +describe("the bootstrap job with an external PostgreSQL and the site's Keycloak", () => { + test("writes the DBA's provisioning script, puts the server's CA first in the bundle and takes the site's client secrets", async () => { + const objects = new MemoryObjects(); + const postgresCa = await createCa({ commonName: "Corp PostgreSQL CA", now: new Date() }); + const config = installation({ ...EXTERNAL_POSTGRES, ...MANUAL_KEYCLOAK }); + await bootstrap( + objects, + config, + files({ + [`${INPUT_DIR}/ca/postgres-ca.crt`]: postgresCa.certPem, + [`${INPUT_DIR}/keycloak-clients/studio-client-secret`]: "corp-studio-secret\n", + [`${INPUT_DIR}/keycloak-clients/platform-client-secret`]: "corp-platform-secret\n", + }), + ); + expect(objects.secrets.has(K8S_NAMES.postgresProvision)).toBe(false); + expect(objects.secrets.has(K8S_NAMES.postgresSuperuser)).toBe(false); + expect(objects.secrets.has(K8S_NAMES.keycloakEnv)).toBe(false); + expect(objects.configMaps.has(K8S_NAMES.keycloakEdge)).toBe(false); + expect(objects.secrets.has(K8S_NAMES.ingressTls("auth"))).toBe(false); + const script = objects.secret(K8S_NAMES.postgresDbaScript)["provision.sql"] as string; + expect(script).toContain("pg.corp.example:5432"); + expect(script).not.toContain("libredb_keycloak"); + const bundle = objects.configMaps.get(K8S_NAMES.caBundle)?.["ca.crt"] as string; + // Prisma trusts only the first certificate of sslcert, so the external server's CA leads the bundle. + expect(bundle.split("\n")[1]).toBe(postgresCa.certPem.split("\n")[1] as string); + expect(objects.secret(STUDIO_SECRET)[STUDIO_SECRET_KEYS.oidcClientSecret]).toBe("corp-studio-secret"); + expect(objects.secret(K8S_NAMES.platformEnv).AUTH_KEYCLOAK_SECRET).toBe("corp-platform-secret"); + expect(objects.secret(K8S_NAMES.platformEnv).DATABASE_URL).toContain("@pg.corp.example:5432/"); + }); + + test("a changed site client secret replaces the stored one on the next run", async () => { + const objects = new MemoryObjects(); + const config = installation(MANUAL_KEYCLOAK); + const secrets = (studio: string) => + files({ + [`${INPUT_DIR}/keycloak-clients/studio-client-secret`]: studio, + [`${INPUT_DIR}/keycloak-clients/platform-client-secret`]: "corp-platform-secret", + }); + await bootstrap(objects, config, secrets("first")); + await bootstrap(objects, config, secrets("second")); + expect(objects.secret(K8S_NAMES.secretStore)[CLIENT_SECRET_NAMES.studio]).toBe("second"); + }); + + test("refuses a site client secret that is not mounted", async () => { + await expect(bootstrap(new MemoryObjects(), installation(MANUAL_KEYCLOAK))).rejects.toThrow("keycloak.external.studioClientSecret"); + }); + + test("refuses an external server with provisioning installer, which would bring an admin credential into the cluster", async () => { + const config = installation({ + postgres: { mode: "external", external: { host: "pg.corp.example", adminUser: "postgres", adminPasswordEnv: "PG_ADMIN" } }, + }); + await expect(bootstrap(new MemoryObjects(), config)).rejects.toMatchObject({ exitCode: 2, message: expect.stringContaining("provisioning manual") }); + }); + + test("an external Keycloak with provisioning installer gets generated client secrets and no accounts", async () => { + const objects = new MemoryObjects(); + await bootstrap(objects, installation(INSTALLER_KEYCLOAK)); + const config = objects.secret(K8S_NAMES.keycloakConfig); + expect(Object.keys(config).sort()).toEqual([CLIENT_SECRET_NAMES.studio, CLIENT_SECRET_NAMES.platform].sort()); + expect(objects.secret(K8S_NAMES.platformEnv).KEYCLOAK_ADMIN_URL).toBe("https://kc-admin.corp.example"); + }); +}); + +describe("the bootstrap job's parts", () => { + test("generatedNames adds the chart's own secrets to the installer's set", () => { + expect(generatedNames(installation())).toEqual( + expect.arrayContaining([STORE_KEYS.keycloakBootstrapSecret, STORE_KEYS.firstAdminPassword, STORE_KEYS.userManagerPassword]), + ); + const site = generatedNames(installation(MANUAL_KEYCLOAK)); + expect(site).not.toContain(STORE_KEYS.keycloakBootstrapSecret); + expect(site).not.toContain(STORE_KEYS.firstAdminPassword); + expect(site).not.toContain(CLIENT_SECRET_NAMES.studio); + }); + + test("sealKeysOf issues the first key once and refuses an emptied store", () => { + const first = sealKeysOf(undefined); + expect(first.issued).toBe(true); + expect(first.keys).toHaveLength(1); + expect(sealKeysOf(JSON.stringify(first.keys))).toEqual({ keys: first.keys, issued: false }); + expect(() => sealKeysOf("[]")).toThrow("holds no seal key"); + expect(() => sealKeysOf("{}")).toThrow("holds no seal key"); + }); + + test("convergeStore keeps a stored value and registers every value with the redactor", () => { + const redactor = new Redactor(); + const stored = { [APP_SECRET_NAMES.platformAuthSecret]: "kept-auth-secret" }; + const { store, changed } = convergeStore(installation(), stored, { readFile: files(), env: {}, redactor }); + expect(store[APP_SECRET_NAMES.platformAuthSecret]).toBe("kept-auth-secret"); + expect(changed).not.toContain(APP_SECRET_NAMES.platformAuthSecret); + expect(redactor.redact("kept-auth-secret")).toBe("[REDACTED]"); + }); + + test("convergeTls re-issues a leaf close to its expiry and keeps the CA", async () => { + const config = installation(); + const issued = await convergeTls(config, {}, new Date("2026-10-20T00:00:00Z"), () => {}); + const lines: string[] = []; + const later = await convergeTls(config, issued.data, new Date("2029-01-05T00:00:00Z"), (line) => lines.push(line)); + expect(later.changed).toBe(true); + expect(later.data["ca.crt"]).toBe(issued.data["ca.crt"] as string); + expect(later.data["postgres.crt"]).not.toBe(issued.data["postgres.crt"] as string); + expect(lines.join("\n")).toContain("days left"); + const kept = await convergeTls(config, issued.data, new Date("2026-10-21T00:00:00Z"), () => {}); + expect(kept.changed).toBe(false); + }); + + test("observedTls reads only the keys that exist", () => { + expect(observedTls({ "ca.crt": "c" }, ["postgres"])).toEqual({ ca: { certPem: "c" }, leaves: { postgres: {} } }); + }); + + test("caBundleOf refuses a configured CA file that is not mounted", async () => { + const ca = await createCa({ commonName: "internal", now: new Date() }); + expect(() => caBundleOf(installation(EXTERNAL_POSTGRES), ca.certPem, () => undefined)).toThrow("does not exist"); + }); + + test("the edge Caddyfile serves the auth site alone behind the ingress", () => { + const caddyfile = keycloakEdgeCaddyfile(installation(), manifestFixture()); + expect(caddyfile).toContain("trusted_proxies static private_ranges"); + expect(caddyfile).toContain("respond 400"); + }); + + test("the state round-trips through its Secret, and the first run records the immutable fields", async () => { + const objects = new MemoryObjects(); + expect(await loadState(objects)).toEqual(emptyState()); + const state = checkInstallation(emptyState(), installation()); + expect(state.installation).toBeDefined(); + await saveState(objects, state); + expect(objects.secret(K8S_NAMES.state)[STATE_KEY]).toEndWith("\n"); + expect(await loadState(objects)).toEqual(state); + }); +}); + +describe("the bootstrap job across upgrades", () => { + const SECRETS_MODE = { tls: { mode: "external", caBundleFile: `${INPUT_DIR}/ca/tls-ca.crt`, external: { trustedProxies: ["private_ranges"] } } }; + + test("asks to restart only the workloads whose inputs changed, and nothing on a run that changes nothing", async () => { + const objects = new MemoryObjects(); + await bootstrap(objects); + // A first install asks for every reader; the restart job then finds their pods started after the change. + expect(pendingRestartsOf(objects.secret(K8S_NAMES.state))?.workloads).toHaveLength(4); + await runRestarts(objects, "1.0.0", () => new Date(), () => {}); + expect(pendingRestartsOf(objects.secret(K8S_NAMES.state))).toBeUndefined(); + const lines = await bootstrap(objects); + expect(pendingRestartsOf(objects.secret(K8S_NAMES.state))).toBeUndefined(); + expect(lines.join("\n")).not.toContain("asked to restart"); + const renewed = "ldl1.fedcba9876543210.eyJpZCI6IlJFTkVXRUQifQ.c2lnbmF0dXJl"; + await bootstrap(objects, installation(), files({ [`${INPUT_DIR}/license/license`]: renewed })); + expect(pendingRestartsOf(objects.secret(K8S_NAMES.state))?.workloads).toEqual([WORKLOADS.platform]); + }); + + test("a renewed PostgreSQL leaf asks to restart the server", async () => { + const objects = new MemoryObjects(); + await bootstrap(objects); + await runRestarts(objects, "1.0.0", () => new Date(), () => {}); + await bootstrap(objects, installation(), files(), new Date("2029-01-05T00:00:00Z")); + expect(pendingRestartsOf(objects.secret(K8S_NAMES.state))?.workloads).toEqual([WORKLOADS.postgres]); + }); + + test("the restart job restarts old pods, skips pods started after the change and absent workloads, then clears the request", async () => { + const objects = new MemoryObjects(); + await bootstrap(objects, installation(), files(), new Date("2026-10-20T00:00:00Z")); + objects.workloads.add("deployments/libredb-platform"); + objects.workloads.add("deployments/libredb-studio"); + objects.pods.set("deployments/libredb-studio", [new Date("2026-10-20T00:05:00Z")]); + const lines: string[] = []; + await runRestarts(objects, "1.0.0", () => new Date("2026-10-20T00:10:00Z"), (line) => lines.push(line)); + expect(objects.restarts).toEqual(["deployments/libredb-platform"]); + expect(lines).toContain("deployments/libredb-studio started after its inputs changed"); + expect(lines).toContain("deployments/keycloak does not exist"); + expect(pendingRestartsOf(objects.secret(K8S_NAMES.state))).toBeUndefined(); + expect(objects.secret(K8S_NAMES.state)[RELEASE_KEY]).toBeDefined(); + const again: string[] = []; + await runRestarts(objects, "1.0.0", () => new Date(), (line) => again.push(line)); + expect(again).toEqual(["no restart asked for", "the installation runs LibreDB 1.0.0"]); + }); + + test("a restart request survives a run that fails after it, and a later run adds to it with the newer time", async () => { + const objects = new MemoryObjects(); + await bootstrap(objects); + await runRestarts(objects, "1.0.0", () => new Date(), () => {}); + const renewed = (id: string) => `ldl1.fedcba9876543210.${Buffer.from(JSON.stringify({ id })).toString("base64url")}.c2lnbmF0dXJl`; + await bootstrap(objects, installation(), files({ [`${INPUT_DIR}/license/license`]: renewed("A") }), new Date("2026-10-21T00:00:00Z")); + const failing = Object.assign(Object.create(Object.getPrototypeOf(objects)), objects) as MemoryObjects; + failing.restart = async () => { + throw new Error("the API server is down"); + }; + objects.workloads.add("deployments/libredb-platform"); + await expect(runRestarts(failing, "1.0.0", () => new Date(), () => {})).rejects.toThrow("down"); + expect(pendingRestartsOf(objects.secret(K8S_NAMES.state))?.workloads).toEqual([WORKLOADS.platform]); + await bootstrap(objects, installation(INSTALLER_KEYCLOAK), files({ [`${INPUT_DIR}/license/license`]: renewed("A") }), new Date("2026-10-22T00:00:00Z")).catch(() => {}); + expect(addPendingRestarts(pendingRestartsOf(objects.secret(K8S_NAMES.state)), [WORKLOADS.studio], new Date("2026-10-23T00:00:00Z"))).toEqual({ + since: "2026-10-23T00:00:00.000Z", + workloads: [WORKLOADS.platform, WORKLOADS.studio], + }); + }); + + test("a switch to tls.mode secrets writes no ingress Secret any more, though the store keeps the old leaves", async () => { + const objects = new MemoryObjects(); + await bootstrap(objects); + for (const name of [K8S_NAMES.ingressTls("platform"), K8S_NAMES.ingressTls("studio"), K8S_NAMES.ingressTls("auth")]) objects.secrets.delete(name); + const ca = await createCa({ commonName: "site", now: new Date() }); + await bootstrap(objects, installation(SECRETS_MODE), files({ [`${INPUT_DIR}/ca/tls-ca.crt`]: ca.certPem })); + expect(objects.secrets.has(K8S_NAMES.ingressTls("platform"))).toBe(false); + expect(objects.secret(K8S_NAMES.tls)["platform.crt"]).toBeDefined(); + }); + + test("Studio's network environment holds the installer's NO_PROXY, and the proxy when one is set", async () => { + const objects = new MemoryObjects(); + await bootstrap(objects, installation({ network: { httpsProxy: "http://proxy.corp:3128" } })); + const env = objects.secret(K8S_NAMES.studioNetworkEnv); + expect(env.HTTPS_PROXY).toBe("http://proxy.corp:3128"); + expect(env.NODE_USE_ENV_PROXY).toBe("1"); + expect(env.NO_PROXY).toContain("libredb-postgres"); + expect(env.NO_PROXY).toBe(objects.secret(K8S_NAMES.platformEnv).NO_PROXY as string); + }); + + test("the release is recorded once the restart job ran, kept by later state saves, and an older one is refused", async () => { + const objects = new MemoryObjects(); + await bootstrap(objects); + // A failed upgrade, whose restart job never ran, records nothing, so its --atomic rollback is not refused. + expect(objects.secret(K8S_NAMES.state)[RELEASE_KEY]).toBeUndefined(); + await runRestarts(objects, manifestFixture().version, () => new Date(), () => {}); + expect(objects.secret(K8S_NAMES.state)[RELEASE_KEY]).toBe(manifestFixture().version); + await saveState(objects, emptyState()); + expect(objects.secret(K8S_NAMES.state)[RELEASE_KEY]).toBe(manifestFixture().version); + await expect(checkRelease(objects, "0.9.0")).rejects.toThrow("a downgrade is not supported"); + await expect(checkRelease(objects, "0.9.0")).rejects.toMatchObject({ name: "RefusedError", exitCode: 6 }); + await checkRelease(objects, "9.0.0"); + }); + + test("sameData compares keys and values", () => { + expect(sameData({ a: "1" }, { a: "1" })).toBe(true); + expect(sameData({ a: "1" }, { a: "2" })).toBe(false); + expect(sameData({ a: "1" }, { a: "1", b: "2" })).toBe(false); + }); +}); + +describe("the bootstrap job's seal key rotation", () => { + type StoredKey = { kid: string; publicKey: string; state: string }; + const storedKeys = (objects: MemoryObjects) => JSON.parse(objects.secret(K8S_NAMES.secretStore)[STORE_KEYS.sealKeys] as string) as StoredKey[]; + const keyringSize = (objects: MemoryObjects) => + (JSON.parse(objects.secret(STUDIO_SECRET)[STUDIO_SECRET_KEYS.governancePrivateKeys] as string) as { keys: string[] }).keys.length; + const sealing = (sealKeys: Record) => installation({ bridge: { sealKeys } }); + const uses = (rows: unknown[]) => files({ [SEAL_ENVELOPES_FILE]: `libredb-result envelope-uses ${JSON.stringify(rows)}\n` }); + + test("walks the runbook: issue a key, switch Platform to it, retire the old one, each step once", async () => { + const objects = new MemoryObjects(); + await bootstrap(objects); + const [old] = storedKeys(objects) as [StoredKey]; + + const issued = await bootstrap(objects, sealing({ issueNew: true })); + const [, fresh] = storedKeys(objects) as [StoredKey, StoredKey]; + expect(issued).toContain(`seal keys: issued seal key ${fresh.kid} into the Studio keyring`); + expect(keyringSize(objects)).toBe(2); + // Platform keeps sealing to the oldest live key until activeKid names the new one. + expect(objects.secret(K8S_NAMES.platformEnv).BRIDGE_SEAL_PUBLIC_KEY).toBe(old.publicKey); + // The same values again issue no second key. + await bootstrap(objects, sealing({ issueNew: true })); + expect(storedKeys(objects)).toHaveLength(2); + + await bootstrap(objects, sealing({ issueNew: false, activeKid: fresh.kid })); + expect(objects.secret(K8S_NAMES.platformEnv).BRIDGE_SEAL_PUBLIC_KEY).toBe(fresh.publicKey); + expect(pendingRestartsOf(objects.secret(K8S_NAMES.state))?.workloads).toContainEqual(WORKLOADS.platform); + + const retired = await bootstrap(objects, sealing({ issueNew: false, activeKid: fresh.kid, retire: [old.kid] }), uses([])); + expect(retired).toContain(`seal keys: retired seal key ${old.kid}: out of the Studio keyring, kept in the secret store`); + expect(storedKeys(objects).map((key) => [key.kid, key.state])).toEqual([ + [old.kid, "retired"], + [fresh.kid, "live"], + ]); + expect(keyringSize(objects)).toBe(1); + // Retiring an already retired kid needs neither the envelopes nor a change. + const again = await bootstrap(objects, sealing({ issueNew: false, activeKid: fresh.kid, retire: [old.kid] }), files()); + expect(again.join("\n")).not.toContain("seal keys:"); + }); + + test("refuses with exit 6 a retirement while a connection still uses the kid, and changes nothing", async () => { + const objects = new MemoryObjects(); + await bootstrap(objects); + await bootstrap(objects, sealing({ issueNew: true })); + const [old, fresh] = storedKeys(objects) as [StoredKey, StoredKey]; + const before = objects.secret(K8S_NAMES.secretStore); + const config = sealing({ activeKid: fresh.kid, retire: [old.kid] }); + const run = bootstrap(objects, config, uses([{ kid: old.kid, connectionId: "c1", name: "orders" }])); + await expect(run).rejects.toMatchObject({ exitCode: 6 }); + await expect(bootstrap(objects, config, uses([{ kid: old.kid, connectionId: "c1", name: "orders" }]))).rejects.toThrow( + `kid ${old.kid} still seals credentials of 1 connection in bridge.connections_v1: orders (c1)`, + ); + expect(objects.secret(K8S_NAMES.secretStore)).toEqual(before); + }); + + test("refuses to retire the active kid, and needs the init container's output for a live kid", async () => { + const objects = new MemoryObjects(); + await bootstrap(objects); + const [only] = storedKeys(objects) as [StoredKey]; + await expect(bootstrap(objects, sealing({ retire: [only.kid] }), uses([]))).rejects.toMatchObject({ exitCode: 6 }); + await bootstrap(objects, sealing({ issueNew: true })); + const [, fresh] = storedKeys(objects) as [StoredKey, StoredKey]; + await expect(bootstrap(objects, sealing({ activeKid: fresh.kid, retire: [only.kid] }))).rejects.toThrow( + `retiring seal key ${only.kid} needs the connections each still seals`, + ); + }); + + test("rotateSealKeys leaves the keys alone without bridge.sealKeys", () => { + const keys = sealKeysOf(undefined).keys; + expect(rotateSealKeys(keys, installation(), files())).toEqual({ keys, done: [] }); + }); +}); diff --git a/installer/test/unit/kubernetes/bundle.test.ts b/installer/test/unit/kubernetes/bundle.test.ts new file mode 100644 index 0000000..d939c12 --- /dev/null +++ b/installer/test/unit/kubernetes/bundle.test.ts @@ -0,0 +1,8 @@ +import { describe, expect, test } from "bun:test"; +import { buildHelmJobs, committedBundle } from "../../../scripts/build-helm-jobs"; + +describe("the Helm chart's job bundle", () => { + test("is the bundle of the installer's current code: run `bun run build:helm-jobs` after changing what the jobs import", () => { + expect(committedBundle() === buildHelmJobs()).toBe(true); + }, 60_000); +}); diff --git a/installer/test/unit/kubernetes/chart.test.ts b/installer/test/unit/kubernetes/chart.test.ts new file mode 100644 index 0000000..00aead2 --- /dev/null +++ b/installer/test/unit/kubernetes/chart.test.ts @@ -0,0 +1,496 @@ +// The Helm chart charts/libredb-platform against the installer it drives, through the real `helm template` output: +// every render guard, the installation document the jobs validate with the installer's schema, the image and Studio +// chart pins of the release manifest, the environments of Studio and Keycloak against the installer's renderers, the +// bootstrap Role against every object the bootstrap job writes, and the pod security of every workload. +import { beforeAll, describe, expect, test } from "bun:test"; +import { execFileSync } from "node:child_process"; +import { copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { parse, parseAllDocuments } from "yaml"; +import { parseInstallation } from "../../../src/config/schema"; +import { runBootstrap } from "../../../src/kubernetes/bootstrap"; +import { computeDesired } from "../../../src/model/desired"; +import { renderStudioEnv } from "../../../src/model/studio-env"; +import { parseReleaseManifest } from "../../../src/release/manifest"; +import { composeTrust, renderCompose } from "../../../src/render/compose"; +import manifestDocument from "../../../release/release.config.json" with { type: "json" }; +import { files, MemoryObjects } from "./fixture"; +import { Redactor } from "../../../src/log/redact"; +import { INPUT_DIR, WORKLOADS } from "../../../src/kubernetes/names"; +import { deleteCanaryScript, envelopeUsesScript } from "../../../src/secrets/seal-rotation"; +import { SEAL_ENVELOPES_FILE } from "../../../src/kubernetes/bootstrap"; +import { createCa } from "../../../src/tls/ca"; + +const CHART = join(import.meta.dir, "..", "..", "..", "..", "charts", "libredb-platform"); +const LICENSE = "ldl1.0123456789abcdef.eyJpZCI6IlRFU1QifQ.c2lnbmF0dXJl"; +const manifest = parseReleaseManifest(manifestDocument); + +type Doc = { kind: string; metadata: { name: string; annotations?: Record }; spec?: any; data?: Record; rules?: any[] }; + +const BASE = [ + "hosts.platform=platform.libredb.test", + "hosts.studio=studio.libredb.test", + "hosts.auth=auth.libredb.test", + "platform.organizationName=Acme", + `platform.license.value=${LICENSE}`, + "studio.config.oidcIssuer=https://auth.libredb.test/realms/libredb", + "studio.ingress.hosts[0].host=studio.libredb.test", + "studio.ingress.hosts[0].paths[0].path=/", + "studio.ingress.hosts[0].paths[0].pathType=Prefix", + "studio.ingress.tls[0].secretName=libredb-tls-studio", + "studio.ingress.tls[0].hosts[0]=studio.libredb.test", +]; + +function helm(sets: readonly string[], args: readonly string[] = []): string { + return execFileSync("helm", ["template", "libredb", CHART, "-n", "libredb", ...sets.flatMap((set) => ["--set", set]), ...args], { + encoding: "utf8", + maxBuffer: 64 * 1024 * 1024, + stdio: ["ignore", "pipe", "pipe"], + }); +} + +function render(sets: readonly string[] = [], args: readonly string[] = []): Doc[] { + return parseAllDocuments(helm([...BASE, ...sets], args)) + .map((doc) => doc.toJS() as Doc | null) + .filter((doc): doc is Doc => doc !== null); +} + +function failure(sets: readonly string[]): string { + try { + helm([...BASE, ...sets]); + } catch (error) { + return String((error as { stderr?: string }).stderr); + } + throw new Error("the render succeeded"); +} + +function find(docs: Doc[], kind: string, name: string): Doc { + const doc = docs.find((candidate) => candidate.kind === kind && candidate.metadata.name === name); + if (doc === undefined) throw new Error(`no ${kind} ${name}`); + return doc; +} + +function installationOf(docs: Doc[]) { + return JSON.parse(find(docs, "ConfigMap", "libredb-bootstrap-chart").data?.["installation.json"] as string); +} + +beforeAll(() => { + // The Studio chart dependency: the one `bun run embed` downloaded and checked against the manifest's sha256 when it + // exists, as in CI, or else Helm's build from Chart.lock. + const archive = `libredb-studio-${manifest.studioChart.version}.tgz`; + if (existsSync(join(CHART, "charts", archive))) return; + const embedded = join(import.meta.dir, "..", "..", "..", "generated", archive); + if (existsSync(embedded)) { + mkdirSync(join(CHART, "charts"), { recursive: true }); + copyFileSync(embedded, join(CHART, "charts", archive)); + } else { + execFileSync("helm", ["dependency", "build", CHART], { stdio: "ignore" }); + } +}); + +describe("the chart's pins", () => { + test("its images are the release manifest's, by digest", () => { + const values = parse(readFileSync(join(CHART, "values.yaml"), "utf8")); + const pinned = (entry: { repository: string; version: string; digest: string }) => ({ repository: entry.repository, tag: `${entry.version}@${entry.digest}` }); + const pick = (image: { repository: string; tag: string }) => ({ repository: image.repository, tag: image.tag }); + expect(pick(values.platform.image)).toEqual(pinned(manifest.images.platform)); + expect(pick(values.studio.image)).toEqual(pinned(manifest.images.studio)); + expect(pick(values.keycloak.managed.image)).toEqual(pinned(manifest.images.keycloak)); + expect(pick(values.postgresql.managed.image)).toEqual(pinned(manifest.images.postgres)); + expect(pick(values.keycloak.managed.edge.image)).toEqual(pinned(manifest.images.edge)); + }); + + test("its appVersion is the release, and its Studio dependency the manifest's chart with its sha256", () => { + const chart = parse(readFileSync(join(CHART, "Chart.yaml"), "utf8")); + expect(chart.appVersion).toBe(manifest.version); + const studio = chart.dependencies.find((dependency: { name: string }) => dependency.name === "libredb-studio"); + expect({ version: studio.version, repository: studio.repository }).toEqual({ + version: manifest.studioChart.version, + repository: manifest.studioChart.repository, + }); + const archive = readFileSync(join(CHART, "charts", `libredb-studio-${manifest.studioChart.version}.tgz`)); + expect(new Bun.CryptoHasher("sha256").update(archive).digest("hex")).toBe(manifest.studioChart.sha256); + }); +}); + +describe("the chart's render guards", () => { + test.each([ + [["hosts.auth="], "hosts.auth is required"], + [["platform.organizationName="], "platform.organizationName is required"], + [["platform.license.value="], "platform.license is required"], + [["platform.license.value=not-a-token"], "must hold the license token alone"], + [["tls.mode=letsencrypt"], "tls.mode must be generated or secrets"], + [["tls.mode=secrets"], "tls.mode secrets needs tls.secrets.platform"], + [["tls.mode=secrets", "tls.secrets.platform=a", "tls.secrets.studio=b", "tls.secrets.auth=c"], "tls.mode secrets needs tls.caBundle"], + [["tls.mode=secrets", "tls.secrets.platform=a", "tls.secrets.studio=b"], "needs tls.secrets.auth"], + [["postgresql.mode=cloud"], "postgresql.mode must be managed or external"], + [["postgresql.mode=external"], "postgresql.mode external needs postgresql.external.host"], + [["keycloak.mode=cloud"], "keycloak.mode must be managed or external"], + [["keycloak.mode=external"], "keycloak.mode external needs keycloak.external.baseUrl"], + [["keycloak.mode=external", "keycloak.external.baseUrl=https://auth.libredb.test", "keycloak.external.provisioning=auto"], "must be installer or manual"], + [["keycloak.mode=external", "keycloak.external.baseUrl=https://auth.libredb.test"], "needs keycloak.external.adminSecret"], + [["keycloak.mode=external", "keycloak.external.baseUrl=https://auth.libredb.test", "keycloak.external.adminSecret=kc"], "needs exactly one of keycloak.external.adminClientId and keycloak.external.adminUser"], + [["keycloak.mode=external", "keycloak.external.baseUrl=https://auth.libredb.test", "keycloak.external.adminSecret=kc", "keycloak.external.adminUser=a", "keycloak.external.adminClientId=c"], "needs exactly one of"], + [["network.httpsProxy=http://user:secret@proxy.corp:3128"], "network.httpsProxy must not hold credentials"], + [["network.httpsProxy=HTTP://user:secret@proxy.corp:3128"], "network.httpsProxy must not hold credentials"], + [["studio.fullnameOverride=studio"], "studio.fullnameOverride must stay libredb-studio"], + [["studio.ingress.tls[0].secretName=other"], "studio.ingress.tls must hold one entry with secretName libredb-tls-studio and hosts [studio.libredb.test]"], + [["studio.ingress.tls[0].hosts[0]=other.libredb.test"], "studio.ingress.tls must hold one entry"], + [["tls.mode=secrets", "tls.secrets.platform=a", "tls.secrets.studio=b", "tls.secrets.auth=c", "tls.caBundle=PEM"], "studio.ingress.tls must hold one entry with secretName b"], + [["keycloak.mode=external", "keycloak.external.baseUrl=https://auth.libredb.test", "keycloak.external.provisioning=manual"], "needs keycloak.external.clientSecretsSecret"], + [["studio.config.oidcIssuer=https://other/realms/libredb"], "studio.config.oidcIssuer must be https://auth.libredb.test/realms/libredb"], + [["studio.secrets.existingSecret=mine"], "studio.secrets.existingSecret must stay libredb-studio-env"], + [["studio.ingress.hosts[0].host=other.libredb.test"], "studio.ingress.hosts must hold exactly one host"], + ])("%j fails naming the value", (sets, message) => { + expect(failure(sets as string[])).toContain(message as string); + }); +}); + +describe("the installation document", () => { + test.each([ + ["managed PostgreSQL and Keycloak", []], + ["an external PostgreSQL with its CA", ["postgresql.mode=external", "postgresql.external.host=pg.corp.example", "postgresql.external.caBundle=PEM"]], + [ + "the site's Keycloak, provisioning manual", + ["keycloak.mode=external", "keycloak.external.baseUrl=https://auth.libredb.test", "keycloak.external.provisioning=manual", "keycloak.external.clientSecretsSecret=kc"], + ], + [ + "the site's Keycloak, provisioning installer with an admin user", + ["keycloak.mode=external", "keycloak.external.baseUrl=https://auth.libredb.test", "keycloak.external.adminSecret=kc", "keycloak.external.adminUser=admin", "keycloak.external.caBundle=PEM"], + ], + [ + "TLS Secrets of the site", + ["tls.mode=secrets", "tls.secrets.platform=a", "tls.secrets.studio=b", "tls.secrets.auth=c", "tls.caBundle=PEM", "studio.ingress.tls[0].secretName=b", "platform.queryLogRetentionDays=30", "network.httpsProxy=http://proxy.corp:3128"], + ], + ])("with %s passes the installer's schema", (_name, sets) => { + const document = installationOf(render(sets as string[])); + expect(() => parseInstallation(document)).not.toThrow(); + expect(document.mode).toBe("kubernetes"); + expect(JSON.stringify(document)).not.toContain(LICENSE); + }); + + test("the first accounts follow keycloak.firstAdmin.enabled and keycloak.userManager.enabled when set", () => { + const document = installationOf(render(["keycloak.firstAdmin.enabled=false", "keycloak.userManager.enabled=true"])); + expect(parseInstallation(document).keycloak.firstAdmin.enabled).toBe(false); + }); +}); + +describe("the seal key and AUTH_SECRET rotations", () => { + const OLD = "0123456789abcdef"; + const NEW = "fedcba9876543210"; + const podOf = (doc: Doc) => doc.spec.template.spec; + + test("platform.sealKeys becomes bridge.sealKeys, and by default no job reaches the database for it", () => { + const defaults = render(); + expect(parseInstallation(installationOf(defaults)).bridge.sealKeys).toEqual({ issueNew: false, retire: [] }); + expect(podOf(find(defaults, "Job", "libredb-bootstrap")).initContainers).toBeUndefined(); + expect(podOf(find(defaults, "Job", "libredb-restart")).initContainers).toBeUndefined(); + const issuing = parseInstallation(installationOf(render(["platform.sealKeys.issueNew=true", `platform.sealKeys.activeKid=${NEW}`]))); + expect(issuing.bridge.sealKeys).toEqual({ issueNew: true, activeKid: NEW, retire: [] }); + }); + + test("a retirement runs the installer's own SQL: the envelope query before the bootstrap, the canary delete after the release", () => { + const docs = render([`platform.sealKeys.activeKid=${NEW}`, `platform.sealKeys.retire[0]=${OLD}`, `platform.sealKeys.retire[1]=${NEW.replace("f", "e")}`]); + const kids = [OLD, NEW.replace("f", "e")]; + expect(parseInstallation(installationOf(docs)).bridge.sealKeys.retire).toEqual(kids); + expect(find(docs, "ConfigMap", "libredb-bootstrap-chart").data?.["seal-envelopes.sql"]).toBe(envelopeUsesScript(kids)); + expect(find(docs, "ConfigMap", "libredb-chart").data?.["seal-canary.sql"]).toBe(kids.map(deleteCanaryScript).join("")); + const bootstrap = podOf(find(docs, "Job", "libredb-bootstrap")); + expect(bootstrap.initContainers.map((container: { name: string }) => container.name)).toEqual(["seal-envelopes"]); + expect(bootstrap.initContainers[0].command[2]).toContain(`-f /etc/libredb/chart/seal-envelopes.sql > ${SEAL_ENVELOPES_FILE}`); + const restart = podOf(find(docs, "Job", "libredb-restart")); + expect(restart.initContainers[0].command[2]).toContain("-f /etc/libredb/chart/seal-canary.sql"); + for (const pod of [bootstrap, restart]) { + const init = pod.initContainers[0]; + expect(init.env).toEqual([{ name: "DATABASE_URL", valueFrom: { secretKeyRef: { name: "libredb-platform-migrate-env", key: "DATABASE_URL" } } }]); + // libpq gets the password in PGPASSWORD, never on psql's command line. + expect(init.command[2]).toContain('export PGPASSWORD="${rest%%@*}"'); + expect(pod.volumes.find((volume: { name: string }) => volume.name === "installation-ca").configMap.name).toBe("libredb-ca"); + } + }); + + test("the retirement's SQL waits for a connection, which a new pod's network policy may refuse at first, and runs once", () => { + const docs = render([`platform.sealKeys.activeKid=${NEW}`, `platform.sealKeys.retire[0]=${OLD}`]); + const work = mkdtempSync(join(tmpdir(), "libredb-seal-sql-")); + // A psql that refuses the first two connections, as kube-router does until it has the pod's address, and records + // each script it runs; SCRIPT_EXIT is what the script run exits with. + writeFileSync( + join(work, "psql"), + [ + "#!/bin/sh", + `n=$(cat "${work}/tries" 2>/dev/null || echo 0); echo $((n + 1)) > "${work}/tries"`, + '[ "$n" -ge 2 ] || exit 2', + `case " $* " in *" -f "*) echo run >> "${work}/runs"; exit "$SCRIPT_EXIT" ;; esac`, + ].join("\n"), + { mode: 0o755 }, + ); + const run = (pod: Doc, scriptExit: number) => { + rmSync(join(work, "tries"), { force: true }); + rmSync(join(work, "runs"), { force: true }); + const script = (podOf(pod).initContainers[0].command[2] as string).replaceAll(SEAL_ENVELOPES_FILE, join(work, "out")); + const result = Bun.spawnSync(["/bin/sh", "-c", script], { + env: { PATH: `${work}:/usr/bin:/bin`, SCRIPT_EXIT: String(scriptExit), DATABASE_URL: "postgresql://libredb_platform:pw@libredb-postgres:5432/libredb?schema=platform" }, + }); + const runs = existsSync(join(work, "runs")) ? readFileSync(join(work, "runs"), "utf8").trim().split("\n").length : 0; + return { code: result.exitCode, runs }; + }; + try { + for (const job of ["libredb-bootstrap", "libredb-restart"]) { + const pod = find(docs, "Job", job); + expect(run(pod, 0), job).toEqual({ code: 0, runs: 1 }); + // A failing script is the refusal itself, so it is never run again. + expect(run(pod, 3), job).toEqual({ code: 3, runs: 1 }); + } + } finally { + rmSync(work, { recursive: true, force: true }); + } + // Each of the four runs waits 1 s per refused connection. + }, 30_000); + + test("a kid that is no seal key id fails the render, naming the value", () => { + expect(failure(["platform.sealKeys.retire[0]=x'; DROP TABLE y;--"])).toContain("platform.sealKeys.retire[0]"); + expect(failure(["platform.sealKeys.activeKid=ABCDEF0123456789"])).toContain("platform.sealKeys.activeKid"); + }); + + test("platform.authSecretPrevious mounts its Secret where platform.authSecretPreviousFile names it", () => { + const docs = render(["platform.authSecretPrevious.existingSecret=auth-previous"]); + const file = parseInstallation(installationOf(docs)).platform.authSecretPreviousFile; + expect(file).toBe(`${INPUT_DIR}/auth-secret-previous/auth-secret-previous`); + const pod = podOf(find(docs, "Job", "libredb-bootstrap")); + const mount = pod.containers[0].volumeMounts.find((volume: { name: string }) => volume.name === "auth-secret-previous"); + expect(`${mount.mountPath}/auth-secret-previous`).toBe(file as string); + expect(pod.volumes.find((volume: { name: string }) => volume.name === "auth-secret-previous").secret).toEqual({ + secretName: "auth-previous", + items: [{ key: "auth-secret-previous", path: "auth-secret-previous" }], + }); + expect(installationOf(render()).platform.authSecretPreviousFile).toBeUndefined(); + }); +}); + +describe("the workloads", () => { + const docs = () => render(); + + test("serve gets only Platform's env Secret, and only the migrate job gets the owner's URL", () => { + const all = docs(); + const platform = find(all, "Deployment", "libredb-platform").spec.template.spec.containers[0]; + expect(platform.args).toEqual(["serve"]); + expect(platform.envFrom).toEqual([{ secretRef: { name: "libredb-platform-env" } }]); + expect(platform.env).toBeUndefined(); + const migrate = all.find((doc) => doc.kind === "Job" && doc.metadata.name.startsWith("libredb-platform-migrate-")); + expect(migrate?.spec.template.spec.containers[0].envFrom).toEqual([{ secretRef: { name: "libredb-platform-migrate-env" } }]); + const mentions = all.filter((doc) => JSON.stringify(doc).includes("libredb-platform-migrate-env")).map((doc) => `${doc.kind}/${doc.metadata.name}`); + expect(mentions.every((name) => name.startsWith("Job/libredb-platform-migrate-") || name === "Role/libredb-bootstrap")).toBe(true); + }); + + test("every pod runs as non-root with the RuntimeDefault seccomp profile, and only the three jobs mount a token", () => { + for (const doc of docs()) { + const pod = doc.kind === "Job" || doc.kind === "Deployment" || doc.kind === "StatefulSet" ? doc.spec.template.spec : undefined; + if (pod === undefined) continue; + const name = `${doc.kind}/${doc.metadata.name}`; + expect(pod.securityContext?.runAsNonRoot ?? true, name).toBe(true); + expect(pod.securityContext?.seccompProfile?.type, name).toBe("RuntimeDefault"); + const tokenJob = ["libredb-bootstrap", "libredb-restart"].includes(doc.metadata.name) || doc.metadata.name.startsWith("libredb-keycloak-configure-"); + if (!doc.metadata.name.startsWith("libredb-studio")) expect(pod.automountServiceAccountToken, name).toBe(tokenJob); + for (const container of pod.containers) { + expect(container.securityContext?.allowPrivilegeEscalation, `${name} ${container.name}`).toBe(false); + expect(container.securityContext?.capabilities?.drop, `${name} ${container.name}`).toEqual(["ALL"]); + } + } + }); + + test("on OpenShift every pod, Studio's included, leaves its uids to the SCC; elsewhere each keeps its image's", () => { + const uids = (all: Doc[]) => + all + .filter((doc) => doc.kind === "Job" || doc.kind === "Deployment" || doc.kind === "StatefulSet") + .map((doc) => { + const security = doc.spec.template.spec.securityContext ?? {}; + return [`${doc.kind}/${doc.metadata.name}`, security.runAsUser, security.runAsGroup, security.fsGroup]; + }); + for (const [name, user, group, fs] of uids(docs())) { + expect([user, group, fs].every((id) => typeof id === "number" && id > 0), String(name)).toBe(true); + } + for (const all of [render([], ["--api-versions", "security.openshift.io/v1"]), render(["global.compatibility.openshift.adaptSecurityContext=force"])]) { + for (const [name, ...ids] of uids(all)) expect(ids, String(name)).toEqual([undefined, undefined, undefined]); + } + const disabled = render(["global.compatibility.openshift.adaptSecurityContext=disabled"], ["--api-versions", "security.openshift.io/v1"]); + expect(find(disabled, "StatefulSet", "libredb-postgres").spec.template.spec.securityContext.runAsUser).toBe(999); + }); + + test("every pod of the amd64-only Platform image, its jobs included, runs on amd64 nodes and follows Platform's scheduling", () => { + const platformPods = (all: Doc[]) => + all.filter( + (doc) => + (doc.kind === "Job" || doc.kind === "Deployment") && + doc.spec.template.spec.containers.some((container: { image: string }) => container.image.includes("/libredb-platform:")), + ); + const defaults = platformPods(docs()); + expect(defaults.map((doc) => doc.metadata.name.replace(/-[0-9a-f]{8}$/, "")).sort()).toEqual([ + "libredb-bootstrap", + "libredb-keycloak-configure", + "libredb-platform", + "libredb-platform-migrate", + "libredb-restart", + ]); + for (const doc of defaults) { + expect(doc.spec.template.spec.nodeSelector, doc.metadata.name).toEqual({ "kubernetes.io/arch": "amd64" }); + expect(doc.spec.template.spec.tolerations, doc.metadata.name).toBeUndefined(); + } + const placed = render(["platform.nodeSelector.pool=apps", "platform.tolerations[0].key=dedicated", "platform.tolerations[0].operator=Exists"]); + for (const doc of platformPods(placed)) { + expect(doc.spec.template.spec.nodeSelector, doc.metadata.name).toEqual({ "kubernetes.io/arch": "amd64", pool: "apps" }); + expect(doc.spec.template.spec.tolerations, doc.metadata.name).toEqual([{ key: "dedicated", operator: "Exists" }]); + } + const arm = render(["platform.nodeSelector.kubernetes\\.io/arch=arm64"]); + expect(find(arm, "Deployment", "libredb-platform").spec.template.spec.nodeSelector).toEqual({ "kubernetes.io/arch": "arm64" }); + }); + + test("Studio's environment under the chart's defaults equals the installer's renderStudioEnv", () => { + const all = docs(); + const configMap = find(all, "ConfigMap", "libredb-studio-config").data ?? {}; + const container = find(all, "Deployment", "libredb-studio").spec.template.spec.containers[0]; + const env = new Map(Object.entries(configMap)); + for (const entry of container.env as { name: string; value?: string }[]) if (entry.value !== undefined) env.set(entry.name, entry.value); + const config = parseInstallation(installationOf(all)); + const expected = renderStudioEnv(computeDesired(config, manifest), { + postgres: { host: "libredb-postgres", port: 5432 }, + secrets: { oidcClientSecret: "s", jwtSecret: "j", storageEncryptionKey: "k", databasePassword: "p" }, + }); + const secretKeys = new Set(["OIDC_CLIENT_ID", "OIDC_CLIENT_SECRET", "JWT_SECRET", "STORAGE_POSTGRES_URL", "GOVERNANCE_POSTGRES_URL", "STORAGE_ENCRYPTION_KEY"]); + for (const [key, value] of Object.entries(expected)) { + if (value === undefined || secretKeys.has(key) || key === "NO_PROXY" || key === "no_proxy") continue; + expect(`${key}=${env.get(key)}`).toBe(`${key}=${value}`); + } + const fromSecret = (container.env as { name: string; valueFrom?: { secretKeyRef?: { name: string } } }[]).filter((entry) => entry.valueFrom?.secretKeyRef?.name === "libredb-studio-env").map((entry) => entry.name); + expect(fromSecret).toEqual(expect.arrayContaining(["JWT_SECRET", "OIDC_CLIENT_ID", "OIDC_CLIENT_SECRET", "STORAGE_POSTGRES_URL", "GOVERNANCE_POSTGRES_URL", "STORAGE_ENCRYPTION_KEY"])); + }); + + test("the managed Keycloak's environment equals the installer's compose service", () => { + const all = docs(); + const container = find(all, "Deployment", "keycloak").spec.template.spec.containers.find((candidate: { name: string }) => candidate.name === "keycloak"); + const env = Object.fromEntries((container.env as { name: string; value: string }[]).map((entry) => [entry.name, entry.value])); + const compose = parse(renderCompose(computeDesired(parseInstallation(installationOf(all)), manifest), composeTrust(undefined, []))); + expect(env).toEqual(compose.services.keycloak.environment); + expect(container.envFrom).toEqual([{ secretRef: { name: "libredb-keycloak-env" } }]); + }); + + test("the auth ingress reaches only the edge, and the Admin API stays in-cluster", () => { + const ingress = find(docs(), "Ingress", "keycloak"); + expect(ingress.spec.rules[0].http.paths[0].backend.service.name).toBe("keycloak-edge"); + }); + + test("Platform's Ingress asks ingress-nginx for a 16k header buffer, which Platform's sign-in callback needs", () => { + const annotation = "nginx.ingress.kubernetes.io/proxy-buffer-size"; + expect(find(docs(), "Ingress", "libredb-platform").metadata.annotations?.[annotation]).toBe("16k"); + expect(find(render(["ingress.annotations.team=dba"]), "Ingress", "libredb-platform").metadata.annotations).toEqual({ [annotation]: "16k", team: "dba" }); + }); + + test("a job's name follows the revision and what it runs: the same values keep it, a new installation document renames it", () => { + const names = (sets: string[]) => render(sets).filter((doc) => doc.kind === "Job").map((doc) => doc.metadata.name).sort(); + expect(names([])).toEqual(names([])); + const changed = names(["platform.organizationName=Other"]); + expect(changed.filter((name) => names([]).includes(name))).toEqual(["libredb-bootstrap", "libredb-restart"]); + }); + + test("the uploads volume outlives helm uninstall, as the database's volume claims do by Kubernetes' rule", () => { + const all = docs(); + expect(find(all, "PersistentVolumeClaim", "libredb-platform-uploads").metadata.annotations?.["helm.sh/resource-policy"]).toBe("keep"); + expect(find(all, "StatefulSet", "libredb-postgres").spec.volumeClaimTemplates[0].metadata.annotations).toBeUndefined(); + const off = render(["platform.persistence.enabled=false"]); + expect(off.some((doc) => doc.kind === "PersistentVolumeClaim")).toBe(false); + }); + + test("network policies guard PostgreSQL and Keycloak's Admin API by default, and render nothing when disabled", () => { + expect(docs().filter((doc) => doc.kind === "NetworkPolicy").map((doc) => doc.metadata.name).sort()).toEqual(["keycloak", "libredb-postgres"]); + expect(render(["networkPolicy.enabled=false"]).some((doc) => doc.kind === "NetworkPolicy")).toBe(false); + }); + + test("the jobs after the hook are Argo CD sync hooks, so a sync runs them again as an upgrade does", () => { + for (const job of docs().filter((doc) => doc.kind === "Job" && doc.metadata.name !== "libredb-bootstrap" && !doc.metadata.annotations?.["helm.sh/hook"])) { + expect(job.metadata.annotations?.["argocd.argoproj.io/hook"], job.metadata.name).toBe("Sync"); + expect(job.metadata.annotations?.["argocd.argoproj.io/hook-delete-policy"], job.metadata.name).toBe("BeforeHookCreation"); + } + }); + + test("the bootstrap and keycloak jobs fail at once on a refusal a retry cannot change", () => { + for (const job of docs().filter((doc) => doc.kind === "Job" && (doc.metadata.name === "libredb-bootstrap" || doc.metadata.name.startsWith("libredb-keycloak-configure-")))) { + const rule = job.spec.podFailurePolicy.rules[0]; + expect(rule.action, job.metadata.name).toBe("FailJob"); + expect(rule.onExitCodes.values).toEqual([2, 3, 6]); + expect(rule.onExitCodes.containerName).toBe(job.spec.template.spec.containers[0].name); + } + }); + + test("the bootstrap also runs before a rollback, where it refuses an older release, and the restart job after each change", () => { + const all = docs(); + expect(find(all, "Job", "libredb-bootstrap").metadata.annotations?.["helm.sh/hook"]).toBe("pre-install,pre-upgrade,pre-rollback"); + const restart = find(all, "Job", "libredb-restart"); + expect(restart.metadata.annotations?.["helm.sh/hook"]).toBe("post-install,post-upgrade,post-rollback"); + expect(restart.spec.template.spec.serviceAccountName).toBe("libredb-restart"); + }); + + test("the restart Role may patch exactly the workloads the bootstrap job asks to restart", () => { + const role = find(docs(), "Role", "libredb-restart"); + const names = (resource: string) => role.rules?.find((rule) => rule.resources[0] === resource)?.resourceNames ?? []; + const expected = Object.values(WORKLOADS); + expect([...names("deployments")].sort()).toEqual(expected.filter((w) => w.kind === "deployments").map((w) => w.name).sort()); + expect(names("statefulsets")).toEqual(expected.filter((w) => w.kind === "statefulsets").map((w) => w.name)); + const studio = find(docs(), "Deployment", WORKLOADS.studio.name); + expect(studio.metadata.name).toBe(WORKLOADS.studio.name); + }); + + test("the objects the jobs write carry the installation's label and no Helm or Argo CD tracking label", () => { + const settings = JSON.parse(find(docs(), "ConfigMap", "libredb-bootstrap-chart").data?.["chart.json"] as string); + expect(settings.labels["install.libredb.org/installation"]).toBe("libredb"); + expect(settings.labels["app.kubernetes.io/instance"]).toBeUndefined(); + }); + + test("the migrate job's wait passes the password in PGPASSWORD, never on a command line", () => { + const migrate = docs().find((doc) => doc.kind === "Job" && doc.metadata.name.startsWith("libredb-platform-migrate-")); + const script = migrate?.spec.template.spec.initContainers[0].command.join("\n") as string; + expect(script).toContain('export PGPASSWORD="${rest%%@*}"'); + expect(script).not.toContain('psql "$url"'); + }); +}); + +describe("the bootstrap Role", () => { + async function written(sets: string[], inputs: Record = {}) { + const all = render(sets); + const config = parseInstallation(installationOf(all)); + const objects = new MemoryObjects(); + await runBootstrap( + { config, manifest, studioSecretName: "libredb-studio-env", jobsBundle: new Uint8Array([1]) }, + { objects, readFile: files(inputs), env: {}, now: () => new Date(), log: () => {}, redactor: new Redactor() }, + ); + const role = find(all, "Role", "libredb-bootstrap"); + const allowed = (resource: string) => new Set(role.rules?.find((rule) => rule.resources.length === 1 && rule.resources[0] === resource)?.resourceNames ?? []); + return { objects, secrets: allowed("secrets"), configMaps: allowed("configmaps") }; + } + + test.each([ + ["managed PostgreSQL and Keycloak", [] as string[]], + ["the site's TLS Secrets", ["tls.mode=secrets", "tls.secrets.platform=a", "tls.secrets.studio=b", "tls.secrets.auth=c", "tls.caBundle=PEM", "studio.ingress.tls[0].secretName=b"]], + ])("with %s may replace exactly the objects the job writes", async (_name, sets) => { + const ca = (await createCa({ commonName: "site", now: new Date() })).certPem; + const { objects, secrets, configMaps } = await written( + sets.map((set) => (set === "tls.caBundle=PEM" ? `tls.caBundle=${ca.replaceAll(",", "\\,")}` : set)), + { [`${INPUT_DIR}/ca/tls-ca.crt`]: ca }, + ); + expect([...objects.secrets.keys()].sort()).toEqual([...secrets].sort()); + expect([...objects.configMaps.keys()].sort()).toEqual([...configMaps].sort()); + }); + + test("with an external PostgreSQL and the site's Keycloak may replace exactly the objects the job writes", async () => { + const ca = (await createCa({ commonName: "pg", now: new Date() })).certPem; + const { objects, secrets, configMaps } = await written( + ["postgresql.mode=external", "postgresql.external.host=pg.corp.example", "postgresql.external.caBundle=PEM", "keycloak.mode=external", "keycloak.external.baseUrl=https://auth.libredb.test", "keycloak.external.provisioning=manual", "keycloak.external.clientSecretsSecret=kc"], + { + [`${INPUT_DIR}/ca/postgres-ca.crt`]: ca, + [`${INPUT_DIR}/keycloak-clients/studio-client-secret`]: "s", + [`${INPUT_DIR}/keycloak-clients/platform-client-secret`]: "p", + }, + ); + expect([...objects.secrets.keys()].sort()).toEqual([...secrets].sort()); + expect([...objects.configMaps.keys()].sort()).toEqual([...configMaps].sort()); + }); +}); diff --git a/installer/test/unit/kubernetes/fixture.ts b/installer/test/unit/kubernetes/fixture.ts new file mode 100644 index 0000000..dadd81a --- /dev/null +++ b/installer/test/unit/kubernetes/fixture.ts @@ -0,0 +1,127 @@ +// Fixtures of the Helm chart's jobs: installations of mode kubernetes as the chart renders them, an in-memory +// namespace, and the input files the chart mounts. +import { parseInstallation, type LibreDBInstallation } from "../../../src/config/schema"; +import type { KubernetesObjects, ObjectData } from "../../../src/kubernetes/bootstrap"; +import { INPUT_DIR, type Workload } from "../../../src/kubernetes/names"; +import { Redactor } from "../../../src/log/redact"; + +export const LICENSE = "ldl1.0123456789abcdef.eyJpZCI6IlRFU1QifQ.c2lnbmF0dXJl"; + +export const HOSTS = { platform: "platform.libredb.test", studio: "studio.libredb.test", auth: "auth.libredb.test" }; + +/** The installation document of the chart's default values, with optional overrides of whole sections. */ +export function installation(overrides: Record = {}): LibreDBInstallation { + return parseInstallation({ + apiVersion: "install.libredb.org/v1", + kind: "LibreDBInstallation", + metadata: { name: "libredb" }, + mode: "kubernetes", + endpoints: { style: "hostnames", hosts: HOSTS }, + tls: { mode: "generated" }, + postgres: { mode: "managed" }, + keycloak: { mode: "managed", realm: "libredb" }, + platform: { organizationName: "Acme", licenseFile: `${INPUT_DIR}/license/license` }, + kubernetes: { namespace: "libredb" }, + ...overrides, + }); +} + +export const EXTERNAL_POSTGRES = { + postgres: { mode: "external", external: { host: "pg.corp.example", port: 5432, provisioning: "manual", caBundleFile: `${INPUT_DIR}/ca/postgres-ca.crt` } }, +}; + +export const MANUAL_KEYCLOAK = { + keycloak: { + mode: "external", + realm: "corp", + external: { + baseUrl: "https://auth.libredb.test", + provisioning: "manual", + studioClientSecretFile: `${INPUT_DIR}/keycloak-clients/studio-client-secret`, + platformClientSecretFile: `${INPUT_DIR}/keycloak-clients/platform-client-secret`, + }, + }, +}; + +export const INSTALLER_KEYCLOAK = { + keycloak: { + mode: "external", + realm: "corp", + external: { + baseUrl: "https://auth.libredb.test", + adminUrl: "https://kc-admin.corp.example", + provisioning: "installer", + adminRealm: "master", + adminClientId: "corp-admin", + adminClientSecretFile: `${INPUT_DIR}/keycloak-admin/admin-client-secret`, + }, + }, +}; + +/** A namespace's Secrets and ConfigMaps in memory, with every write recorded. */ +export class MemoryObjects implements KubernetesObjects { + readonly secrets = new Map(); + readonly configMaps = new Map(); + readonly binaryData = new Map>>(); + readonly writes: string[] = []; + readonly restarts: string[] = []; + /** The workloads that exist, which a restart finds. */ + readonly workloads = new Set(); + + async getConfigMap(name: string): Promise { + const data = this.configMaps.get(name); + return data === undefined ? undefined : { ...data }; + } + + /** Pod creation times per workload; a workload in `workloads` without an entry has one pod from long ago. */ + readonly pods = new Map(); + + async podsCreated(workload: Workload): Promise { + const key = `${workload.kind}/${workload.name}`; + if (!this.workloads.has(key)) return undefined; + return this.pods.get(key) ?? [new Date("2000-01-01T00:00:00Z")]; + } + + async restart(workload: Workload): Promise<"restarted" | "absent"> { + const key = `${workload.kind}/${workload.name}`; + if (!this.workloads.has(key)) return "absent"; + this.restarts.push(key); + return "restarted"; + } + + async getSecret(name: string): Promise { + const secret = this.secrets.get(name); + return secret === undefined ? undefined : { ...secret.data }; + } + + async putSecret(name: string, data: ObjectData, type: "Opaque" | "kubernetes.io/tls" = "Opaque"): Promise { + this.secrets.set(name, { data: { ...data }, type }); + this.writes.push(`secret/${name}`); + } + + async putConfigMap(name: string, data: ObjectData, binaryData?: Readonly>): Promise { + this.configMaps.set(name, { ...data }); + if (binaryData !== undefined) this.binaryData.set(name, binaryData); + this.writes.push(`configmap/${name}`); + } + + secret(name: string): ObjectData { + const secret = this.secrets.get(name); + if (secret === undefined) throw new Error(`no Secret ${name}`); + return secret.data; + } +} + +/** The input files the chart mounts, by path. */ +export function files(entries: Record = {}): (path: string) => string | undefined { + const all: Record = { [`${INPUT_DIR}/license/license`]: `${LICENSE}\n`, ...entries }; + return (path) => all[path]; +} + +export function io(objects: MemoryObjects, readFile = files(), now = new Date("2026-10-20T00:00:00Z")) { + const lines: string[] = []; + return { + io: { objects, readFile, env: {}, now: () => now, log: (line: string) => lines.push(line), redactor: new Redactor() }, + lines, + }; +} diff --git a/installer/test/unit/kubernetes/job-exit.test.ts b/installer/test/unit/kubernetes/job-exit.test.ts new file mode 100644 index 0000000..1a649c5 --- /dev/null +++ b/installer/test/unit/kubernetes/job-exit.test.ts @@ -0,0 +1,24 @@ +import { describe, expect, test } from "bun:test"; +import { ConfigError } from "../../../src/config/schema"; +import { RefusedError } from "../../../src/kubernetes/bootstrap"; +import { UnsafeChangeError } from "../../../src/state/immutable"; + +// The CLI module runs its job on import, so the exit code mapping is checked through a dynamic import guarded by an +// unknown command, which prints the usage and changes nothing. +describe("the jobs' exit code", () => { + test("a refusal keeps the installer's exit code, anything else is a runtime failure", async () => { + const argv = process.argv; + const error = console.error; + process.argv = [argv[0] as string, "job", "--help"]; + console.error = () => {}; + const { jobExitCode } = await import("../../../src/cli/kubernetes-job"); + process.argv = argv; + console.error = error; + process.exitCode = 0; + expect(jobExitCode(new ConfigError(["x: bad"]))).toBe(2); + expect(jobExitCode(new RefusedError("downgrade", 6))).toBe(6); + expect(jobExitCode(new UnsafeChangeError([]))).toBe(6); + expect(jobExitCode(new RefusedError("lock", 5))).toBe(1); + expect(jobExitCode(new TypeError("fetch failed"))).toBe(1); + }); +}); diff --git a/installer/test/unit/kubernetes/keycloak.test.ts b/installer/test/unit/kubernetes/keycloak.test.ts new file mode 100644 index 0000000..56d8656 --- /dev/null +++ b/installer/test/unit/kubernetes/keycloak.test.ts @@ -0,0 +1,199 @@ +import { describe, expect, test } from "bun:test"; +import { CLIENT_SECRET_NAMES } from "../../../src/keycloak/desired"; +import { MANAGED_ATTRIBUTE } from "../../../src/keycloak/representations"; +import { + adminTarget, + ensureManagedRealm, + KEYCLOAK_CONFIG_DIR, + MANAGED_KEYCLOAK_URL, + runKeycloakJob, + waitForKeycloak, + type ConvergeFunctions, + type ConvergeTarget, +} from "../../../src/kubernetes/keycloak"; +import { INPUT_DIR, K8S_NAMES, STORE_KEYS } from "../../../src/kubernetes/names"; +import { Redactor } from "../../../src/log/redact"; +import { STATE_KEY } from "../../../src/kubernetes/bootstrap"; +import type { InstallerState } from "../../../src/state/store"; +import { observedFixture } from "../keycloak/fixture"; +import { manifestFixture } from "../release/fixture"; +import { installation, INSTALLER_KEYCLOAK, MANUAL_KEYCLOAK, MemoryObjects } from "./fixture"; + +const CONFIG: Record = { + [`${KEYCLOAK_CONFIG_DIR}/${STORE_KEYS.keycloakBootstrapSecret}`]: "bootstrap-secret", + [`${KEYCLOAK_CONFIG_DIR}/${CLIENT_SECRET_NAMES.studio}`]: "studio-secret", + [`${KEYCLOAK_CONFIG_DIR}/${CLIENT_SECRET_NAMES.platform}`]: "platform-secret", + [`${KEYCLOAK_CONFIG_DIR}/${STORE_KEYS.firstAdminPassword}`]: "first-admin-temporary", + [`${KEYCLOAK_CONFIG_DIR}/${STORE_KEYS.userManagerPassword}`]: "user-manager-temporary", + [`${INPUT_DIR}/keycloak-admin/admin-client-secret`]: "corp-admin-secret\n", +}; + +const readFile = (path: string) => CONFIG[path]; + +/** A Keycloak that answers the token request and the realm lookup, and records every request. */ +function keycloak(realm: "missing" | "managed" | "foreign") { + const requests: string[] = []; + const fetcher = (async (input: string | URL | Request, init?: RequestInit) => { + const url = String(input); + const method = init?.method ?? "GET"; + requests.push(`${method} ${url}`); + if (url.endsWith("/protocol/openid-connect/token")) return Response.json({ access_token: "token" }); + if (method === "GET" && url.endsWith("/admin/realms/libredb")) { + if (realm === "missing") return new Response("", { status: 404 }); + return Response.json({ attributes: realm === "managed" ? { [MANAGED_ATTRIBUTE]: "true" } : {} }); + } + if (method === "POST" && url.endsWith("/admin/realms")) return new Response("", { status: 201 }); + return new Response("", { status: 500 }); + }) as typeof fetch; + return { fetcher, requests }; +} + +function jobIO(objects: MemoryObjects, fetcher?: typeof fetch) { + const written = new Map(); + const lines: string[] = []; + return { + io: { + objects, + readFile, + writeFile: (path: string, content: string) => written.set(path, content), + env: {}, + tmpDir: "/tmp", + owner: { uid: 1001, gid: 1001 }, + log: (line: string) => lines.push(line), + redactor: new Redactor(), + ...(fetcher === undefined ? {} : { fetch: fetcher }), + }, + written, + lines, + }; +} + +function converge() { + const seen: { target?: ConvergeTarget; applied?: number } = {}; + const functions: ConvergeFunctions = { + observe: async (target) => { + seen.target = target; + return { ...observedFixture("empty-realm"), realm: target.realm }; + }, + apply: async (planned, _target, state: InstallerState) => { + seen.applied = planned.length; + state.ownership.push({ kind: "keycloak-user", name: "libredb-admin", step: "keycloak-configure" }); + }, + }; + return { functions, seen }; +} + +describe("the keycloak-configure job", () => { + test("creates a managed realm, writes the temporary passwords for the step and records the accounts in the state", async () => { + const objects = new MemoryObjects(); + const { fetcher, requests } = keycloak("missing"); + const run = jobIO(objects, fetcher); + const step = converge(); + await runKeycloakJob(installation(), manifestFixture(), run.io, step.functions); + expect(requests).toContain(`POST ${MANAGED_KEYCLOAK_URL}/admin/realms`); + expect(run.written.get("/tmp/first-admin.txt")).toBe("first-admin-temporary\n"); + expect(run.written.get("/tmp/user-manager.txt")).toBe("user-manager-temporary\n"); + expect(step.seen.target?.baseUrl).toBe(MANAGED_KEYCLOAK_URL); + expect(step.seen.target?.credentials).toEqual({ realm: "master", clientId: "libredb-bootstrap", clientSecret: "bootstrap-secret" }); + expect(step.seen.target?.desired.appClients.map((client) => client.representation.secret)).toEqual(["studio-secret", "platform-secret"]); + expect(step.seen.applied).toBe(15); + expect(run.lines).toContain("keycloak: realm libredb created"); + expect(run.lines.at(-1)).toBe("keycloak: applied 15 actions"); + const state = JSON.parse(objects.secret(K8S_NAMES.state)[STATE_KEY] as string) as InstallerState; + expect(state.ownership).toEqual([{ kind: "keycloak-user", name: "libredb-admin", step: "keycloak-configure" }]); + }); + + test("keeps a managed realm the chart created, and refuses one it did not", async () => { + const managed = keycloak("managed"); + const run = jobIO(new MemoryObjects(), managed.fetcher); + await runKeycloakJob(installation(), manifestFixture(), run.io, converge().functions); + expect(run.lines).toContain("keycloak: realm libredb exists"); + expect(managed.requests.some((request) => request.startsWith("POST") && request.endsWith("/admin/realms"))).toBe(false); + const foreign = keycloak("foreign"); + await expect(runKeycloakJob(installation(), manifestFixture(), jobIO(new MemoryObjects(), foreign.fetcher).io, converge().functions)).rejects.toThrow( + MANAGED_ATTRIBUTE, + ); + }); + + test("an external Keycloak with provisioning installer converges with the site's admin credential and no realm creation", async () => { + const run = jobIO(new MemoryObjects(), (async () => new Response("", { status: 200 })) as unknown as typeof fetch); + const step = converge(); + await runKeycloakJob(installation(INSTALLER_KEYCLOAK), manifestFixture(), run.io, step.functions); + expect(step.seen.target?.baseUrl).toBe("https://kc-admin.corp.example"); + expect(step.seen.target?.credentials).toEqual({ realm: "master", clientId: "corp-admin", clientSecret: "corp-admin-secret" }); + expect(run.written.size).toBe(0); + expect(step.seen.applied).toBeGreaterThan(0); + expect(run.lines.at(-1)).toBe(`keycloak: applied ${step.seen.applied} actions`); + }); +}); + +describe("adminTarget", () => { + test("refuses provisioning manual and a missing admin credential", () => { + const sources = { readFile: () => undefined, env: {}, redactor: new Redactor() }; + expect(() => adminTarget(installation(MANUAL_KEYCLOAK), sources)).toThrow("provisioning is manual"); + expect(() => adminTarget(installation(INSTALLER_KEYCLOAK), sources)).toThrow("does not exist"); + }); + + test("refuses a managed Keycloak whose bootstrap secret is not mounted", () => { + expect(() => adminTarget(installation(), { readFile: () => undefined, env: {}, redactor: new Redactor() })).toThrow( + `Secret ${K8S_NAMES.keycloakConfig} holds no ${STORE_KEYS.keycloakBootstrapSecret}`, + ); + }); + + test("an admin user signs in through admin-cli and reaches the Admin API at the base URL without an adminUrl", () => { + const config = installation({ + keycloak: { + mode: "external", + realm: "corp", + external: { baseUrl: "https://auth.libredb.test", adminUser: "admin", adminPasswordEnv: "CORP_ADMIN_PASSWORD" }, + }, + }); + const target = adminTarget(config, { readFile, env: { CORP_ADMIN_PASSWORD: "pw" }, redactor: new Redactor() }); + expect(target).toEqual({ + baseUrl: "https://auth.libredb.test", + credentials: { realm: "master", clientId: "admin-cli", username: "admin", password: "pw" }, + }); + }); +}); + +describe("ensureManagedRealm", () => { + test("creates the realm with the installer's managed settings", async () => { + const writes: unknown[] = []; + const admin = { find: async () => undefined, write: async (...args: unknown[]) => writes.push(args) } as never; + expect(await ensureManagedRealm(admin, "libredb")).toBe("created"); + expect(JSON.stringify(writes)).toContain(`"${MANAGED_ATTRIBUTE}":"true"`); + }); +}); + +describe("waitForKeycloak", () => { + test("waits while Keycloak refuses connections, then returns once it answers", async () => { + let attempts = 0; + const lines: string[] = []; + const slept: number[] = []; + const fetcher = (async () => { + attempts += 1; + if (attempts < 3) throw new TypeError("fetch failed"); + return new Response("", { status: 404 }); + }) as unknown as typeof fetch; + await waitForKeycloak("http://keycloak:8080", { fetch: fetcher, log: (line) => lines.push(line), sleep: async (ms) => void slept.push(ms), now: () => 0 }); + expect(attempts).toBe(3); + expect(slept).toEqual([3000, 3000]); + expect(lines).toEqual(["keycloak: waiting for http://keycloak:8080 to answer", "keycloak: http://keycloak:8080 answers"]); + }); + + test("fails with the last connection error once the timeout passed", async () => { + let clock = 0; + const fetcher = (async () => { + throw new TypeError("fetch failed"); + }) as unknown as typeof fetch; + const waiting = waitForKeycloak("http://keycloak:8080", { + fetch: fetcher, + log: () => {}, + sleep: async (ms) => void (clock += ms), + now: () => clock, + timeoutMs: 10_000, + intervalMs: 4_000, + }); + await expect(waiting).rejects.toThrow("Keycloak at http://keycloak:8080 did not answer within 10 s: fetch failed"); + }); +}); diff --git a/installer/test/unit/verify/oidc-chain.test.ts b/installer/test/unit/verify/oidc-chain.test.ts index c695afc..7303325 100644 --- a/installer/test/unit/verify/oidc-chain.test.ts +++ b/installer/test/unit/verify/oidc-chain.test.ts @@ -125,6 +125,13 @@ describe("studio-oidc-chain", () => { chain(authorize({ ...good, redirect_uri: "https://localhost:3000/api/auth/oidc/callback" })), `the authorization request sends redirect_uri https://localhost:3000/api/auth/oidc/callback, and Studio's callback is ${REDIRECT_URI}`, ], + [ + chain(authorize({ ...good, redirect_uri: "http://studio.example.com/api/auth/oidc/callback" })), + `the authorization request sends redirect_uri http://studio.example.com/api/auth/oidc/callback, and Studio's callback is ${REDIRECT_URI}: ` + + "Studio saw plain HTTP, so the edge did not pass X-Forwarded-Proto on; in tls.mode external, " + + "tls.external.trustedProxies must hold the address the upstream proxy arrives from, " + + "which for a proxy on the Docker host is the gateway of the network libredb_edge", + ], [ chain(authorize({ ...good, prompt: "login" }), { authStatus: 200 }), "the authorization request sends prompt=login, and OIDC_PROMPT=omit sends none",