OAuth authentication for the gRPC reader and sink

CHANGELOG.md

@@ -7,9 +7,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 ### Added
 - Replicator docs (hugo site) to repository. [replicator#90](https://github.com/EventStore/replicator/pull/90)
 - Replicator docs workflows. [replicator#90](https://github.com/EventStore/replicator/pull/90)
+- OAuth 2.0 authentication for the gRPC reader and sink (client credentials or token file), configured independently per side, with automatic token refresh and recovery. Helm values for extra env, volumes, service account and pod labels.
 
 ### Changed
 - Updated workflows to add clear naming separation between replicator (app) and replicator (docs) workflows. [replicator#90](https://github.com/EventStore/replicator/pull/90)
 - Updated hugo site to use hugo modules, removed blog content. [replicator#90](https://github.com/EventStore/replicator/pull/90)
+- When environment variables are printed at startup, the values of connection strings, auth settings (except `auth.type`) and any value containing credentials (a URI with user info such as a MongoDB checkpoint connection string, or `DefaultUserCredentials=`) are masked as `***`.
+- The realtime subscription used for scavenging resubscribes reliably after drops, and its metadata cache is rebuilt after a gap.
+- Replication no longer hangs on shutdown when the writer has stopped; the metrics reporter keeps running after a failed position read.
 
-
+### Fixed
+- With `scavenge: true`, replication no longer stalls on a stream that has metadata but no events, such as the system streams KurrentDB 26 creates at startup (`$connectors-mngt/state-projection`): the scavenge filter treats it as empty instead of failing with StreamNotFound. Streams without metadata no longer log a warning per event. Auth failures on these reads still stop replication when OAuth is enabled. (DEV-1903)

charts/replicator/templates/statefulset.yaml

@@ -22,11 +22,25 @@ spec:
         app: {{ template "replicator.name" . }}
         release: {{ .Release.Name }}
         tier: web
+{{- with .Values.podLabels }}
+{{ toYaml . | indent 8 }}
+{{- end }}
     spec:
+{{- with .Values.serviceAccountName }}
+      serviceAccountName: {{ . }}
+{{- end }}
       containers:
       - name: {{ .Chart.Name }}
         image: {{ template "replicator.image" . }}
         imagePullPolicy: {{ .Values.image.pullPolicy | quote }}
+{{- with .Values.extraEnv }}
+        env:
+{{ toYaml . | indent 10 }}
+{{- end }}
+{{- with .Values.extraEnvFrom }}
+        envFrom:
+{{ toYaml . | indent 10 }}
+{{- end }}
         livenessProbe:
           httpGet:
             path: /health
@@ -59,6 +73,9 @@ spec:
         - name: partitioner-js
           mountPath: /app{{ include "replicator.sink.partitioner.filepath" . }}
           subPath: {{ include "replicator.sink.partitioner.filename" . }}
+{{- end }}
+{{- with .Values.extraVolumeMounts }}
+{{ toYaml . | indent 8 }}
 {{- end }}
         resources:
 {{ toYaml .Values.resources | indent 10 }}
@@ -86,3 +103,6 @@ spec:
       - name: {{ template "replicator.name" . }}
         persistentVolumeClaim:
           claimName: {{ template "replicator.fullname" . }}
+{{- with .Values.extraVolumes }}
+{{ toYaml . | indent 6 }}
+{{- end }}

charts/replicator/values.yaml

@@ -21,12 +21,24 @@ replicator:
   reader:
     connectionString:
     protocol: tcp
+    # auth:                       # optional, gRPC only. See docs: deployment/authentication
+    #   type: oauthClientCredentials
+    #   tokenEndpoint: https://login.microsoftonline.com/<tenant>/oauth2/v2.0/token
+    #   clientId: <client-id>
+    #   clientSecretFile: /var/run/secrets/replicator/client-secret   # do not put clientSecret here
+    #   scope: api://<kurrentdb-app>/.default
   sink:
     connectionString:
     protocol: grpc
     partitionCount: 1
     partitioner:
     bufferSize: 1000
+    # auth:                       # optional, gRPC only. See docs: deployment/authentication
+    #   type: oauthClientCredentials
+    #   tokenEndpoint: https://login.microsoftonline.com/<tenant>/oauth2/v2.0/token
+    #   clientId: <client-id>
+    #   clientSecretFile: /var/run/secrets/replicator/client-secret   # do not put clientSecret here
+    #   scope: api://<kurrentdb-app>/.default
   scavenge: true
   filters: []
   transform: null
@@ -36,3 +48,16 @@ replicator:
     checkpointAfter: 1000
 #  restartOnFailure: false
 #  runContinuously: true
+
+# Service account for the pod (e.g. one federated with Azure Workload Identity).
+serviceAccountName: ""
+# Extra labels on the pod template, e.g. azure.workload.identity/use: "true".
+podLabels: {}
+# Extra environment variables. Use these (not replicator.*.auth in this file) for secrets, e.g.
+# - name: REPLICATOR_SINK_AUTH_CLIENTSECRET
+#   valueFrom: { secretKeyRef: { name: replicator-oauth, key: client-secret } }
+extraEnv: []
+extraEnvFrom: []
+# Extra volumes and mounts, e.g. a Secret holding clientSecretFile or a token file written by a sidecar.
+extraVolumes: []
+extraVolumeMounts: []

compose/oauth/.env.example

@@ -0,0 +1,5 @@
+# Copy to .env and fill in. .env is git-ignored.
+# OAuth authentication in KurrentDB requires a license key.
+KURRENTDB_LICENSE_KEY=
+# Native arm64 images make startup much faster on Apple Silicon; use an amd64 tag elsewhere.
+KURRENTDB_IMAGE=kurrentplatform/kurrentdb:26.1.1-experimental-arm64-10.0-noble

compose/oauth/.gitignore

@@ -0,0 +1,5 @@
+.env
+certs/
+tokens/
+scenarios/*/checkpoint
+scenarios/*/*.log

compose/oauth/README.md

@@ -0,0 +1,113 @@
+# Replicator OAuth: local end-to-end test bed
+
+Runs a test identity provider and three KurrentDB nodes in Docker, then runs Replicator from this branch on the host against them. It follows KurrentDB's "Testing with a local identity server" guide (`docs/server/security/user-authentication.md` in the KurrentDB repo) and uses the same IdentityServer 4 image as KurrentDB's own OAuth tests.
+
+| Service | Host port | Auth |
+|:--|:--|:--|
+| `idsrv4` (IdentityServer 4) | `5080` (http), `5001` (https) | issues tokens; client `replicator` / secret `replicator-secret` gets role `$admins`, 5-minute tokens |
+| `kurrentdb-basic` | `2113` | basic, `admin:changeit` |
+| `kurrentdb-oauth` | `2114` | OAuth only |
+| `kurrentdb-oauth2` | `2115` | OAuth only |
+
+All nodes use TLS with a test CA generated into `certs/`. Replicator trusts it through `tlsCaFile` in the connection strings. Data is in memory, so restarting a node empties it.
+
+## Setup
+
+1. OAuth in KurrentDB needs a license key. Copy `.env.example` to `.env` and set `KURRENTDB_LICENSE_KEY`. `.env` is git-ignored.
+2. Generate certificates and start everything:
+
+   ```bash
+   ./scripts/gen-certs.sh
+   docker compose up -d
+   ```
+
+3. Check the OAuth nodes picked up the IdP. Their logs should show "OAuthAuthentication ... plugin enabled" and "Issuer signing keys have been retrieved":
+
+   ```bash
+   docker compose logs kurrentdb-oauth | grep -i -E "oauth|signing keys|licen"
+   ```
+
+## Scenarios
+
+Each scenario folder holds a Replicator `config/appsettings.yaml`. Run it with `./scripts/run-replicator.sh <scenario>`, which builds Replicator in Release and runs it with the scenario folder as the working directory, with the HTTP API on `localhost:5005`. Delete `scenarios/<name>/checkpoint` to start a scenario from scratch.
+
+Helpers:
+- `./scripts/write-events.sh <basic|oauth|oauth2> <stream> <count> [delay-seconds]` appends events over HTTP.
+- `./scripts/stream-head.sh <node> <stream>` prints the last event number.
+- `./scripts/get-token.sh` prints a token for the `replicator` client.
+
+Use a fresh stream name for each run.
+
+### 1. Basic source → OAuth sink (client credentials, secret file)
+
+```bash
+./scripts/write-events.sh basic s1-a 100
+./scripts/run-replicator.sh 1-basic-to-oauth        # in another terminal
+./scripts/stream-head.sh oauth s1-a                 # expect 99
+```
+
+To test across token expiry, keep writing while Replicator runs for at least 3 token lifetimes (5 minutes each):
+
+```bash
+./scripts/write-events.sh basic s1-long 1000 1
+```
+
+Expected: Replicator logs "access token acquired" about every 2.5 minutes, no write failures, and the target stream head matches the source.
+
+### 2. OAuth source (client credentials) → basic sink
+
+```bash
+./scripts/write-events.sh oauth s2-a 100
+./scripts/run-replicator.sh 2-oauth-to-basic
+./scripts/stream-head.sh basic s2-a                 # expect 99
+```
+
+For the token-expiry run, write slowly for at least 15 minutes (`./scripts/write-events.sh oauth s2-long 1000 1`). Then record which of these happened:
+- (a) No reader restart or subscription drop happened at token expiry.
+- (b) The read and/or the realtime subscription were ended at expiry and restarted with a fresh token.
+
+Either way the target must end up with every event.
+
+### 3. OAuth source (client credentials) → OAuth sink (token file, rotated)
+
+```bash
+./scripts/refresh-token-file.sh 60 &                # keeps tokens/sink.token fresh
+./scripts/write-events.sh oauth s3-a 100
+./scripts/run-replicator.sh 3-oauth-to-oauth-tokenfile
+./scripts/stream-head.sh oauth2 s3-a                # expect 99
+```
+
+### 4. Token endpoint outage
+
+While scenario 1 or 2 is replicating, run:
+
+```bash
+docker compose stop idsrv4
+```
+
+Wait past a token lifetime (more than 5 minutes), then:
+
+```bash
+docker compose start idsrv4
+```
+
+Expected: rate-limited warnings, replication pauses, then resumes on its own with no process restart.
+
+### 5. Rejected token / misconfiguration
+
+- **Wrong audience.** Change `Audience` in `kurrentdb/oauth.conf` and run `docker compose up -d --force-recreate kurrentdb-oauth`. Each rejection fetches a new token, because idsrv4 issues a new value every time, so retries settle at one every 30 s. Expected: KurrentDB rejects the token as Unauthenticated. Replicator logs a warning, quarantines the token, re-tests it once a minute, and recovers once the audience is fixed.
+- **Missing role.** Remove the `Claims` from the `replicator` client in `idsrv4/idsrv4.conf.json`, then recreate `idsrv4` and restart the OAuth node so it fetches idsrv4's new signing key.
+  - Sink: with KurrentDB's default ACLs, any authenticated identity can append to user streams, so plain writes still succeed. Metadata writes and deletes need `$admins`.
+  - Reader: reading `$all` needs `$admins`. Replicator retries the subscription with backoff and logs PermissionDenied warnings, and replication doesn't start until the role is restored.
+
+## Known issues
+
+- Recreating `idsrv4` generates a new signing key. Restart the OAuth nodes afterwards so they fetch it; otherwise every token is rejected as Unauthenticated.
+
+## Teardown
+
+```bash
+docker compose down
+```
+
+`certs/`, `tokens/` and checkpoints are git-ignored; delete them to start clean.

compose/oauth/docker-compose.yml

@@ -0,0 +1,70 @@
+# Local end-to-end test bed for Replicator OAuth authentication.
+# See README.md in this folder. Replicator itself runs on the host from this branch.
+
+x-kurrentdb-env: &kurrentdb-env
+  KURRENTDB_CLUSTER_SIZE: 1
+  KURRENTDB_MEM_DB: "true"
+  KURRENTDB_RUN_PROJECTIONS: None
+  KURRENTDB_NODE_IP: 0.0.0.0
+  KURRENTDB_REPLICATION_IP: 0.0.0.0
+  KURRENTDB_ENABLE_ATOM_PUB_OVER_HTTP: "true"
+  KURRENTDB_CERTIFICATE_FILE: /certs/node/node.crt
+  KURRENTDB_CERTIFICATE_PRIVATE_KEY_FILE: /certs/node/node.key
+  KURRENTDB_TRUSTED_ROOT_CERTIFICATES_PATH: /certs/ca
+  KURRENTDB_CERTIFICATE_RESERVED_NODE_COMMON_NAME: kurrentdb-node
+
+x-oauth-env: &oauth-env
+  <<: *kurrentdb-env
+  KURRENTDB_CONFIG: /etc/kurrentdb/oauth.conf
+  KURRENTDB__LICENSING__LICENSE_KEY: ${KURRENTDB_LICENSE_KEY:-}
+
+services:
+  # IdentityServer 4 test IdP, as used by KurrentDB's own OAuth tests and docs.
+  # HTTP on host port 5080 (Replicator only allows plain http for loopback token endpoints),
+  # HTTPS on 5001 for KurrentDB's discovery/key fetches inside the network.
+  idsrv4:
+    image: ghcr.io/kurrent-io/idsrv4/idsrv4:latest
+    environment:
+      DOTNET_URLS: http://+:5000;https://+:5001
+    ports:
+      - "5080:5000"
+      - "5001:5001"
+    volumes:
+      - ./idsrv4/users.conf.json:/etc/idsrv4/users.conf:ro
+      - ./idsrv4/idsrv4.conf.json:/etc/idsrv4/idsrv4.conf:ro
+
+  # Basic auth (admin:changeit), no license needed.
+  kurrentdb-basic:
+    image: ${KURRENTDB_IMAGE:-kurrentplatform/kurrentdb:26.1.1-experimental-arm64-10.0-noble}
+    environment:
+      <<: *kurrentdb-env
+    ports:
+      - "2113:2113"
+    volumes:
+      - ./certs:/certs:ro
+
+  # OAuth-only node #1.
+  kurrentdb-oauth:
+    image: ${KURRENTDB_IMAGE:-kurrentplatform/kurrentdb:26.1.1-experimental-arm64-10.0-noble}
+    environment:
+      <<: *oauth-env
+    ports:
+      - "2114:2113"
+    volumes:
+      - ./certs:/certs:ro
+      - ./kurrentdb/oauth.conf:/etc/kurrentdb/oauth.conf:ro
+    depends_on:
+      - idsrv4
+
+  # OAuth-only node #2 (target for the OAuth-to-OAuth scenario).
+  kurrentdb-oauth2:
+    image: ${KURRENTDB_IMAGE:-kurrentplatform/kurrentdb:26.1.1-experimental-arm64-10.0-noble}
+    environment:
+      <<: *oauth-env
+    ports:
+      - "2115:2113"
+    volumes:
+      - ./certs:/certs:ro
+      - ./kurrentdb/oauth.conf:/etc/kurrentdb/oauth.conf:ro
+    depends_on:
+      - idsrv4

compose/oauth/idsrv4/idsrv4.conf.json

@@ -0,0 +1,43 @@
+{
+  "IdentityResources": [
+    { "Name": "openid", "DisplayName": "Your user identifier", "Required": true, "UserClaims": ["sub", "role"] },
+    { "Name": "profile", "DisplayName": "User profile", "UserClaims": ["name", "preferred_username"] }
+  ],
+  "ApiResources": [
+    { "Name": "kurrentdb", "Scopes": ["streams"] }
+  ],
+  "ApiScopes": [
+    { "Name": "streams", "UserClaims": ["http://schemas.microsoft.com/ws/2008/06/identity/claims/role", "role"] }
+  ],
+  "Clients": [
+    {
+      "ClientId": "kurrentdb-client",
+      "AllowedGrantTypes": ["password", "authorization_code"],
+      "ClientSecrets": [ { "Value": "K7gNU3sdo+OL0wNhqoVWhr3g6s1xYv72ol/pe/Unols=" } ],
+      "AllowedScopes": ["streams", "openid", "profile"],
+      "RedirectUris": [
+        "https://localhost:2114/signin-oidc", "https://localhost:2114/oauth/callback",
+        "https://localhost:2115/signin-oidc", "https://localhost:2115/oauth/callback"
+      ],
+      "AlwaysIncludeUserClaimsInIdToken": true,
+      "RequireConsent": false,
+      "AlwaysSendClientClaims": true,
+      "AllowOfflineAccess": true,
+      "RequireClientSecret": false,
+      "AllowAccessTokensViaBrowser": true
+    },
+    {
+      "ClientId": "replicator",
+      "AllowedGrantTypes": ["client_credentials"],
+      "ClientSecrets": [ { "Value": "KAt78nlfli+lNvMiwB6+C9QvQTpwYPuQ4luobTdP+2M=" } ],
+      "AllowedScopes": ["streams"],
+      "Claims": [
+        { "Type": "role", "Value": "$admins" },
+        { "Type": "http://schemas.microsoft.com/ws/2008/06/identity/claims/role", "Value": "$admins" }
+      ],
+      "ClientClaimsPrefix": "",
+      "AlwaysSendClientClaims": true,
+      "AccessTokenLifetime": 300
+    }
+  ]
+}

compose/oauth/idsrv4/users.conf.json

@@ -0,0 +1,11 @@
+[
+  {
+    "subjectId": "1",
+    "username": "admin",
+    "password": "password",
+    "claims": [
+      { "type": "role", "value": "$admins" },
+      { "type": "role", "value": "$ops" }
+    ]
+  }
+]

compose/oauth/kurrentdb/oauth.conf

@@ -0,0 +1,12 @@
+---
+AuthenticationType: oauth
+LogFailedAuthenticationAttempts: true
+OAuth:
+  Issuer: https://idsrv4:5001/
+  Audience: kurrentdb
+  ClientId: kurrentdb-client
+  ClientSecret: K7gNU3sdo+OL0wNhqoVWhr3g6s1xYv72ol/pe/Unols=
+  # Test-only: idsrv4 uses a self-signed dev certificate, and tokens fetched via
+  # http://localhost:5080 carry that issuer rather than https://idsrv4:5001.
+  Insecure: true
+  DisableIssuerValidation: true

compose/oauth/scenarios/1-basic-to-oauth/config/appsettings.yaml

@@ -0,0 +1,18 @@
+# Scenario 1: basic-auth source -> OAuth sink (client credentials, secret file)
+replicator:
+  reader:
+    protocol: grpc
+    connectionString: "esdb://admin:changeit@localhost:2113?tls=true&tlsCaFile=../../certs/ca/ca.crt"
+  sink:
+    protocol: grpc
+    connectionString: "esdb://localhost:2114?tls=true&tlsCaFile=../../certs/ca/ca.crt"
+    auth:
+      type: oauthClientCredentials
+      tokenEndpoint: http://localhost:5080/connect/token
+      clientId: replicator
+      clientSecretFile: ../../secrets/replicator-client-secret
+      scope: streams
+  scavenge: true
+  checkpoint:
+    type: file
+    path: ./checkpoint