From d92577cd7ea930f6d3c25d95249a3db28ac12b5a Mon Sep 17 00:00:00 2001 From: Blocks Task Runner Date: Mon, 14 Sep 2026 22:00:21 +0000 Subject: [PATCH 1/4] feat(proxy): accept a split client cert/key for internal mTLS The inference proxy could only take its Control API client identity as one combined PEM. cert-manager can produce that, but only via additionalOutputFormats, which is off by default before 1.15 and needs a cluster-wide feature gate on both the controller and the webhook. Vault, SPIRE and `kubectl create secret tls` can't produce it at all. Add HARNESS_PROXY_CLIENT_CERT_FILE/_PEM + _KEY_FILE/_PEM alongside the existing identity vars, mirroring the split pair the control-api already takes. The two shapes are mutually exclusive and half a pair is an error, so a half-migrated deployment fails loudly and names the offending variable. reqwest's two-buffer constructor is native-tls-only and we're rustls-only, so the pair is joined in memory and handed to the same Identity::from_pem; the combined path is unchanged. Blank env values now read as unset, which is how a template says "not configured". The fingerprint hashes whichever identity files are configured, so rotating only the key is still picked up within 30s. Chart gets blue.internalTransport.clientSecretFormat (combined|split), default combined. Co-Authored-By: Claude Opus 5 --- apps/docs/0.1.0/concepts/gateway-mode.mdx | 8 + apps/docs/0.1.0/deployment/production.mdx | 16 ++ apps/docs/next/concepts/gateway-mode.mdx | 8 + apps/docs/next/deployment/production.mdx | 16 ++ deploy/helm/README.md | 9 +- .../templates/inference-proxy-deployment.yaml | 5 + deploy/helm/values.schema.json | 3 +- deploy/helm/values.yaml | 10 +- scripts/verify-deployment.sh | 45 +++ services/inference-proxy/src/main.rs | 260 ++++++++++++++++-- tests/e2e/README.md | 12 + tests/e2e/docker-compose.yml | 8 +- tests/e2e/fixtures/mtls/generate.sh | 3 + 13 files changed, 376 insertions(+), 27 deletions(-) diff --git a/apps/docs/0.1.0/concepts/gateway-mode.mdx b/apps/docs/0.1.0/concepts/gateway-mode.mdx index 010f76e..42bed31 100644 --- a/apps/docs/0.1.0/concepts/gateway-mode.mdx +++ b/apps/docs/0.1.0/concepts/gateway-mode.mdx @@ -115,6 +115,14 @@ invalid rotations retain the last-known-good configuration. The Control API also pins the token's `sub`, `client_id`, and `azp` claims to the configured proxy client id. +The proxy's client certificate and private key may arrive as one combined PEM +(`HARNESS_PROXY_CLIENT_IDENTITY_FILE`) or as a separate pair +(`HARNESS_PROXY_CLIENT_CERT_FILE` and `HARNESS_PROXY_CLIENT_KEY_FILE`), matching +the split files the Control API already takes. Configure exactly one of the two; +setting both, or only half of the pair, is a startup error that names the +offending variable. Rotating either file of a split pair is picked up by the same +30-second reload. + `insecure-http` is available only as a deliberate trusted-network tradeoff. OAuth M2M and NetworkPolicy isolation remain mandatory, but virtual keys are plaintext on the wire and there is no client-certificate check. A VPC or an API diff --git a/apps/docs/0.1.0/deployment/production.mdx b/apps/docs/0.1.0/deployment/production.mdx index 81f9bf7..038a2ce 100644 --- a/apps/docs/0.1.0/deployment/production.mdx +++ b/apps/docs/0.1.0/deployment/production.mdx @@ -124,6 +124,22 @@ requests, and retain the last-known-good configuration after an invalid update. Set both `blue.internalTransport.serverSecret` and `blue.internalTransport.clientSecret`; empty names render unusable Secret volumes. +The server Secret holds `ca.crt`, `tls.crt`, and `tls.key`. The client Secret +holds `ca.crt` plus the proxy's own identity, in whichever of two layouts your +issuer produces: + +| `blue.internalTransport.clientSecretFormat` | Client Secret keys | Use it when | +| --- | --- | --- | +| `combined` (default) | `ca.crt`, `client.pem` | You build `client.pem` yourself by concatenating the certificate and its key | +| `split` | `ca.crt`, `tls.crt`, `tls.key` | Your issuer emits the certificate and key as separate files | + +Prefer `split` with an automated issuer. cert-manager, Vault, SPIRE, and +`kubectl create secret tls` all write `tls.crt` + `tls.key`, so the Certificate +needs no `additionalOutputFormats` stanza — that field is only on by default +from cert-manager 1.15, and enabling it earlier takes a cluster-wide feature +gate on both the controller and the webhook. `kubectl create secret tls` cannot +produce a combined file at all. + `blue.internalTransport.mode: insecure-http` is supported for fully private, trusted networks. OAuth M2M remains mandatory and port 8082 must stay restricted to inference-proxy pods, but resolved virtual keys are not encrypted in transit. diff --git a/apps/docs/next/concepts/gateway-mode.mdx b/apps/docs/next/concepts/gateway-mode.mdx index fdf3c8a..efb9857 100644 --- a/apps/docs/next/concepts/gateway-mode.mdx +++ b/apps/docs/next/concepts/gateway-mode.mdx @@ -113,6 +113,14 @@ invalid rotations retain the last-known-good configuration. The Control API also pins the token's `sub`, `client_id`, and `azp` claims to the configured proxy client id. +The proxy's client certificate and private key may arrive as one combined PEM +(`HARNESS_PROXY_CLIENT_IDENTITY_FILE`) or as a separate pair +(`HARNESS_PROXY_CLIENT_CERT_FILE` and `HARNESS_PROXY_CLIENT_KEY_FILE`), matching +the split files the Control API already takes. Configure exactly one of the two; +setting both, or only half of the pair, is a startup error that names the +offending variable. Rotating either file of a split pair is picked up by the same +30-second reload. + `insecure-http` is available only as a deliberate trusted-network tradeoff. OAuth M2M and NetworkPolicy isolation remain mandatory, but virtual keys are plaintext on the wire and there is no client-certificate check. A VPC or an API diff --git a/apps/docs/next/deployment/production.mdx b/apps/docs/next/deployment/production.mdx index d1e3726..a81f152 100644 --- a/apps/docs/next/deployment/production.mdx +++ b/apps/docs/next/deployment/production.mdx @@ -124,6 +124,22 @@ requests, and retain the last-known-good configuration after an invalid update. Set both `blue.internalTransport.serverSecret` and `blue.internalTransport.clientSecret`; empty names render unusable Secret volumes. +The server Secret holds `ca.crt`, `tls.crt`, and `tls.key`. The client Secret +holds `ca.crt` plus the proxy's own identity, in whichever of two layouts your +issuer produces: + +| `blue.internalTransport.clientSecretFormat` | Client Secret keys | Use it when | +| --- | --- | --- | +| `combined` (default) | `ca.crt`, `client.pem` | You build `client.pem` yourself by concatenating the certificate and its key | +| `split` | `ca.crt`, `tls.crt`, `tls.key` | Your issuer emits the certificate and key as separate files | + +Prefer `split` with an automated issuer. cert-manager, Vault, SPIRE, and +`kubectl create secret tls` all write `tls.crt` + `tls.key`, so the Certificate +needs no `additionalOutputFormats` stanza — that field is only on by default +from cert-manager 1.15, and enabling it earlier takes a cluster-wide feature +gate on both the controller and the webhook. `kubectl create secret tls` cannot +produce a combined file at all. + `blue.internalTransport.mode: insecure-http` is supported for fully private, trusted networks. OAuth M2M remains mandatory and port 8082 must stay restricted to inference-proxy pods, but resolved virtual keys are not encrypted in transit. diff --git a/deploy/helm/README.md b/deploy/helm/README.md index 4e8d227..ad709cf 100644 --- a/deploy/helm/README.md +++ b/deploy/helm/README.md @@ -95,8 +95,13 @@ ClusterIP Service on port 8082. Choose the transport explicitly with `blue.internalTransport.serverSecret` to a Secret containing `ca.crt`, `tls.crt`, and `tls.key`; the server certificate SAN must cover `-control-api-internal`. Set `blue.internalTransport.clientSecret` - to a Secret containing `ca.crt` and `client.pem`, where `client.pem` contains - the proxy certificate followed by its private key. + to a Secret containing `ca.crt` plus the proxy identity, and pick its layout + with `blue.internalTransport.clientSecretFormat`: + `combined` (default) reads `client.pem`, the proxy certificate followed by its + private key; `split` reads `tls.crt` and `tls.key`, which is what cert-manager, + Vault, SPIRE, and `kubectl create secret tls` emit. Use `split` with an + automated issuer — the Certificate then needs no `additionalOutputFormats` + stanza, which is only on by default from cert-manager 1.15. - `insecure-http` disables transport encryption and certificate authentication. OAuth M2M remains mandatory, but decrypted virtual keys cross the pod network in plaintext. Use it only on a private, trusted network with enforced diff --git a/deploy/helm/templates/inference-proxy-deployment.yaml b/deploy/helm/templates/inference-proxy-deployment.yaml index 6517bd0..4d129d8 100644 --- a/deploy/helm/templates/inference-proxy-deployment.yaml +++ b/deploy/helm/templates/inference-proxy-deployment.yaml @@ -63,8 +63,13 @@ spec: value: {{ printf "%s/api/auth/oauth2/token" (.Values.blue.publicUrls.dashboard | trimSuffix "/") | quote }} {{- if eq .Values.blue.internalTransport.mode "mtls" }} - { name: HARNESS_INTERNAL_CA_FILE, value: /var/run/blue/internal-tls/ca.crt } + {{- if eq .Values.blue.internalTransport.clientSecretFormat "split" }} + - { name: HARNESS_PROXY_CLIENT_CERT_FILE, value: /var/run/blue/internal-tls/tls.crt } + - { name: HARNESS_PROXY_CLIENT_KEY_FILE, value: /var/run/blue/internal-tls/tls.key } + {{- else }} - { name: HARNESS_PROXY_CLIENT_IDENTITY_FILE, value: /var/run/blue/internal-tls/client.pem } {{- end }} + {{- end }} - { name: HARNESS_PROXY_OAUTH_CLIENT_ID, value: {{ .Values.blue.inferenceProxyClientId | quote }} } - { name: HARNESS_PROXY_OAUTH_SCOPE, value: "gateway:resolve" } - { name: HARNESS_PROXY_OAUTH_RESOURCE, value: {{ .Values.blue.publicUrls.controlApi | quote }} } diff --git a/deploy/helm/values.schema.json b/deploy/helm/values.schema.json index 013528a..a3f49ba 100644 --- a/deploy/helm/values.schema.json +++ b/deploy/helm/values.schema.json @@ -36,7 +36,8 @@ "properties": { "mode": { "enum": ["mtls", "insecure-http"] }, "serverSecret": { "type": "string" }, - "clientSecret": { "type": "string" } + "clientSecret": { "type": "string" }, + "clientSecretFormat": { "enum": ["combined", "split"] } } }, "existingSecret": { "type": "string" }, diff --git a/deploy/helm/values.yaml b/deploy/helm/values.yaml index cdc15c2..9f8ea12 100644 --- a/deploy/helm/values.yaml +++ b/deploy/helm/values.yaml @@ -34,10 +34,16 @@ blue: internalTransport: # mtls (recommended/default) or insecure-http (trusted private networks only). mode: mtls - # Server Secret: ca.crt, tls.crt, tls.key. Client Secret: ca.crt and - # client.pem (certificate followed by its private key). + # Server Secret: ca.crt, tls.crt, tls.key. Client Secret: ca.crt plus the + # proxy identity in the layout named by clientSecretFormat. serverSecret: "" clientSecret: "" + # Layout of the client Secret's identity material: + # combined - client.pem, the certificate followed by its private key. + # split - tls.crt and tls.key, what cert-manager, Vault, SPIRE and + # `kubectl create secret tls` emit. No additionalOutputFormats + # stanza, so no cert-manager 1.15 floor. + clientSecretFormat: combined existingSecret: "" config: existingConfigMap: "" diff --git a/scripts/verify-deployment.sh b/scripts/verify-deployment.sh index f9f2366..3d974d2 100755 --- a/scripts/verify-deployment.sh +++ b/scripts/verify-deployment.sh @@ -139,6 +139,51 @@ helm template blue "$chart" \ --set blue.internalTransport.mode=insecure-http \ > /tmp/blue-gateway-insecure-production.yaml +# The proxy's client identity comes either as one combined client.pem or as the +# tls.crt + tls.key pair every non-cert-manager issuer emits. Each layout must +# render its own variables and none of the other's, or the proxy either reads a +# file that is not mounted or trips its combined-plus-split conflict check. +grep -q 'name: HARNESS_PROXY_CLIENT_IDENTITY_FILE' /tmp/blue-gateway-production.yaml +if grep -Eq 'HARNESS_PROXY_CLIENT_(CERT|KEY)_FILE' /tmp/blue-gateway-production.yaml; then + echo "default client-secret render emitted split cert/key variables" >&2 + exit 1 +fi +helm template blue "$chart" \ + "${production_network[@]}" \ + "${gateway_jwt[@]}" \ + --set blue.existingSecret=blue-runtime \ + --set image.digest="$digest" \ + --set blue.enableInferenceProxy=true \ + --set blue.gatewayType=litellm \ + --set blue.internalTransport.serverSecret=blue-internal-server \ + --set blue.internalTransport.clientSecret=blue-internal-client \ + --set blue.internalTransport.clientSecretFormat=split \ + > /tmp/blue-gateway-split-identity.yaml +grep -q 'name: HARNESS_PROXY_CLIENT_CERT_FILE' /tmp/blue-gateway-split-identity.yaml +grep -q 'name: HARNESS_PROXY_CLIENT_KEY_FILE' /tmp/blue-gateway-split-identity.yaml +grep -q 'name: HARNESS_INTERNAL_CA_FILE' /tmp/blue-gateway-split-identity.yaml +if grep -q 'HARNESS_PROXY_CLIENT_IDENTITY_FILE' /tmp/blue-gateway-split-identity.yaml; then + echo "split client-secret render still emitted the combined identity variable" >&2 + exit 1 +fi +# The mount is what makes either layout readable; it is emitted under a separate +# mtls conditional in this template, so assert it alongside the split render. +grep -q 'name: internal-tls, mountPath: /var/run/blue/internal-tls' /tmp/blue-gateway-split-identity.yaml +grep -q 'secretName: "blue-internal-client"' /tmp/blue-gateway-split-identity.yaml +if helm template blue "$chart" \ + "${production_network[@]}" \ + "${gateway_jwt[@]}" \ + --set blue.existingSecret=blue-runtime \ + --set image.digest="$digest" \ + --set blue.enableInferenceProxy=true \ + --set blue.gatewayType=litellm \ + --set blue.internalTransport.serverSecret=blue-internal-server \ + --set blue.internalTransport.clientSecret=blue-internal-client \ + --set blue.internalTransport.clientSecretFormat=bogus >/dev/null 2>&1; then + echo "gateway render unexpectedly accepted an unknown clientSecretFormat" >&2 + exit 1 +fi + helm lint "$chart" -f "$chart/values-evaluation.yaml" helm template blue "$chart" -f "$chart/values-evaluation.yaml" > /tmp/blue-evaluation.yaml helm template blue "$chart" -f "$chart/values-evaluation.yaml" \ diff --git a/services/inference-proxy/src/main.rs b/services/inference-proxy/src/main.rs index 8631d0d..087abe4 100644 --- a/services/inference-proxy/src/main.rs +++ b/services/inference-proxy/src/main.rs @@ -372,14 +372,9 @@ async fn main() { let oauth = oauth_config_from_env(); let internal_transport = InternalTransportMode::from_env(); if internal_transport == InternalTransportMode::InsecureHttp - && [ - "HARNESS_INTERNAL_CA_PEM", - "HARNESS_INTERNAL_CA_FILE", - "HARNESS_PROXY_CLIENT_IDENTITY_PEM", - "HARNESS_PROXY_CLIENT_IDENTITY_FILE", - ] - .iter() - .any(|name| std::env::var(name).is_ok()) + && INTERNAL_TLS_ENV_VARS + .iter() + .any(|name| env_setting(name).is_some()) { panic!("internal TLS settings must not be configured in insecure-http mode"); } @@ -495,8 +490,7 @@ async fn main() { let (shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(false); if dynamic_mode && internal_transport == InternalTransportMode::Mtls - && std::env::var("HARNESS_INTERNAL_CA_FILE").is_ok() - && std::env::var("HARNESS_PROXY_CLIENT_IDENTITY_FILE").is_ok() + && tls_file_fingerprint().is_some() { tokio::spawn(control_tls_reload_worker( state.clone(), @@ -597,10 +591,7 @@ fn build_control_client(mode: InternalTransportMode) -> anyhow::Result Option<[u8; 32]> { - let ca = std::fs::read(std::env::var("HARNESS_INTERNAL_CA_FILE").ok()?).ok()?; - let identity = std::fs::read(std::env::var("HARNESS_PROXY_CLIENT_IDENTITY_FILE").ok()?).ok()?; let mut hash = Sha256::new(); - hash.update(ca); - hash.update(identity); - Some(hash.finalize().into()) + hash.update(std::fs::read(env_setting("HARNESS_INTERNAL_CA_FILE")?).ok()?); + let mut identity_files = 0; + for name in [ + "HARNESS_PROXY_CLIENT_IDENTITY_FILE", + "HARNESS_PROXY_CLIENT_CERT_FILE", + "HARNESS_PROXY_CLIENT_KEY_FILE", + ] { + if let Some(path) = env_setting(name) { + hash.update(std::fs::read(path).ok()?); + identity_files += 1; + } + } + (identity_files > 0).then(|| hash.finalize().into()) +} + +/// Every environment variable that carries internal mTLS material. Configuring +/// any of them in `insecure-http` mode is a hard error. +const INTERNAL_TLS_ENV_VARS: [&str; 8] = [ + "HARNESS_INTERNAL_CA_PEM", + "HARNESS_INTERNAL_CA_FILE", + "HARNESS_PROXY_CLIENT_IDENTITY_PEM", + "HARNESS_PROXY_CLIENT_IDENTITY_FILE", + "HARNESS_PROXY_CLIENT_CERT_PEM", + "HARNESS_PROXY_CLIENT_CERT_FILE", + "HARNESS_PROXY_CLIENT_KEY_PEM", + "HARNESS_PROXY_CLIENT_KEY_FILE", +]; + +/// Resolve the proxy's client identity into a single PEM bundle. +/// +/// Two shapes are accepted, and they are mutually exclusive so a half-migrated +/// deployment fails loudly instead of silently ignoring one of them: +/// +/// - combined `HARNESS_PROXY_CLIENT_IDENTITY_{PEM,FILE}` — certificate and key +/// in one file, Blue's original contract; +/// - split `HARNESS_PROXY_CLIENT_{CERT,KEY}_{PEM,FILE}` — what cert-manager, +/// Vault, SPIRE and `kubectl create secret tls` emit as `tls.crt` + `tls.key`. +/// +/// `reqwest::Identity::from_pkcs8_pem` (the two-buffer constructor) is +/// `native-tls`-only and Blue is rustls-only, so the split pair is concatenated +/// here and handed to the same `Identity::from_pem` the combined path uses. +/// That scan is order-independent, so the join order does not matter. +fn client_identity_pem(get: impl Fn(&str, &str) -> Option) -> Result { + let identity = get( + "HARNESS_PROXY_CLIENT_IDENTITY_PEM", + "HARNESS_PROXY_CLIENT_IDENTITY_FILE", + ); + let cert = get( + "HARNESS_PROXY_CLIENT_CERT_PEM", + "HARNESS_PROXY_CLIENT_CERT_FILE", + ); + let key = get( + "HARNESS_PROXY_CLIENT_KEY_PEM", + "HARNESS_PROXY_CLIENT_KEY_FILE", + ); + match (identity, cert, key) { + (Some(_), Some(_), _) | (Some(_), _, Some(_)) => Err( + "HARNESS_PROXY_CLIENT_IDENTITY_FILE/_PEM cannot be combined with \ + HARNESS_PROXY_CLIENT_CERT_FILE/_PEM or HARNESS_PROXY_CLIENT_KEY_FILE/_PEM; \ + configure either the combined identity or the split cert/key pair" + .into(), + ), + (None, Some(cert), Some(key)) => Ok(join_pem(&cert, &key)), + (None, Some(_), None) => Err( + "HARNESS_PROXY_CLIENT_KEY_FILE or HARNESS_PROXY_CLIENT_KEY_PEM is required \ + alongside HARNESS_PROXY_CLIENT_CERT_FILE/_PEM" + .into(), + ), + (None, None, Some(_)) => Err( + "HARNESS_PROXY_CLIENT_CERT_FILE or HARNESS_PROXY_CLIENT_CERT_PEM is required \ + alongside HARNESS_PROXY_CLIENT_KEY_FILE/_PEM" + .into(), + ), + (Some(identity), None, None) => Ok(identity), + (None, None, None) => Err( + "HARNESS_PROXY_CLIENT_IDENTITY_FILE or HARNESS_PROXY_CLIENT_IDENTITY_PEM \ + (or the HARNESS_PROXY_CLIENT_CERT_FILE/_PEM and HARNESS_PROXY_CLIENT_KEY_FILE/_PEM \ + pair) is required in mtls mode" + .into(), + ), + } +} + +/// Concatenate two PEM documents, inserting the separating newline that a file +/// written without a trailing one would otherwise be missing. +fn join_pem(first: &str, second: &str) -> String { + let first = first.trim_end(); + format!("{first}\n{second}") +} + +/// Read one environment variable, treating blank as unset. A variable set to +/// the empty string is how a deployment template says "not configured"; taking +/// it literally would trip the mode guards and the all-or-nothing checks. +fn env_setting(name: &str) -> Option { + std::env::var(name) + .ok() + .filter(|value| !value.trim().is_empty()) } fn pem_setting(value_name: &str, file_name: &str) -> Option { - std::env::var(value_name).ok().or_else(|| { - std::env::var(file_name).ok().map(|path| { + env_setting(value_name).or_else(|| { + env_setting(file_name).map(|path| { std::fs::read_to_string(&path) .unwrap_or_else(|error| panic!("reading {file_name} {path}: {error}")) }) @@ -2813,4 +2904,131 @@ mod tests { None ); } + /// Stand-ins for PEM documents. The functions under test only join and + /// select material, so real key bytes would add nothing but a secret-scan + /// finding; the handshake itself is covered end to end. + const CERT: &str = "--cert--\n"; + const KEY: &str = "--key--\n"; + + fn identity_from(pairs: &[(&str, &str)]) -> Result { + let pairs: Vec<(String, String)> = pairs + .iter() + .map(|(name, value)| ((*name).to_string(), (*value).to_string())) + .collect(); + client_identity_pem(|value_name, file_name| { + pairs + .iter() + .find(|(name, _)| name == value_name || name == file_name) + .map(|(_, value)| value.clone()) + }) + } + + #[test] + fn combined_identity_is_used_verbatim() { + let combined = format!("{CERT}{KEY}"); + assert_eq!( + identity_from(&[("HARNESS_PROXY_CLIENT_IDENTITY_FILE", &combined)]), + Ok(combined) + ); + } + + #[test] + fn split_pair_is_equivalent_to_the_combined_file() { + let combined = identity_from(&[( + "HARNESS_PROXY_CLIENT_IDENTITY_FILE", + &format!("{CERT}{KEY}"), + )]) + .expect("combined identity"); + let split = identity_from(&[ + ("HARNESS_PROXY_CLIENT_CERT_FILE", CERT), + ("HARNESS_PROXY_CLIENT_KEY_FILE", KEY), + ]) + .expect("split identity"); + assert_eq!(split, combined); + } + + #[test] + fn split_pair_separates_a_cert_file_with_no_trailing_newline() { + let split = identity_from(&[ + ("HARNESS_PROXY_CLIENT_CERT_PEM", "--cert--"), + ("HARNESS_PROXY_CLIENT_KEY_PEM", KEY), + ]) + .expect("split identity"); + assert_eq!(split, format!("{CERT}{KEY}")); + } + + #[test] + fn half_a_split_pair_names_the_missing_half() { + let missing_key = identity_from(&[("HARNESS_PROXY_CLIENT_CERT_FILE", CERT)]) + .expect_err("cert without key must fail"); + assert!( + missing_key.contains("HARNESS_PROXY_CLIENT_KEY_FILE"), + "{missing_key}" + ); + let missing_cert = identity_from(&[("HARNESS_PROXY_CLIENT_KEY_FILE", KEY)]) + .expect_err("key without cert must fail"); + assert!( + missing_cert.contains("HARNESS_PROXY_CLIENT_CERT_FILE"), + "{missing_cert}" + ); + } + + #[test] + fn combined_and_split_are_not_silently_reconciled() { + for extra in [ + ("HARNESS_PROXY_CLIENT_CERT_FILE", CERT), + ("HARNESS_PROXY_CLIENT_KEY_FILE", KEY), + ] { + let error = identity_from(&[ + ( + "HARNESS_PROXY_CLIENT_IDENTITY_FILE", + &format!("{CERT}{KEY}"), + ), + extra, + ]) + .expect_err("combined plus split must fail"); + assert!( + error.contains("HARNESS_PROXY_CLIENT_IDENTITY_FILE"), + "{error}" + ); + assert!(error.contains(extra.0), "{error}"); + } + } + + #[test] + fn no_client_identity_at_all_is_rejected() { + let error = identity_from(&[]).expect_err("mtls without an identity must fail"); + assert!( + error.contains("HARNESS_PROXY_CLIENT_IDENTITY_FILE"), + "{error}" + ); + assert!(error.contains("HARNESS_PROXY_CLIENT_CERT_FILE"), "{error}"); + } + + /// The insecure-http guard rejects TLS settings by name, so a new variable + /// that `client_identity_pem` reads must be added there too or it escapes + /// the guard entirely. + #[test] + fn insecure_http_guard_covers_every_client_identity_var() { + for suffix in ["IDENTITY", "CERT", "KEY"] { + for kind in ["FILE", "PEM"] { + let name = format!("HARNESS_PROXY_CLIENT_{suffix}_{kind}"); + assert!( + INTERNAL_TLS_ENV_VARS.contains(&name.as_str()), + "{name} is not guarded in insecure-http mode" + ); + } + } + } + + #[test] + fn blank_env_values_read_as_unset() { + let name = "BLUE_TEST_BLANK_ENV_SETTING"; + std::env::set_var(name, " "); + assert_eq!(env_setting(name), None); + std::env::set_var(name, "/certs/client.pem"); + assert_eq!(env_setting(name), Some("/certs/client.pem".to_string())); + std::env::remove_var(name); + assert_eq!(env_setting(name), None); + } } diff --git a/tests/e2e/README.md b/tests/e2e/README.md index 8fc15e3..36d149f 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -24,6 +24,18 @@ tests/e2e/run.sh full tests/e2e/run.sh mtls ``` +The proxy reads its client identity from the combined `client.pem` by default. +To run the same stack against the split cert/key pair instead — the layout +cert-manager, Vault, SPIRE, and `kubectl create secret tls` produce — point it +at the separate files the fixture already generates: + +```bash +BLUE_E2E_PROXY_IDENTITY_FILE= \ +BLUE_E2E_PROXY_CERT_FILE=/certs/client.crt \ +BLUE_E2E_PROXY_KEY_FILE=/certs/client.key \ +tests/e2e/run.sh mtls +``` + The smoke suite covers deployment health, dashboard password authentication, OAuth device approval, policy reconciliation, custom-provisioner invocation, encrypted gateway provisioning, managed packages, a governed Codex launch, and diff --git a/tests/e2e/docker-compose.yml b/tests/e2e/docker-compose.yml index a2ed77b..b16331f 100644 --- a/tests/e2e/docker-compose.yml +++ b/tests/e2e/docker-compose.yml @@ -160,7 +160,13 @@ services: HARNESS_INTERNAL_TLS_KEY_FILE: /certs/tls.key HARNESS_INTERNAL_TLS_CLIENT_CA_FILE: /certs/ca.crt HARNESS_INTERNAL_CA_FILE: /certs/ca.crt - HARNESS_PROXY_CLIENT_IDENTITY_FILE: /certs/client.pem + # The proxy accepts its client identity as one combined file or as a + # separate cert/key pair. Default to combined, the layout the chart ships; + # export BLUE_E2E_PROXY_IDENTITY_FILE= plus the CERT/KEY pair to run the + # same stack against the split path. Blank reads as unset. + HARNESS_PROXY_CLIENT_IDENTITY_FILE: ${BLUE_E2E_PROXY_IDENTITY_FILE-/certs/client.pem} + HARNESS_PROXY_CLIENT_CERT_FILE: ${BLUE_E2E_PROXY_CERT_FILE-} + HARNESS_PROXY_CLIENT_KEY_FILE: ${BLUE_E2E_PROXY_KEY_FILE-} HARNESS_GATEWAY_RESOLVER_URL: https://127.0.0.1:8082/internal/gateway/resolve HARNESS_GATEWAY_EVENT_URL: https://127.0.0.1:8082/internal/gateway/events HARNESS_GATEWAY_CREDENTIAL_INVALID_URL: https://127.0.0.1:8082/internal/gateway/credential-invalid diff --git a/tests/e2e/fixtures/mtls/generate.sh b/tests/e2e/fixtures/mtls/generate.sh index c0633ca..dd733dc 100644 --- a/tests/e2e/fixtures/mtls/generate.sh +++ b/tests/e2e/fixtures/mtls/generate.sh @@ -20,6 +20,9 @@ printf '%s\n' 'extendedKeyUsage=clientAuth' > "$out/client.ext" openssl x509 -req -days 2 -sha256 -in "$out/client.csr" \ -CA "$out/ca.crt" -CAkey "$out/ca.key" -CAcreateserial \ -extfile "$out/client.ext" -out "$out/client.crt" +# Keep all three: client.pem is the proxy's combined-identity default, and the +# separate files back both its split cert/key path and the Node specs, whose +# https client wants cert and key apart. cat "$out/client.crt" "$out/client.key" > "$out/client.pem" openssl req -x509 -newkey rsa:2048 -nodes -days 2 -sha256 \ From 70f71f5f447016374e3bd5ee56f9e7fdec2ccb17 Mon Sep 17 00:00:00 2001 From: Blocks Task Runner Date: Mon, 14 Sep 2026 22:23:39 +0000 Subject: [PATCH 2/4] test(proxy): drop a key-shaped binding name from the split-pair test CodeQL read `missing_key` as key material being written to a log. It holds an error message; loop over the two half-pair cases instead so the binding is just `error`. Co-Authored-By: Claude Opus 5 --- services/inference-proxy/src/main.rs | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/services/inference-proxy/src/main.rs b/services/inference-proxy/src/main.rs index 087abe4..880c12a 100644 --- a/services/inference-proxy/src/main.rs +++ b/services/inference-proxy/src/main.rs @@ -2959,18 +2959,20 @@ mod tests { #[test] fn half_a_split_pair_names_the_missing_half() { - let missing_key = identity_from(&[("HARNESS_PROXY_CLIENT_CERT_FILE", CERT)]) - .expect_err("cert without key must fail"); - assert!( - missing_key.contains("HARNESS_PROXY_CLIENT_KEY_FILE"), - "{missing_key}" - ); - let missing_cert = identity_from(&[("HARNESS_PROXY_CLIENT_KEY_FILE", KEY)]) - .expect_err("key without cert must fail"); - assert!( - missing_cert.contains("HARNESS_PROXY_CLIENT_CERT_FILE"), - "{missing_cert}" - ); + for (configured, absent) in [ + ( + "HARNESS_PROXY_CLIENT_CERT_FILE", + "HARNESS_PROXY_CLIENT_KEY_FILE", + ), + ( + "HARNESS_PROXY_CLIENT_KEY_FILE", + "HARNESS_PROXY_CLIENT_CERT_FILE", + ), + ] { + let error = + identity_from(&[(configured, CERT)]).expect_err("half a pair must be rejected"); + assert!(error.contains(absent), "{error}"); + } } #[test] From 251a97464bd4d268752ae41064629336167370b1 Mon Sep 17 00:00:00 2001 From: Blocks Task Runner Date: Mon, 14 Sep 2026 23:07:55 +0000 Subject: [PATCH 3/4] feat(deploy): let the chart issue the internal mTLS certificates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds blue.internalTransport.certManager so the chart declares the Issuer and Certificate resources itself. cert-manager then generates and renews both certificates, and the chart derives the two things a hand-written manifest gets wrong: the internal Service name in the server SAN, and the Secret names the pods mount. Leave issuerRef empty for a self-signed CA scoped to the release, or point it at an existing CA-type Issuer/ClusterIssuer. ACME issuers do not work — both sides read ca.crt out of their own Secret and ACME does not write it. serverSecret/clientSecret installs render byte-identically; their guard still fails when neither name nor certManager is set. Co-Authored-By: Claude Opus 5 --- apps/docs/0.1.0/concepts/gateway-mode.mdx | 5 + apps/docs/0.1.0/deployment/production.mdx | 39 +++++++ apps/docs/next/concepts/gateway-mode.mdx | 5 + apps/docs/next/deployment/production.mdx | 39 +++++++ deploy/helm/README.md | 17 ++- deploy/helm/templates/_helpers.tpl | 28 ++++- deploy/helm/templates/deployment.yaml | 2 +- .../templates/inference-proxy-deployment.yaml | 4 +- .../templates/internal-tls-certificates.yaml | 105 +++++++++++++++++ deploy/helm/values.schema.json | 34 +++++- deploy/helm/values.yaml | 19 ++++ scripts/verify-deployment.sh | 107 ++++++++++++++++++ 12 files changed, 391 insertions(+), 13 deletions(-) create mode 100644 deploy/helm/templates/internal-tls-certificates.yaml diff --git a/apps/docs/0.1.0/concepts/gateway-mode.mdx b/apps/docs/0.1.0/concepts/gateway-mode.mdx index 42bed31..10f0f59 100644 --- a/apps/docs/0.1.0/concepts/gateway-mode.mdx +++ b/apps/docs/0.1.0/concepts/gateway-mode.mdx @@ -123,6 +123,11 @@ setting both, or only half of the pair, is a startup error that names the offending variable. Rotating either file of a split pair is picked up by the same 30-second reload. +The Helm chart can issue this pair for you: with +`blue.internalTransport.certManager.enabled`, it declares the `Issuer` and +`Certificate` resources and cert-manager generates and renews both certificates, +so no Secret has to be made by hand. + `insecure-http` is available only as a deliberate trusted-network tradeoff. OAuth M2M and NetworkPolicy isolation remain mandatory, but virtual keys are plaintext on the wire and there is no client-certificate check. A VPC or an API diff --git a/apps/docs/0.1.0/deployment/production.mdx b/apps/docs/0.1.0/deployment/production.mdx index 038a2ce..8c72cf1 100644 --- a/apps/docs/0.1.0/deployment/production.mdx +++ b/apps/docs/0.1.0/deployment/production.mdx @@ -122,6 +122,44 @@ accepts plaintext east-west credentials. The processes check mounted certificate files every 30 seconds, adopt valid rotations without dropping in-flight requests, and retain the last-known-good configuration after an invalid update. +#### Let the chart issue the certificates + +If cert-manager is installed in the cluster, set +`blue.internalTransport.certManager.enabled=true` and skip the rest of this +section. The chart declares the `Issuer` and `Certificate` resources, names the +two Secrets after the release, and fills in the server SAN from the internal +Service name — the two values a hand-written manifest gets wrong. cert-manager +generates the keys and renews them; nothing secret goes into the release. + +```yaml +blue: + internalTransport: + mode: mtls + certManager: + enabled: true + # Omit issuerRef for a self-signed CA scoped to this release, or point it + # at your own PKI: + # issuerRef: { name: corporate-pki, kind: ClusterIssuer } +``` + +The issuer must be a CA-type issuer. Both sides read `ca.crt` out of their own +Secret, and cert-manager writes that key only when the CA is known — true for +`SelfSigned` and `CA` issuers, **false for ACME**. `issuerRef.kind` is required +alongside `issuerRef.name`: cert-manager defaults it to `Issuer`, which looks +only in the release namespace and leaves the Certificate stuck `Pending` if you +meant `ClusterIssuer`. + +Under `certManager`, `serverSecret`, `clientSecret`, and `clientSecretFormat` +are ignored. Installing cert-manager itself is out of the chart's scope: it is a +cluster-scoped singleton with its own CRDs. + +Turning `certManager.enabled` on for a running mTLS install is a CA cutover, not +an in-place upgrade. Both Secrets get new content from a new CA and they do not +swap atomically, so expect a short window of rejected handshakes; the 30-second +reload and last-known-good behaviour keep it from becoming an outage. + +#### Or bring your own certificates + Set both `blue.internalTransport.serverSecret` and `blue.internalTransport.clientSecret`; empty names render unusable Secret volumes. The server Secret holds `ca.crt`, `tls.crt`, and `tls.key`. The client Secret @@ -132,6 +170,7 @@ issuer produces: | --- | --- | --- | | `combined` (default) | `ca.crt`, `client.pem` | You build `client.pem` yourself by concatenating the certificate and its key | | `split` | `ca.crt`, `tls.crt`, `tls.key` | Your issuer emits the certificate and key as separate files | +| chart-issued (`certManager.enabled`) | `ca.crt`, `tls.crt`, `tls.key` | The chart declares the Certificates and cert-manager fills the Secrets; the layout is forced to `split` | Prefer `split` with an automated issuer. cert-manager, Vault, SPIRE, and `kubectl create secret tls` all write `tls.crt` + `tls.key`, so the Certificate diff --git a/apps/docs/next/concepts/gateway-mode.mdx b/apps/docs/next/concepts/gateway-mode.mdx index efb9857..f60a056 100644 --- a/apps/docs/next/concepts/gateway-mode.mdx +++ b/apps/docs/next/concepts/gateway-mode.mdx @@ -121,6 +121,11 @@ setting both, or only half of the pair, is a startup error that names the offending variable. Rotating either file of a split pair is picked up by the same 30-second reload. +The Helm chart can issue this pair for you: with +`blue.internalTransport.certManager.enabled`, it declares the `Issuer` and +`Certificate` resources and cert-manager generates and renews both certificates, +so no Secret has to be made by hand. + `insecure-http` is available only as a deliberate trusted-network tradeoff. OAuth M2M and NetworkPolicy isolation remain mandatory, but virtual keys are plaintext on the wire and there is no client-certificate check. A VPC or an API diff --git a/apps/docs/next/deployment/production.mdx b/apps/docs/next/deployment/production.mdx index a81f152..d11ae04 100644 --- a/apps/docs/next/deployment/production.mdx +++ b/apps/docs/next/deployment/production.mdx @@ -122,6 +122,44 @@ accepts plaintext east-west credentials. The processes check mounted certificate files every 30 seconds, adopt valid rotations without dropping in-flight requests, and retain the last-known-good configuration after an invalid update. +#### Let the chart issue the certificates + +If cert-manager is installed in the cluster, set +`blue.internalTransport.certManager.enabled=true` and skip the rest of this +section. The chart declares the `Issuer` and `Certificate` resources, names the +two Secrets after the release, and fills in the server SAN from the internal +Service name — the two values a hand-written manifest gets wrong. cert-manager +generates the keys and renews them; nothing secret goes into the release. + +```yaml +blue: + internalTransport: + mode: mtls + certManager: + enabled: true + # Omit issuerRef for a self-signed CA scoped to this release, or point it + # at your own PKI: + # issuerRef: { name: corporate-pki, kind: ClusterIssuer } +``` + +The issuer must be a CA-type issuer. Both sides read `ca.crt` out of their own +Secret, and cert-manager writes that key only when the CA is known — true for +`SelfSigned` and `CA` issuers, **false for ACME**. `issuerRef.kind` is required +alongside `issuerRef.name`: cert-manager defaults it to `Issuer`, which looks +only in the release namespace and leaves the Certificate stuck `Pending` if you +meant `ClusterIssuer`. + +Under `certManager`, `serverSecret`, `clientSecret`, and `clientSecretFormat` +are ignored. Installing cert-manager itself is out of the chart's scope: it is a +cluster-scoped singleton with its own CRDs. + +Turning `certManager.enabled` on for a running mTLS install is a CA cutover, not +an in-place upgrade. Both Secrets get new content from a new CA and they do not +swap atomically, so expect a short window of rejected handshakes; the 30-second +reload and last-known-good behaviour keep it from becoming an outage. + +#### Or bring your own certificates + Set both `blue.internalTransport.serverSecret` and `blue.internalTransport.clientSecret`; empty names render unusable Secret volumes. The server Secret holds `ca.crt`, `tls.crt`, and `tls.key`. The client Secret @@ -132,6 +170,7 @@ issuer produces: | --- | --- | --- | | `combined` (default) | `ca.crt`, `client.pem` | You build `client.pem` yourself by concatenating the certificate and its key | | `split` | `ca.crt`, `tls.crt`, `tls.key` | Your issuer emits the certificate and key as separate files | +| chart-issued (`certManager.enabled`) | `ca.crt`, `tls.crt`, `tls.key` | The chart declares the Certificates and cert-manager fills the Secrets; the layout is forced to `split` | Prefer `split` with an automated issuer. cert-manager, Vault, SPIRE, and `kubectl create secret tls` all write `tls.crt` + `tls.key`, so the Certificate diff --git a/deploy/helm/README.md b/deploy/helm/README.md index ad709cf..febfe63 100644 --- a/deploy/helm/README.md +++ b/deploy/helm/README.md @@ -92,16 +92,25 @@ ClusterIP Service on port 8082. Choose the transport explicitly with - `mtls` is the default and recommended mode. It encrypts decrypted virtual keys in transit and authenticates both workloads. Set - `blue.internalTransport.serverSecret` to a Secret containing `ca.crt`, + `blue.internalTransport.certManager.enabled=true` and the chart issues both + certificates itself: it declares the `Issuer` and `Certificate` resources, and + cert-manager generates the keys, fills in the server SAN the proxy dials, and + renews everything. Install the cert-manager CRDs in the cluster first; the + chart does not install cert-manager. Leave `certManager.issuerRef` empty for a + self-signed CA scoped to this release, or point it at your own + `Issuer`/`ClusterIssuer`. It must be a CA-type issuer — Blue reads `ca.crt` + out of each Secret, and ACME issuers do not write that key. Under + `certManager`, `serverSecret`, `clientSecret`, and `clientSecretFormat` are + ignored: the chart names its own Secrets and always reads the split layout. +- To bring your own certificates instead, leave `certManager.enabled=false` and + set `blue.internalTransport.serverSecret` to a Secret containing `ca.crt`, `tls.crt`, and `tls.key`; the server certificate SAN must cover `-control-api-internal`. Set `blue.internalTransport.clientSecret` to a Secret containing `ca.crt` plus the proxy identity, and pick its layout with `blue.internalTransport.clientSecretFormat`: `combined` (default) reads `client.pem`, the proxy certificate followed by its private key; `split` reads `tls.crt` and `tls.key`, which is what cert-manager, - Vault, SPIRE, and `kubectl create secret tls` emit. Use `split` with an - automated issuer — the Certificate then needs no `additionalOutputFormats` - stanza, which is only on by default from cert-manager 1.15. + Vault, SPIRE, and `kubectl create secret tls` emit. - `insecure-http` disables transport encryption and certificate authentication. OAuth M2M remains mandatory, but decrypted virtual keys cross the pod network in plaintext. Use it only on a private, trusted network with enforced diff --git a/deploy/helm/templates/_helpers.tpl b/deploy/helm/templates/_helpers.tpl index a7d515e..10026d6 100644 --- a/deploy/helm/templates/_helpers.tpl +++ b/deploy/helm/templates/_helpers.tpl @@ -63,6 +63,23 @@ ingress-fronted install is unconfigurable without them. {{- define "blue.provisionerExecutableImage" -}} {{- printf "%s@%s" .Values.blue.provisionerExecutable.image.repository .Values.blue.provisionerExecutable.image.digest }} {{- end }} +{{/* +Internal mTLS Secret names and client layout. When +blue.internalTransport.certManager.enabled the chart issues both Secrets itself +and names them after the release, so no consumer template branches on the flag. +cert-manager always writes tls.crt + tls.key, so the client layout is derived +rather than validated: Helm cannot tell a user-set "combined" from the default, +and failing on that pair would reject every ordinary cert-manager install. +*/}} +{{- define "blue.internalTransport.serverSecretName" -}} +{{- if .Values.blue.internalTransport.certManager.enabled }}{{ printf "%s-internal-tls-server" (include "blue.fullname" .) }}{{- else }}{{ .Values.blue.internalTransport.serverSecret }}{{- end }} +{{- end }} +{{- define "blue.internalTransport.clientSecretName" -}} +{{- if .Values.blue.internalTransport.certManager.enabled }}{{ printf "%s-internal-tls-client" (include "blue.fullname" .) }}{{- else }}{{ .Values.blue.internalTransport.clientSecret }}{{- end }} +{{- end }} +{{- define "blue.internalTransport.clientSecretFormat" -}} +{{- if .Values.blue.internalTransport.certManager.enabled }}split{{- else }}{{ .Values.blue.internalTransport.clientSecretFormat }}{{- end }} +{{- end }} {{- define "blue.validate" -}} {{- if and .Values.blue.production (not .Values.blue.existingSecret) -}} {{- fail "blue.existingSecret is required when blue.production=true" -}} @@ -94,7 +111,14 @@ ingress-fronted install is unconfigurable without them. {{- if and .Values.blue.enableInferenceProxy (not .Values.blue.gatewayType) -}} {{- fail "blue.gatewayType is required when blue.enableInferenceProxy=true" -}} {{- end -}} -{{- if and .Values.blue.enableInferenceProxy (eq .Values.blue.internalTransport.mode "mtls") (or (not .Values.blue.internalTransport.serverSecret) (not .Values.blue.internalTransport.clientSecret)) -}} -{{- fail "blue.internalTransport.serverSecret and clientSecret are required for gateway mTLS" -}} +{{- if and .Values.blue.enableInferenceProxy (eq .Values.blue.internalTransport.mode "mtls") (not .Values.blue.internalTransport.certManager.enabled) (or (not .Values.blue.internalTransport.serverSecret) (not .Values.blue.internalTransport.clientSecret)) -}} +{{- fail "blue.internalTransport.serverSecret and clientSecret are required for gateway mTLS unless blue.internalTransport.certManager.enabled=true" -}} +{{- end -}} +{{- if and .Values.blue.internalTransport.certManager.enabled (ne .Values.blue.internalTransport.mode "mtls") -}} +{{- fail "blue.internalTransport.certManager.enabled requires blue.internalTransport.mode=mtls; nothing consumes the certificates otherwise" -}} +{{- end -}} +{{- $issuerRef := default dict .Values.blue.internalTransport.certManager.issuerRef -}} +{{- if and $issuerRef.name (not $issuerRef.kind) -}} +{{- fail "blue.internalTransport.certManager.issuerRef.kind is required alongside issuerRef.name; set Issuer or ClusterIssuer explicitly" -}} {{- end -}} {{- end }} diff --git a/deploy/helm/templates/deployment.yaml b/deploy/helm/templates/deployment.yaml index 7adbd26..85c405a 100644 --- a/deploy/helm/templates/deployment.yaml +++ b/deploy/helm/templates/deployment.yaml @@ -165,7 +165,7 @@ spec: {{- end }} {{- if and .Values.blue.enableInferenceProxy (eq .Values.blue.internalTransport.mode "mtls") }} - name: internal-tls - secret: { secretName: {{ .Values.blue.internalTransport.serverSecret | quote }} } + secret: { secretName: {{ include "blue.internalTransport.serverSecretName" . | quote }} } {{- end }} {{- with .Values.blue.config.existingConfigMap }} - name: blue-config diff --git a/deploy/helm/templates/inference-proxy-deployment.yaml b/deploy/helm/templates/inference-proxy-deployment.yaml index 4d129d8..3c11903 100644 --- a/deploy/helm/templates/inference-proxy-deployment.yaml +++ b/deploy/helm/templates/inference-proxy-deployment.yaml @@ -63,7 +63,7 @@ spec: value: {{ printf "%s/api/auth/oauth2/token" (.Values.blue.publicUrls.dashboard | trimSuffix "/") | quote }} {{- if eq .Values.blue.internalTransport.mode "mtls" }} - { name: HARNESS_INTERNAL_CA_FILE, value: /var/run/blue/internal-tls/ca.crt } - {{- if eq .Values.blue.internalTransport.clientSecretFormat "split" }} + {{- if eq (include "blue.internalTransport.clientSecretFormat" .) "split" }} - { name: HARNESS_PROXY_CLIENT_CERT_FILE, value: /var/run/blue/internal-tls/tls.crt } - { name: HARNESS_PROXY_CLIENT_KEY_FILE, value: /var/run/blue/internal-tls/tls.key } {{- else }} @@ -94,6 +94,6 @@ spec: {{- if eq .Values.blue.internalTransport.mode "mtls" }} volumes: - name: internal-tls - secret: { secretName: {{ .Values.blue.internalTransport.clientSecret | quote }} } + secret: { secretName: {{ include "blue.internalTransport.clientSecretName" . | quote }} } {{- end }} {{- end }} diff --git a/deploy/helm/templates/internal-tls-certificates.yaml b/deploy/helm/templates/internal-tls-certificates.yaml new file mode 100644 index 0000000..1e98dfb --- /dev/null +++ b/deploy/helm/templates/internal-tls-certificates.yaml @@ -0,0 +1,105 @@ +{{- include "blue.validate" . -}} +{{/* +Chart-issued certificates for the internal proxy <-> Control API mTLS hop. +Requires the cert-manager.io/v1 CRDs in the cluster; the chart declares intent +and cert-manager generates and renews the key material. + +Gated on a plain values flag rather than .Capabilities.APIVersions, which is +false under `helm template` with no cluster and would silently drop every +Certificate from rendered output. +*/}} +{{- if and .Values.blue.enableInferenceProxy (eq .Values.blue.internalTransport.mode "mtls") .Values.blue.internalTransport.certManager.enabled }} +{{- $cm := .Values.blue.internalTransport.certManager }} +{{- $issuerRef := default dict $cm.issuerRef }} +{{- $fullname := include "blue.fullname" . }} +{{- if not $issuerRef.name }} +{{- $issuerRef = dict "name" (printf "%s-internal-ca" $fullname) "kind" "Issuer" }} +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ $fullname }}-internal-selfsigned + labels: + {{- include "blue.labels" . | nindent 4 }} + app.kubernetes.io/component: internal-tls +spec: + selfSigned: {} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ $fullname }}-internal-ca + labels: + {{- include "blue.labels" . | nindent 4 }} + app.kubernetes.io/component: internal-tls +spec: + isCA: true + commonName: {{ $fullname }}-internal-ca + secretName: {{ $fullname }}-internal-ca + duration: {{ $cm.caDuration | quote }} + privateKey: + algorithm: {{ $cm.privateKey.algorithm }} + size: {{ $cm.privateKey.size }} + issuerRef: + name: {{ $fullname }}-internal-selfsigned + kind: Issuer +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ $fullname }}-internal-ca + labels: + {{- include "blue.labels" . | nindent 4 }} + app.kubernetes.io/component: internal-tls +spec: + ca: + secretName: {{ $fullname }}-internal-ca +--- +{{- end }} +{{/* +The server SAN is not cosmetic: it must cover the internal Service name the +proxy dials (templates/service.yaml). The client CN is cosmetic — the Control +API only validates the chain and pins workload identity at the OAuth layer. +*/}} +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ include "blue.internalTransport.serverSecretName" . }} + labels: + {{- include "blue.labels" . | nindent 4 }} + app.kubernetes.io/component: internal-tls +spec: + secretName: {{ include "blue.internalTransport.serverSecretName" . }} + commonName: {{ $fullname }}-control-api-internal + dnsNames: + - {{ $fullname }}-control-api-internal + - {{ printf "%s-control-api-internal.%s.svc" $fullname .Release.Namespace }} + usages: [digital signature, key encipherment, server auth] + duration: {{ $cm.duration | quote }} + renewBefore: {{ $cm.renewBefore | quote }} + privateKey: + algorithm: {{ $cm.privateKey.algorithm }} + size: {{ $cm.privateKey.size }} + issuerRef: + name: {{ $issuerRef.name }} + kind: {{ $issuerRef.kind }} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ include "blue.internalTransport.clientSecretName" . }} + labels: + {{- include "blue.labels" . | nindent 4 }} + app.kubernetes.io/component: internal-tls +spec: + secretName: {{ include "blue.internalTransport.clientSecretName" . }} + commonName: {{ .Values.blue.inferenceProxyClientId }} + usages: [digital signature, key encipherment, client auth] + duration: {{ $cm.duration | quote }} + renewBefore: {{ $cm.renewBefore | quote }} + privateKey: + algorithm: {{ $cm.privateKey.algorithm }} + size: {{ $cm.privateKey.size }} + issuerRef: + name: {{ $issuerRef.name }} + kind: {{ $issuerRef.kind }} +{{- end }} diff --git a/deploy/helm/values.schema.json b/deploy/helm/values.schema.json index a3f49ba..3910b69 100644 --- a/deploy/helm/values.schema.json +++ b/deploy/helm/values.schema.json @@ -37,7 +37,32 @@ "mode": { "enum": ["mtls", "insecure-http"] }, "serverSecret": { "type": "string" }, "clientSecret": { "type": "string" }, - "clientSecretFormat": { "enum": ["combined", "split"] } + "clientSecretFormat": { "enum": ["combined", "split"] }, + "certManager": { + "type": "object", + "required": ["enabled"], + "properties": { + "enabled": { "type": "boolean" }, + "issuerRef": { + "type": "object", + "properties": { + "name": { "type": "string", "minLength": 1 }, + "kind": { "enum": ["Issuer", "ClusterIssuer"] } + } + }, + "duration": { "type": "string", "minLength": 1 }, + "renewBefore": { "type": "string", "minLength": 1 }, + "caDuration": { "type": "string", "minLength": 1 }, + "privateKey": { + "type": "object", + "required": ["algorithm", "size"], + "properties": { + "algorithm": { "enum": ["RSA", "ECDSA", "Ed25519"] }, + "size": { "type": "integer" } + } + } + } + } } }, "existingSecret": { "type": "string" }, @@ -102,9 +127,10 @@ } } } }, { "if": { "properties": { "internalTransport": { "properties": { "mode": { "const": "mtls" } } } } }, - "then": { "properties": { "internalTransport": { "properties": { - "serverSecret": { "minLength": 1 }, "clientSecret": { "minLength": 1 } - } } } } + "then": { "properties": { "internalTransport": { "anyOf": [ + { "required": ["certManager"], "properties": { "certManager": { "required": ["enabled"], "properties": { "enabled": { "const": true } } } } }, + { "properties": { "serverSecret": { "minLength": 1 }, "clientSecret": { "minLength": 1 } } } + ] } } } }] } }, { "if": { "properties": { "production": { "const": true } } }, diff --git a/deploy/helm/values.yaml b/deploy/helm/values.yaml index 9f8ea12..3753093 100644 --- a/deploy/helm/values.yaml +++ b/deploy/helm/values.yaml @@ -36,6 +36,7 @@ blue: mode: mtls # Server Secret: ca.crt, tls.crt, tls.key. Client Secret: ca.crt plus the # proxy identity in the layout named by clientSecretFormat. + # Both are ignored when certManager.enabled; the chart names its own Secrets. serverSecret: "" clientSecret: "" # Layout of the client Secret's identity material: @@ -43,7 +44,25 @@ blue: # split - tls.crt and tls.key, what cert-manager, Vault, SPIRE and # `kubectl create secret tls` emit. No additionalOutputFormats # stanza, so no cert-manager 1.15 floor. + # Ignored when certManager.enabled; cert-manager always writes a split pair. clientSecretFormat: combined + certManager: + # Requires cert-manager in the cluster. The chart only declares Issuer and + # Certificate resources; cert-manager generates and renews the keys. + enabled: false + # Leave issuerRef empty and the chart bootstraps a self-signed CA scoped to + # this release. Point it at your own Issuer/ClusterIssuer to use an + # existing PKI; it must be a CA-type issuer, because Blue needs the + # `ca.crt` key that ACME issuers do not write. + issuerRef: {} + # name: corporate-pki + # kind: ClusterIssuer + duration: 2160h # 90 days + renewBefore: 360h # 15 days + caDuration: 87600h # 10 years; only for the chart-created CA + privateKey: + algorithm: RSA + size: 2048 existingSecret: "" config: existingConfigMap: "" diff --git a/scripts/verify-deployment.sh b/scripts/verify-deployment.sh index 3d974d2..62ba611 100755 --- a/scripts/verify-deployment.sh +++ b/scripts/verify-deployment.sh @@ -184,6 +184,113 @@ if helm template blue "$chart" \ exit 1 fi +# cert-manager issues both internal Secrets from the chart, so the render must +# succeed with no serverSecret/clientSecret set at all. The negative above keeps +# that relaxation from turning into a missing guard: without certManager the +# same value set still has to fail. +cert_manager=( + --set blue.internalTransport.certManager.enabled=true +) +helm template blue "$chart" \ + "${production_network[@]}" \ + "${gateway_jwt[@]}" \ + "${cert_manager[@]}" \ + --set blue.existingSecret=blue-runtime \ + --set image.digest="$digest" \ + --set blue.enableInferenceProxy=true \ + --set blue.gatewayType=litellm \ + > /tmp/blue-gateway-cert-manager.yaml +if [ "$(grep -c '^kind: Certificate$' /tmp/blue-gateway-cert-manager.yaml)" != 3 ]; then + echo "cert-manager render did not emit the CA plus both leaf Certificates" >&2 + exit 1 +fi +if [ "$(grep -c '^kind: Issuer$' /tmp/blue-gateway-cert-manager.yaml)" != 2 ]; then + echo "cert-manager render did not emit the self-signed and CA Issuers" >&2 + exit 1 +fi +# The Secret names the workloads mount must be the ones the Certificates write. +grep -q 'secretName: blue-blue-internal-tls-server' /tmp/blue-gateway-cert-manager.yaml +grep -q 'secretName: blue-blue-internal-tls-client' /tmp/blue-gateway-cert-manager.yaml +grep -q 'secretName: "blue-blue-internal-tls-server"' /tmp/blue-gateway-cert-manager.yaml +grep -q 'secretName: "blue-blue-internal-tls-client"' /tmp/blue-gateway-cert-manager.yaml +# The server SAN must cover the internal Service name the proxy dials; a wrong +# one fails the handshake at runtime, not at render time. +grep -q -- '- blue-blue-control-api-internal$' /tmp/blue-gateway-cert-manager.yaml +grep -q -- '- blue-blue-control-api-internal.default.svc$' /tmp/blue-gateway-cert-manager.yaml +# cert-manager writes tls.crt + tls.key, so the proxy must read the split pair. +grep -q 'name: HARNESS_PROXY_CLIENT_CERT_FILE' /tmp/blue-gateway-cert-manager.yaml +grep -q 'name: HARNESS_PROXY_CLIENT_KEY_FILE' /tmp/blue-gateway-cert-manager.yaml +if grep -q 'HARNESS_PROXY_CLIENT_IDENTITY_FILE' /tmp/blue-gateway-cert-manager.yaml; then + echo "cert-manager render emitted the combined identity variable" >&2 + exit 1 +fi +# clientSecretFormat is derived, not validated: Helm cannot tell a user-set +# "combined" from the default, so an explicit one must not break the install. +helm template blue "$chart" \ + "${production_network[@]}" \ + "${gateway_jwt[@]}" \ + "${cert_manager[@]}" \ + --set blue.existingSecret=blue-runtime \ + --set image.digest="$digest" \ + --set blue.enableInferenceProxy=true \ + --set blue.gatewayType=litellm \ + --set blue.internalTransport.clientSecretFormat=combined \ + > /tmp/blue-gateway-cert-manager-combined.yaml +grep -q 'name: HARNESS_PROXY_CLIENT_CERT_FILE' /tmp/blue-gateway-cert-manager-combined.yaml +if grep -q 'HARNESS_PROXY_CLIENT_IDENTITY_FILE' /tmp/blue-gateway-cert-manager-combined.yaml; then + echo "cert-manager render honoured clientSecretFormat=combined" >&2 + exit 1 +fi +# An operator's own CA issuer replaces the chart's bootstrap chain entirely. +helm template blue "$chart" \ + "${production_network[@]}" \ + "${gateway_jwt[@]}" \ + "${cert_manager[@]}" \ + --set blue.existingSecret=blue-runtime \ + --set image.digest="$digest" \ + --set blue.enableInferenceProxy=true \ + --set blue.gatewayType=litellm \ + --set blue.internalTransport.certManager.issuerRef.name=corporate-pki \ + --set blue.internalTransport.certManager.issuerRef.kind=ClusterIssuer \ + > /tmp/blue-gateway-cert-manager-issuer.yaml +if grep -Eq 'name: blue-blue-internal-(selfsigned|ca)$' /tmp/blue-gateway-cert-manager-issuer.yaml; then + echo "external issuerRef render still bootstrapped the chart's own CA" >&2 + exit 1 +fi +if [ "$(grep -c 'name: corporate-pki' /tmp/blue-gateway-cert-manager-issuer.yaml)" != 2 ]; then + echo "external issuerRef render did not point both Certificates at it" >&2 + exit 1 +fi +grep -q 'kind: ClusterIssuer' /tmp/blue-gateway-cert-manager-issuer.yaml +# cert-manager defaults a missing kind to Issuer, which looks only in the +# release namespace and leaves the Certificate stuck Pending. Demand it. +if helm template blue "$chart" \ + "${production_network[@]}" \ + "${gateway_jwt[@]}" \ + "${cert_manager[@]}" \ + --set blue.existingSecret=blue-runtime \ + --set image.digest="$digest" \ + --set blue.enableInferenceProxy=true \ + --set blue.gatewayType=litellm \ + --set blue.internalTransport.certManager.issuerRef.name=corporate-pki >/dev/null 2>&1; then + echo "cert-manager render unexpectedly accepted an issuerRef without a kind" >&2 + exit 1 +fi +# Nothing consumes the certificates under insecure-http; issuing them silently +# is worse than saying so. +if helm template blue "$chart" \ + "${production_network[@]}" \ + "${gateway_jwt[@]}" \ + "${cert_manager[@]}" \ + --set blue.existingSecret=blue-runtime \ + --set image.digest="$digest" \ + --set blue.enableInferenceProxy=true \ + --set blue.gatewayType=litellm \ + --set blue.internalTransport.mode=insecure-http >/dev/null 2>&1; then + echo "cert-manager render unexpectedly accepted insecure-http" >&2 + exit 1 +fi + helm lint "$chart" -f "$chart/values-evaluation.yaml" helm template blue "$chart" -f "$chart/values-evaluation.yaml" > /tmp/blue-evaluation.yaml helm template blue "$chart" -f "$chart/values-evaluation.yaml" \ From b356a8bced3021426d73a28db39892ba73415ff0 Mon Sep 17 00:00:00 2001 From: Alejandro Esquivel Date: Tue, 15 Sep 2026 14:23:57 -0700 Subject: [PATCH 4/4] feat(deploy): reserve an inference proxy hostname in the AWS module Adds an optional inference_proxy_subdomain. When set, the name joins the ACM certificate's SANs, gets an alias record, and is output as inference_proxy_hostname. Empty by default, so existing plans are unchanged. Co-Authored-By: Claude Opus 5 (1M context) --- deploy/tofu/aws/README.md | 6 ++++-- deploy/tofu/aws/domain.tf | 19 ++++++++++++++----- deploy/tofu/aws/outputs.tf | 1 + deploy/tofu/aws/terraform.tfvars.example | 1 + deploy/tofu/aws/variables.tf | 5 +++++ 5 files changed, 25 insertions(+), 7 deletions(-) diff --git a/deploy/tofu/aws/README.md b/deploy/tofu/aws/README.md index 64a2f84..95b3cd9 100644 --- a/deploy/tofu/aws/README.md +++ b/deploy/tofu/aws/README.md @@ -110,11 +110,13 @@ have no fixed range. `include_domain` takes `route53_zone_id` plus two labels, `dashboard_subdomain` and `api_subdomain`, relative to that zone (`app` and `api` on `example.com` give `app.example.com` and `api.example.com`; an empty label means the apex). +An optional `inference_proxy_subdomain` adds a third name for gateway mode; +reserving it up front costs nothing and saves reissuing the certificate later. The zone's name is read back, so nothing repeats the domain. The load balancer only exists after the chart's Ingress is installed, so the records are a second apply: `tofu apply -var alb_hostname=`. `certificate_arn`, -`dashboard_hostname` and `api_hostname` are output for the chart and -`IngressClassParams`. +`dashboard_hostname`, `api_hostname` and `inference_proxy_hostname` are output +for the chart and `IngressClassParams`. ## Naming and tags diff --git a/deploy/tofu/aws/domain.tf b/deploy/tofu/aws/domain.tf index faf3800..ab90475 100644 --- a/deploy/tofu/aws/domain.tf +++ b/deploy/tofu/aws/domain.tf @@ -15,6 +15,10 @@ data "aws_route53_zone" "blue" { condition = var.dashboard_subdomain != var.api_subdomain error_message = "dashboard_subdomain and api_subdomain must differ; both cannot be the same name." } + precondition { + condition = var.inference_proxy_subdomain == "" || (var.inference_proxy_subdomain != var.dashboard_subdomain && var.inference_proxy_subdomain != var.api_subdomain) + error_message = "inference_proxy_subdomain must differ from dashboard_subdomain and api_subdomain." + } } } @@ -22,12 +26,20 @@ locals { zone_name = var.include_domain ? trimsuffix(data.aws_route53_zone.blue[0].name, ".") : "" dashboard_hostname = var.include_domain ? (var.dashboard_subdomain == "" ? local.zone_name : "${var.dashboard_subdomain}.${local.zone_name}") : null api_hostname = var.include_domain ? (var.api_subdomain == "" ? local.zone_name : "${var.api_subdomain}.${local.zone_name}") : null + # Optional third name for gateway mode. Reserving it costs nothing, so a + # deployment can add the inference proxy later without reissuing the certificate. + inference_proxy_hostname = var.include_domain && var.inference_proxy_subdomain != "" ? "${var.inference_proxy_subdomain}.${local.zone_name}" : null + # Every hostname the load balancer answers for, keyed for the alias records. + hostnames = merge( + var.include_domain ? { dashboard = local.dashboard_hostname, api = local.api_hostname } : {}, + local.inference_proxy_hostname != null ? { inference_proxy = local.inference_proxy_hostname } : {}, + ) } resource "aws_acm_certificate" "blue" { count = var.include_domain ? 1 : 0 domain_name = local.dashboard_hostname - subject_alternative_names = [local.api_hostname] + subject_alternative_names = compact([local.api_hostname, local.inference_proxy_hostname]) validation_method = "DNS" lifecycle { create_before_destroy = true } @@ -65,10 +77,7 @@ data "aws_elb_hosted_zone_id" "alb" { resource "aws_route53_record" "blue" { #checkov:skip=CKV2_AWS_23:the alias target is var.alb_hostname, an ALB created outside this module by the AWS Load Balancer Controller; checkov renders the variable to its "" default and so misses the check's own var. escape hatch - for_each = var.include_domain && var.alb_hostname != "" ? { - dashboard = local.dashboard_hostname - api = local.api_hostname - } : {} + for_each = var.alb_hostname != "" ? local.hostnames : {} zone_id = var.route53_zone_id name = each.value diff --git a/deploy/tofu/aws/outputs.tf b/deploy/tofu/aws/outputs.tf index 2d2480a..cf56e66 100644 --- a/deploy/tofu/aws/outputs.tf +++ b/deploy/tofu/aws/outputs.tf @@ -11,6 +11,7 @@ output "kms_key_arn" { value = aws_kms_key.blue.arn } output "certificate_arn" { value = var.include_domain ? aws_acm_certificate_validation.blue[0].certificate_arn : null } output "dashboard_hostname" { value = local.dashboard_hostname } output "api_hostname" { value = local.api_hostname } +output "inference_proxy_hostname" { value = local.inference_proxy_hostname } output "cluster_name" { value = local.cluster_name } output "cluster_endpoint" { value = local.create_cluster ? aws_eks_cluster.blue[0].endpoint : null } output "vpc_id" { value = local.vpc_id } diff --git a/deploy/tofu/aws/terraform.tfvars.example b/deploy/tofu/aws/terraform.tfvars.example index f4698ef..c8a7e54 100644 --- a/deploy/tofu/aws/terraform.tfvars.example +++ b/deploy/tofu/aws/terraform.tfvars.example @@ -39,6 +39,7 @@ environment = "production" # route53_zone_id = "Z0123456789EXAMPLE" # dashboard_subdomain = "app" # api_subdomain = "api" +# inference_proxy_subdomain = "iproxy" # optional; gateway mode's third name kubernetes_namespace = "blue" kubernetes_service_account = "blue" diff --git a/deploy/tofu/aws/variables.tf b/deploy/tofu/aws/variables.tf index b2943a2..ab741a6 100644 --- a/deploy/tofu/aws/variables.tf +++ b/deploy/tofu/aws/variables.tf @@ -58,6 +58,11 @@ variable "api_subdomain" { default = "api" description = "Control API label relative to the zone. Empty means the zone apex; it must differ from dashboard_subdomain." } +variable "inference_proxy_subdomain" { + type = string + default = "" + description = "Inference proxy label relative to the zone, used in gateway mode. Empty means no proxy hostname: it is left off the certificate and gets no record." +} variable "alb_hostname" { type = string default = ""