diff --git a/CHANGELOG.md b/CHANGELOG.md index ed902e3..7636796 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to this project are documented here, grouped by tagged versi ## [UNRELEASED] - Under development +### Added +- Rewrite Lab Instance service links to a reverse-proxy vhost + (`{port}-{service-name}.{PROXY_DOMAIN}`) when the vhost responds, controlled + by the new `PROXY_DOMAIN` setting. + ## [2.0.15] - 2026-08-27 ### Fixed diff --git a/apps/config.py b/apps/config.py index 2cfa0c0..c0869fb 100644 --- a/apps/config.py +++ b/apps/config.py @@ -178,6 +178,11 @@ class Config(object): LAB_EXPIRATION_WARN_SEC = 48*60*60 LAB_EXPIRATION_TOLERANACE_SEC = 48*60*60 + # Reverse-proxy vhost domain. When set, service links in the Lab Instance + # view are probed against "{port}-{service-name}.{PROXY_DOMAIN}" and swapped + # to that URL when the vhost responds. Empty disables the behavior. + PROXY_DOMAIN = os.getenv("PROXY_DOMAIN", "") + #Testbed infos TESTBED_TITLE = os.getenv("TESTBED_TITLE", "HackInSDN Testbed") TESTBED_NODES_GEOTAG = { diff --git a/apps/controllers/kubernetes.py b/apps/controllers/kubernetes.py index 25f1001..3c2467b 100644 --- a/apps/controllers/kubernetes.py +++ b/apps/controllers/kubernetes.py @@ -513,6 +513,7 @@ def get_lab_resources(self, resources, published_ports={}): owners.update(rs_uid_to_dep.keys()) pod_services = {} + pod_services_proxy = {} pod_names = {} app_pod_map = defaultdict(list) pods_by_uid = {} @@ -534,6 +535,7 @@ def get_lab_resources(self, resources, published_ports={}): if pod.metadata.uid not in owners and not is_child: continue pod_services[pod.metadata.uid] = [] + pod_services_proxy[pod.metadata.uid] = [] statuses = [ status.ready for status in pod.status.container_statuses ] @@ -557,6 +559,10 @@ def get_lab_resources(self, resources, published_ports={}): "pod_ip": pod.status.pod_ip, "containers": containers, "services": pod_services[pod.metadata.uid], + # reverse-proxy vhost prefix ("{scheme}://{port}-{svc-name}") for + # each entry in "services", index-aligned; the view appends the + # configured PROXY_DOMAIN and probes it client-side + "services_proxy": pod_services_proxy[pod.metadata.uid], "phase": pod.status.phase, "labels": pod_labels, "more": str(pod), @@ -625,6 +631,9 @@ def get_lab_resources(self, resources, published_ports={}): ] if port.node_port: pod_services[pod.metadata.uid].append(service_link) + pod_services_proxy[pod.metadata.uid].append( + f"{self.try_get_app(port_name)}{port.port}-{srv.metadata.name}" + ) return labs diff --git a/apps/home/routes.py b/apps/home/routes.py index 1f437a1..4945f2e 100644 --- a/apps/home/routes.py +++ b/apps/home/routes.py @@ -441,6 +441,10 @@ def view_lab_instance(lab_id): "created": "--", "expires_at": datetime_from_ts(lab_instance.expiration_ts), } + # reverse-proxy vhost domain (optional); when set, each web service link + # gets a candidate "https://{port}-{svc-name}.{PROXY_DOMAIN}" URL that the + # page probes client-side and swaps in if the vhost responds + proxy_domain = current_app.config.get("PROXY_DOMAIN", "") created = None #for pod in running_labs[(lab_instance.lab_id, owner.uid)]: for pod in lab_resources: @@ -448,6 +452,19 @@ def view_lab_instance(lab_id): # continue if not created or created > pod['created']: created = pod["created"] + # build the per-service proxy URL, index-aligned with "services"; only + # http(s) services get one (ssh/vnc are left pointing at the NodePort). + # The proxy URL is always https:// regardless of the backend scheme: the + # nginx reverse proxy terminates TLS and translates https->http to the + # NodePort, and an http:// link would be blocked as mixed content on the + # https dashboard page. + services_proxy = [] + for prefix in pod.get("services_proxy", []): + if proxy_domain and (prefix.startswith("http://") or prefix.startswith("https://")): + host = prefix.split("://", 1)[1] + services_proxy.append(f"https://{host}.{proxy_domain}") + else: + services_proxy.append("") lab_dict["resources"].append({ "kind": "pod", "name": pod["name"], @@ -455,6 +472,7 @@ def view_lab_instance(lab_id): "ready": pod["phase"], "links": pod["containers"], "services": pod["services"], + "services_proxy": services_proxy, "age": pod['age'], "node_name": pod.get("node_name", "--"), "pod_ip": pod.get("pod_ip", "--"), diff --git a/apps/templates/pages/lab_instance_view.html b/apps/templates/pages/lab_instance_view.html index ac4e76e..1578d64 100644 --- a/apps/templates/pages/lab_instance_view.html +++ b/apps/templates/pages/lab_instance_view.html @@ -160,7 +160,8 @@

{{ _('Lab title: %(title)s / Lab owner:
{% for service in resource['services'] %} - {{ service[0] }} + {% set proxy_url = resource['services_proxy'][loop.index0] if resource['services_proxy']|length > loop.index0 else '' %} + {{ service[0] }} {% endfor %}
@@ -521,6 +522,58 @@

{{ _('Lab Guide') }}

// here; run immediately instead of waiting for window "load" (which can // be delayed by images or may have already fired) loadAnswers(); + // ---- reverse-proxy vhost links ---- + // Each web service link may carry a candidate reverse-proxy URL + // ("https://{port}-{svc-name}.{PROXY_DOMAIN}"). Probe it and, if the + // vhost is reachable, use it as the actual link; on any failure the + // original NodePort link is left untouched. + // + // The probe uses mode:'no-cors' on purpose. The proxied lab services are + // a different origin than the dashboard and don't emit CORS headers, and + // many require auth (GET / returns 401) -- a CORS-mode probe can't read + // any of those responses and just throws. An opaque no-cors response + // can't expose the status, but that's fine: a reachable vhost (200/302/ + // 401/...) resolves the promise, while a hostname with no configured + // vhost is rejected by the reverse proxy's default server ("return 444", + // see scripts/k8s_nginx_revproxy/NGINX_SETUP.md) and makes fetch reject. + // So "promise resolved" == "vhost exists" -> swap. + function probeProxyLink(anchor) { + var proxyUrl = anchor.getAttribute('data-proxy-url'); + if (!proxyUrl || typeof fetch !== 'function') { + return; + } + var controller = (typeof AbortController === 'function') ? new AbortController() : null; + var timer = controller ? setTimeout(function () { controller.abort(); }, 5000) : null; + fetch(proxyUrl, { + method: 'GET', + mode: 'no-cors', + signal: controller ? controller.signal : undefined + }).then(function (resp) { + if (timer) { clearTimeout(timer); } + // In mode:'no-cors' a cross-origin response is "opaque": resp.type is + // 'opaque' (or 'opaqueredirect'), resp.status is always 0 and resp.ok + // is always false, so the status-based checks below can't actually + // read it -- the opaque branch is what fires here, and that is the + // whole detection mechanism (a resolved opaque response means the + // vhost exists; a missing vhost is closed by the default server and + // rejects instead). The explicit 2xx/3xx checks are kept anyway: + // they are harmless (opaque still triggers the swap) and keep the + // intent clear / correct should the probe ever be switched back to + // CORS mode where the real status becomes readable. + var isOpaque = resp.type === 'opaque' || resp.type === 'opaqueredirect'; + var isSuccess = resp.ok || (resp.status >= 200 && resp.status < 400); + if (isOpaque || isSuccess) { + anchor.setAttribute('href', proxyUrl); + } + }).catch(function () { + if (timer) { clearTimeout(timer); } + // unreachable / rejected by the default server / timed out: keep the + // original NodePort link + }); + } + $('.btn-service[data-proxy-url]').each(function () { + probeProxyLink(this); + }); }) // Extends lab expiration function extendLabExpiration(hours) { diff --git a/scripts/k8s_nginx_revproxy/NGINX_SETUP.md b/scripts/k8s_nginx_revproxy/NGINX_SETUP.md new file mode 100644 index 0000000..02a8bcc --- /dev/null +++ b/scripts/k8s_nginx_revproxy/NGINX_SETUP.md @@ -0,0 +1,258 @@ +# nginx setup — tuned & hardened for the reverse proxy (Debian 12) + +Step-by-step install and hardening of nginx on a Debian 12 host that will front +the Kubernetes NodePort services generated by +[`k8s_nginx_revproxy.py`](k8s_nginx_revproxy.py). All TLS and hardening settings +are kept **global** (in `http {}`), so the vhosts this tool auto-generates inherit +them and stay minimal. + +> Assumes a wildcard TLS certificate and a wildcard DNS record for +> `*.PROXY_DOMAIN` already exist (see the main [README](README.md)). + +## 1. Patch the base system and enable automatic security updates + +```bash +sudo apt update && sudo apt -y full-upgrade +sudo apt -y install unattended-upgrades +sudo dpkg-reconfigure -plow unattended-upgrades +``` + +Keeping nginx and OpenSSL patched automatically is the single most important +control for an internet-facing proxy. + +## 2. Install nginx + +Debian 12 ships nginx 1.22 with backported security fixes — lowest maintenance: + +```bash +sudo apt -y install nginx +``` + +If you want the latest stable (1.26+), use the official nginx.org repo instead: + +```bash +sudo apt -y install curl gnupg2 ca-certificates lsb-release debian-archive-keyring +curl -fsSL https://nginx.org/keys/nginx_signing.key | sudo gpg --dearmor -o /usr/share/keyrings/nginx-archive-keyring.gpg +echo "deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] http://nginx.org/packages/debian bookworm nginx" | sudo tee /etc/apt/sources.list.d/nginx.list +sudo apt update && sudo apt -y install nginx +``` + +Either works with the rest of this guide. + +## 3. Firewall — expose only 80/443 (and SSH) + +```bash +sudo apt -y install ufw +sudo ufw allow OpenSSH +sudo ufw allow 80/tcp +sudo ufw allow 443/tcp +sudo ufw --force enable +``` + +> Make sure SSH is allowed **before** enabling, or you will lock yourself out. + +## 4. Wire in the auto-generated vhost directory + +The daemon writes vhosts into `/etc/nginx/k8s-revproxy/` (its `--output-dir`). + +```bash +sudo mkdir -p /etc/nginx/k8s-revproxy +sudo chmod 755 /etc/nginx/k8s-revproxy +``` + +Add this line inside the `http { ... }` block of `/etc/nginx/nginx.conf`: + +```nginx +include /etc/nginx/k8s-revproxy/*.conf; +``` + +This is where the daemon's `00-maps.conf` (the websocket `map`, which must live +in `http{}` context) and each `-..conf` land. + +## 5. Global hardening snippet + +Debian's `nginx.conf` auto-includes `/etc/nginx/conf.d/*.conf`. Create +`/etc/nginx/conf.d/00-hardening.conf`: + +```nginx +# --- Information disclosure --- +server_tokens off; # hide nginx version in headers/error pages + +# --- Long generated hostnames need a bigger server-name hash bucket --- +server_names_hash_bucket_size 128; + +# --- Request size / slow-loris protection --- +client_max_body_size 50m; # raise per-service if you proxy uploads +client_body_timeout 10s; +client_header_timeout 10s; +send_timeout 10s; +keepalive_timeout 65s; +client_body_buffer_size 16k; +client_header_buffer_size 1k; +large_client_header_buffers 4 8k; + +# --- Security response headers (inherited by all vhosts) --- +add_header X-Content-Type-Options "nosniff" always; +add_header X-Frame-Options "SAMEORIGIN" always; +add_header Referrer-Policy "strict-origin-when-cross-origin" always; +add_header Strict-Transport-Security "max-age=63072000; includeSubDomains" always; + +# --- Rate-limiting zones (apply per-vhost with `limit_req zone=...` if needed) --- +limit_req_zone $binary_remote_addr zone=req_per_ip:10m rate=20r/s; +limit_conn_zone $binary_remote_addr zone=conn_per_ip:10m; +``` + +The generated vhosts set no `add_header` of their own, so these headers +propagate to every proxied response. + +> `server_names_hash_bucket_size 128;` covers hostnames up to ~127 chars. Without +> it nginx fails with *"could not build server_names_hash"* once you have several +> long vhosts. Bump to `256` if you ever generate longer names. + +## 6. TLS hardening snippet + +Create `/etc/nginx/conf.d/10-tls.conf` (Mozilla "intermediate" — TLS 1.2 + 1.3): + +```nginx +ssl_protocols TLSv1.2 TLSv1.3; +ssl_prefer_server_ciphers off; +ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384; + +ssl_session_timeout 1d; +ssl_session_cache shared:MozSSL:10m; # ~40,000 sessions +ssl_session_tickets off; + +ssl_dhparam /etc/nginx/dhparam.pem; + +# OCSP stapling (works with a public wildcard CA) +ssl_stapling on; +ssl_stapling_verify on; +resolver 1.1.1.1 9.9.9.9 valid=300s; +resolver_timeout 5s; +``` + +Generate the DH params once (takes a minute): + +```bash +sudo openssl dhparam -out /etc/nginx/dhparam.pem 2048 +``` + +These global settings mean the generated vhosts only need `ssl_certificate` / +`ssl_certificate_key` (which the daemon writes) — protocols, ciphers, and +stapling are inherited. + +## 7. Default "deny unknown host" server + +Critical for a reverse proxy: without it, nginx serves the *first* vhost to any +request hitting the raw IP or a bogus `Host` header. Add a catch-all that drops +them. Create `/etc/nginx/conf.d/05-default-server.conf`: + +```nginx +server { + listen 80 default_server; + listen [::]:80 default_server; + listen 443 ssl http2 default_server; + listen [::]:443 ssl http2 default_server; + server_name _; + + ssl_certificate /etc/letsencrypt/live/PROXY_DOMAIN/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/PROXY_DOMAIN/privkey.pem; + + return 444; # close the connection without a response +} +``` + +Only explicitly-matched hostnames (the generated `-.` +vhosts) get served; everything else is dropped. Remove Debian's default site so +it doesn't also claim `default_server`: + +```bash +sudo rm -f /etc/nginx/sites-enabled/default +``` + +## 8. Lock down the certificate key permissions + +```bash +sudo chmod 600 /etc/ssl/private/wildcard.key # or the letsencrypt privkey.pem +sudo chown root:root /etc/ssl/private/wildcard.key +``` + +nginx reads certs as the master process (root); workers never need access. + +## 9. Harden the nginx systemd service + +Add sandboxing without editing the packaged unit: + +```bash +sudo systemctl edit nginx +``` + +Paste into the override editor and save: + +```ini +[Service] +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=strict +ProtectHome=true +# '-' makes a missing path non-fatal (avoids 226/NAMESPACE start failures). +ReadWritePaths=-/var/log/nginx -/var/lib/nginx -/var/cache/nginx -/run +ProtectKernelTunables=true +ProtectKernelModules=true +ProtectControlGroups=true +RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX +RestrictNamespaces=true +``` + +> The `-` prefix matters: some builds don't create `/var/cache/nginx`, and +> `ProtectSystem=strict` aborts namespace setup (`status=226/NAMESPACE`, +> *"Failed to set up mount namespacing … No such file or directory"*) on any +> missing `ReadWritePaths` entry. The `-` makes those optional. + +`ProtectSystem=strict` keeps `/etc/nginx` read-only for nginx, which is fine — +the daemon (not nginx) writes the vhost files and then triggers `nginx -t` / +reload. + +## 10. Test, reload, enable + +```bash +sudo systemctl daemon-reload +sudo nginx -t +sudo systemctl enable --now nginx +sudo systemctl restart nginx +``` + +`nginx -t` must pass before restart — the same gate the daemon uses before +reloading, so the two stay consistent. + +## 11. Verify + +Check the effective TLS + headers against one of the generated hostnames: + +```bash +curl -sSI https://-.PROXY_DOMAIN | grep -iE 'strict-transport|x-frame|x-content|server' +nmap --script ssl-enum-ciphers -p 443 PROXY_DOMAIN +``` + +For an external grade, run the host through SSL Labs +(`https://www.ssllabs.com/ssltest/`) or `testssl.sh`. + +## Optional next layers + +- **fail2ban** for the SSH/HTTP surface: `sudo apt -y install fail2ban` + (enable the `sshd` and `nginx-*` jails). +- **Per-vhost rate limiting** — apply the zones from step 5 with + `limit_req zone=req_per_ip burst=40 nodelay;`. Since the generated `location` + blocks are fixed, add it to the script's vhost template if you want it applied + automatically. +- **ModSecurity + OWASP CRS** if you need a WAF in front of the proxied apps + (heavier; only if the threat model calls for it). + +## Notes specific to this reverse proxy + +- The daemon owns `/etc/nginx/k8s-revproxy/*.conf`. Everything in this guide + lives in files *you* manage (`conf.d/*.conf`, `nginx.conf`), so the daemon's + rollback logic never touches your hardening. +- HTTP/2 is enabled via the `listen … ssl http2;` parameter form, which works on + Debian's nginx 1.22 (the separate `http2 on;` directive is 1.25.1+ only). +- Only TCP services are proxied; UDP/SCTP NodePorts are skipped by the daemon. diff --git a/scripts/k8s_nginx_revproxy/README.md b/scripts/k8s_nginx_revproxy/README.md new file mode 100644 index 0000000..f47b044 --- /dev/null +++ b/scripts/k8s_nginx_revproxy/README.md @@ -0,0 +1,225 @@ +# k8s_nginx_revproxy + +A standalone daemon that automatically publishes Kubernetes **NodePort** services +through an external **nginx** HTTPS reverse proxy. + +## What it does and the problem it solves + +Services of type `NodePort` are reachable at `:`, on a random +high port, over plain HTTP, with no friendly name and no TLS. Exposing many such +services to users means either hand-writing an nginx vhost per service (and +keeping it in sync as pods move between nodes) or standing up a full ingress +controller inside the cluster. + +This script removes that toil. It runs *outside* the cluster on the box that also +runs nginx, and on a loop it: + +1. Lists all `NodePort` services and all pods in the namespaces you configure + (via the Kubernetes API). +2. Matches each service to its backing pods (`service.spec.selector` ↔ + `pod.metadata.labels`), keeping only pods that are **Running + Ready** and + have a `status.hostIP`. +3. Builds the concrete upstreams `hostIP:nodePort` — the set of nodes actually + running the pods — so traffic goes straight to the right node(s), with + automatic load-balancing and failover when there is more than one. +4. Renders one nginx `server{}` + `upstream{}` per service (HTTP→HTTPS redirect, + HTTPS front-end using your wildcard certificate, websocket support). +5. Atomically writes the vhosts, runs `nginx -t`, and reloads nginx — **only when + the desired config actually changed**, rolling back if `nginx -t` rejects it. + +It is built to run forever under systemd and to survive Kubernetes hiccups: +every API call is time-bounded, a failed or partial fetch never rewrites nginx +(the last good config stays live), and you can supply **several kubeconfigs for +the same cluster** (pointing at different control-plane nodes) which it cycles +through whenever the current endpoint errors or times out. + +### Naming scheme + +Each service is published at: + +``` +-. +``` + +For example, a service named `webui` in namespace `hackinsdn` exposing port +`8080` becomes: + +``` +8080-webui.k8s-ingress.example.com +``` + +This pairs with the dashboard's new **`PROXY_DOMAIN`** setting: the dashboard +builds the same hostname to link users to a deployed service, and this daemon +makes that hostname actually resolve and terminate TLS. Because the name uses a +single DNS label under `PROXY_DOMAIN`, it is covered by a wildcard DNS record and +a wildcard TLS certificate for `*.PROXY_DOMAIN` (see prerequisites). + +Only **TCP** ports are published — UDP/SCTP ports can't be reverse-proxied at +L7 and are skipped (logged at INFO). A service can override its hostname, scheme, +or which port is exposed with annotations (see [Annotations](#per-service-annotations)). + +## Prerequisites + +- **nginx** on the same host, with a place to `include` the generated vhosts. + For a full tuned & hardened install, follow [`NGINX_SETUP.md`](NGINX_SETUP.md). + At minimum, add this line inside the `http {}` block of `/etc/nginx/nginx.conf`: + + ```nginx + include /etc/nginx/k8s-revproxy/*.conf; + ``` + + Because the generated hostnames are long, also raise the server-name hash + bucket size in the `http {}` context (or a snippet in `conf.d/`): + + ```nginx + server_names_hash_bucket_size 128; + ``` + +- **Wildcard DNS**: an `A`/`AAAA` (or `CNAME`) record for `*.PROXY_DOMAIN` + pointing at this nginx host, so every `-.PROXY_DOMAIN` resolves. + +- **Wildcard TLS certificate** for `*.PROXY_DOMAIN` (e.g. Let's Encrypt DNS-01, + or any CA). Point `--ssl-certificate` / `--ssl-certificate-key` at the + fullchain and private key. The certificate and DNS record are assumed to + already exist — this script does not manage them. + +- **Kubeconfig(s) with RBAC access** to *list* services and pods in the target + namespaces. A read-only ClusterRole/Role is enough: + + ```yaml + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: k8s-nginx-revproxy-reader + rules: + - apiGroups: [""] + resources: ["services", "pods"] + verbs: ["get", "list"] + ``` + + Bind it to the identity used by each kubeconfig. Provide one kubeconfig per + control-plane endpoint for failover. + +- **Python 3.9+** and the dependencies in `requirements.txt` + (`kubernetes`; plus `urllib3[socks]` / PySocks only if you use a SOCKS proxy). + +## Configuration + +All options can be given as a command-line flag or an environment variable; the +flag wins when both are set. Three options are **required**: `--kubeconfig`, +`--namespace`, and `--proxy-domain`. + +| Flag | Env var | Default | Description | +|------|---------|---------|-------------| +| `--kubeconfig` (repeatable) | `KUBECONFIGS` (`:`-separated) | — **required** | Kubeconfig file(s) for the same cluster; cycled on failure. | +| `--namespace` (repeatable) | `NAMESPACES` (comma/space) or `NAMESPACE` | — **required** | Namespace(s) to watch. | +| `--proxy-domain` | `PROXY_DOMAIN` | — **required** | Domain for published vhosts. Hostnames are `-.`. | +| `--poll-interval` | `POLL_INTERVAL` | `30` | Seconds between reconciles. | +| `--connect-timeout` | `CONNECT_TIMEOUT` | `5` | Per-request connect timeout (seconds). | +| `--read-timeout` | `READ_TIMEOUT` | `15` | Per-request read timeout (seconds). | +| `--proxy` | `PROXY` or `HTTPS_PROXY` | *(none)* | Proxy for all API traffic, e.g. `socks5h://127.0.0.1:1080` for a SOCKS5 SSH tunnel. | +| `--output-dir` | `OUTPUT_DIR` | `/etc/nginx/k8s-revproxy` | Directory the tool owns and writes vhosts into (stale `*.conf` are removed). | +| `--ssl-certificate` | `SSL_CERTIFICATE` | `/etc/ssl/certs/wildcard.crt` | Wildcard TLS certificate (fullchain). | +| `--ssl-certificate-key` | `SSL_CERTIFICATE_KEY` | `/etc/ssl/private/wildcard.key` | Wildcard TLS private key. | +| `--upstream-scheme` | `UPSTREAM_SCHEME` | `http` | Default scheme to reach backends (`http` or `https`). Auto-upgrades to `https` when the service port name starts with `https`. | +| `--nginx-bin` | `NGINX_BIN` | `nginx` | nginx binary used for `nginx -t`. | +| `--reload-cmd` | `RELOAD_CMD` | `systemctl reload nginx` | Command to reload nginx (shell words). | +| `--annotation-prefix` | `ANNOTATION_PREFIX` | `revproxy.hackinsdn.io` | Prefix for per-service annotation overrides. | +| `--opt-in` | `OPT_IN` (`1`/`true`/`yes`) | `false` | Expose only services annotated `/enable=true`. Otherwise every NodePort service is exposed unless it sets `/enable=false`. | +| `--no-require-ready` | `REQUIRE_READY` (`false` to disable) | ready required | Route to Running pods even if not Ready. | +| `--log-level` | `LOG_LEVEL` | `INFO` | Python logging level. | +| `--dry-run` | `DRY_RUN` (`1`/`true`/`yes`) | `false` | Print the config that would be written and exit; no disk writes, no `nginx -t`, no reload. Implies a single cycle. | +| `--once` | — | `false` | Run a single reconcile and exit (for testing). | + +### Per-service annotations + +Set on a Service to override defaults (prefix configurable via +`--annotation-prefix`, default `revproxy.hackinsdn.io`): + +| Annotation | Effect | +|------------|--------| +| `/enable` | `"true"` to expose (when `--opt-in`), or `"false"` to opt out. | +| `/hostname` | Override the full published hostname. | +| `/scheme` | Force the upstream scheme (`http` or `https`). | +| `/port` | Expose only the given service port (name or number). | + +## Running it manually + +Install the dependencies (a virtualenv is recommended): + +```bash +python3 -m venv /opt/k8s-nginx-revproxy +/opt/k8s-nginx-revproxy/bin/pip install -r requirements.txt +``` + +Preview what would be generated, without touching nginx: + +```bash +env PROXY_DOMAIN=k8s-ingress.example.com NAMESPACES=hackinsdn \ + /opt/k8s-nginx-revproxy/bin/python3 k8s_nginx_revproxy.py \ + --kubeconfig /etc/k8s-nginx-revproxy/kubeconfig-node1.yaml \ + --kubeconfig /etc/k8s-nginx-revproxy/kubeconfig-node2.yaml \ + --ssl-certificate /etc/letsencrypt/live/k8s-ingress.example.com/fullchain.pem \ + --ssl-certificate-key /etc/letsencrypt/live/k8s-ingress.example.com/privkey.pem \ + --dry-run +``` + +Drop `--dry-run` to actually write vhosts, validate, reload nginx, and keep +reconciling every `--poll-interval` seconds. Add `--once` to run a single cycle. + +### Through a SOCKS5 SSH tunnel + +If the API servers are only reachable via a jump host, open a tunnel and point +`--proxy` at it (use `socks5h` so DNS resolves on the remote side): + +```bash +ssh -f -N -D 127.0.0.1:1080 user@jump-host +# ... --proxy socks5h://127.0.0.1:1080 +``` + +This requires PySocks (`pip install 'urllib3[socks]'`). The tunnel covers only +the Kubernetes API traffic; nginx still reaches `hostIP:nodePort` over the normal +network. + +Please check the `ssh-socks-tunnel.service` for an example on how to setup the SSH +tunnel managed by systemd. + +## Installing as a systemd service + +The unit file `k8s-nginx-revproxy.service` is included. It runs the daemon on a +loop with `Restart=on-failure` and basic sandboxing. + +```bash +# 1. Install the code and dependencies +sudo mkdir -p /opt/k8s-nginx-revproxy /etc/k8s-nginx-revproxy /etc/nginx/k8s-revproxy +sudo cp k8s_nginx_revproxy.py /opt/k8s-nginx-revproxy/ +sudo python3 -m venv /opt/k8s-nginx-revproxy +sudo /opt/k8s-nginx-revproxy/bin/pip install -r requirements.txt + +# 2. Drop your kubeconfigs into /etc/k8s-nginx-revproxy/ +# (kubeconfig-node1.yaml, kubeconfig-node2.yaml, ...) + +# 3. Install and edit the unit (set PROXY_DOMAIN, NAMESPACES, cert paths, kubeconfigs) +sudo cp k8s-nginx-revproxy.service /etc/systemd/system/ +sudo systemctl edit --full k8s-nginx-revproxy.service # or edit the file directly + +# 4. Make sure nginx includes the vhost directory (see Prerequisites), then: +sudo systemctl daemon-reload +sudo systemctl enable --now k8s-nginx-revproxy.service +sudo systemctl status k8s-nginx-revproxy.service +journalctl -u k8s-nginx-revproxy.service -f +``` + +If you route the API through an SSH tunnel, run the tunnel as its own unit (e.g. +with `autossh`) and uncomment the `After=`/`Wants=` lines and the `PROXY=` env in +the unit so the proxy starts only once the tunnel is up. + +## Files + +| File | Purpose | +|------|---------| +| `k8s_nginx_revproxy.py` | The daemon. | +| `requirements.txt` | Python dependencies. | +| `k8s-nginx-revproxy.service` | systemd unit template. | +| `NGINX_SETUP.md` | Tuned & hardened nginx install guide (Debian 12). | +| `README.md` | This document. | diff --git a/scripts/k8s_nginx_revproxy/k8s-nginx-revproxy.service b/scripts/k8s_nginx_revproxy/k8s-nginx-revproxy.service new file mode 100644 index 0000000..903d86c --- /dev/null +++ b/scripts/k8s_nginx_revproxy/k8s-nginx-revproxy.service @@ -0,0 +1,50 @@ +[Unit] +Description=Kubernetes NodePort -> nginx reverse proxy +Documentation=https://github.com/hackinsdn/dashboard/tree/main/scripts/k8s_nginx_revproxy +After=network-online.target nginx.service +Wants=network-online.target +# If the API is reached through a SOCKS5 SSH tunnel, uncomment and provide a +# tunnel unit (see README) so the proxy starts only once the tunnel is up: +# After=socks-tunnel.service +# Wants=socks-tunnel.service + +[Service] +Type=simple + +# --- Configuration (edit to match your environment) -------------------------- +# Domain that the wildcard cert/DNS record cover. Vhosts are published as +# -.. +Environment=PROXY_DOMAIN=k8s-ingress.example.com +# Namespace(s) to watch (comma or space separated). +Environment=NAMESPACES=hackinsdn +# Uncomment to route the Kubernetes API through a local SOCKS5 SSH tunnel: +# Environment=PROXY=socks5h://127.0.0.1:1080 + +ExecStart=/opt/k8s-nginx-revproxy/bin/python3 /opt/k8s-nginx-revproxy/k8s_nginx_revproxy.py \ + --kubeconfig /etc/k8s-nginx-revproxy/kubeconfig-node1.yaml \ + --kubeconfig /etc/k8s-nginx-revproxy/kubeconfig-node2.yaml \ + --kubeconfig /etc/k8s-nginx-revproxy/kubeconfig-node3.yaml \ + --ssl-certificate /etc/letsencrypt/live/k8s-ingress.example.com/fullchain.pem \ + --ssl-certificate-key /etc/letsencrypt/live/k8s-ingress.example.com/privkey.pem \ + --output-dir /etc/nginx/k8s-revproxy \ + --reload-cmd 'systemctl reload nginx' + +Restart=on-failure +RestartSec=5 + +# --- Sandboxing -------------------------------------------------------------- +# The proxy only needs to read /etc/nginx and write its own vhost directory. +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=strict +ProtectHome=true +# Paths the process must be able to write. '-' makes missing paths non-fatal. +ReadWritePaths=/etc/nginx/k8s-revproxy -/var/log/nginx -/var/lib/nginx -/var/cache/nginx -/run +ProtectKernelTunables=true +ProtectKernelModules=true +ProtectControlGroups=true +RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX +RestrictNamespaces=true + +[Install] +WantedBy=multi-user.target diff --git a/scripts/k8s_nginx_revproxy/k8s_nginx_revproxy.py b/scripts/k8s_nginx_revproxy/k8s_nginx_revproxy.py new file mode 100755 index 0000000..99024e9 --- /dev/null +++ b/scripts/k8s_nginx_revproxy/k8s_nginx_revproxy.py @@ -0,0 +1,718 @@ +#!/usr/bin/env python3 +""" +k8s_nginx_revproxy.py + +Standalone daemon that publishes Kubernetes NodePort services through an +external nginx reverse proxy (HTTPS front-end, backed by a wildcard TLS +certificate and a wildcard DNS record that already exist). + +On every poll it: + + 1. Lists all Services of type NodePort and all Pods (via the Kubernetes API). + 2. Matches Services to Pods (service.spec.selector <-> pod.metadata.labels). + 3. For each matched, running+ready Pod it reads status.hostIP and combines it + with the Service's nodePort to build a set of concrete upstreams + (hostIP:nodePort). Traffic is thus sent straight to the node(s) actually + running the backing pods. + 4. Renders one nginx `server{}` (vhost) + `upstream{}` per exposed Service, + with server_name = (covered by the wildcard cert/DNS). + 5. Atomically writes the vhosts, runs `nginx -t`, and reloads nginx only when + the desired configuration actually changed (rolling back on a bad config). + +Robustness: + * Multiple kubeconfigs for the same cluster (different control-plane nodes) + are cycled through whenever the current API endpoint errors or times out. + * Every API call has connect/read timeouts. + * A failed/partial fetch never rewrites nginx -- the last good config stays + in place until a full, successful fetch is obtained again. + * Designed to run forever under systemd with Restart=on-failure. + +Only depends on the official `kubernetes` python client (and nginx on PATH). +""" + +from __future__ import annotations + +import argparse +import hashlib +import logging +import os +import re +import signal +import subprocess +import sys +import time +from dataclasses import dataclass, field +from typing import Dict, List, Optional + +try: + from kubernetes import client, config + from kubernetes.client.rest import ApiException +except ImportError: # pragma: no cover + sys.stderr.write( + "The 'kubernetes' python client is required: pip install kubernetes\n" + ) + raise + +try: + # urllib3 is a transitive dep of the kubernetes client; used to catch + # connect/read timeouts and connection errors distinctly. + from urllib3.exceptions import HTTPError as Urllib3HTTPError +except ImportError: # pragma: no cover + Urllib3HTTPError = Exception # type: ignore + + +log = logging.getLogger("k8s-revproxy") + + +# --------------------------------------------------------------------------- # +# Configuration +# --------------------------------------------------------------------------- # +def _env(name: str, default: str) -> str: + return os.environ.get(name, default) + + +@dataclass +class Settings: + # One or more kubeconfig files pointing at the *same* cluster through + # different control-plane endpoints. They are tried in round-robin order, + # rotating whenever the active one errors out. + kubeconfigs: List[str] = field(default_factory=list) + + # How often (seconds) to reconcile. + poll_interval: float = 30.0 + + # (connect, read) timeout for every API request. + connect_timeout: float = 5.0 + read_timeout: float = 15.0 + + # Optional proxy for all API traffic, e.g. a SOCKS5 SSH tunnel + # (socks5h://127.0.0.1:1080). Empty means direct connection. + proxy: str = "" + + # Directory that nginx `include`s, e.g.: + # include /etc/nginx/k8s-revproxy/*.conf; + # It is treated as owned by this tool -- stale *.conf are removed. + output_dir: str = "/etc/nginx/k8s-revproxy" + + # Wildcard TLS material (already issued for *.). + ssl_certificate: str = "/etc/ssl/certs/wildcard.crt" + ssl_certificate_key: str = "/etc/ssl/private/wildcard.key" + + # Domain under which every vhost is published. Each hostname is built as + # -. + # PROXY_DOMAIN must be covered by the wildcard cert/DNS record. + proxy_domain: str = "" + + # Default scheme used to reach the NodePort backends. It is http unless the + # service port name starts with "https" (then https), and can still be + # forced per-service with the /scheme annotation. + default_upstream_scheme: str = "http" + + # nginx binary + reload command. + nginx_bin: str = "nginx" + reload_cmd: List[str] = field( + default_factory=lambda: ["systemctl", "reload", "nginx"] + ) + + # Annotation prefix for per-service overrides, e.g. + # revproxy.hackinsdn.io/hostname: myapp. (overrides the template) + # revproxy.hackinsdn.io/scheme: https (overrides port-name rule) + # revproxy.hackinsdn.io/port: "8443" (service port name or number) + # revproxy.hackinsdn.io/enable: "false" + annotation_prefix: str = "revproxy.hackinsdn.io" + + # If set, only expose services carrying /enable=true. + # Otherwise every NodePort service is exposed unless it opts out with + # /enable=false. + opt_in: bool = False + + # Require pods to be Ready (not merely Running) before routing to them. + require_ready: bool = True + + # Namespace(s) to watch. At least one must be provided. + namespaces: List[str] = field(default_factory=list) + + # Only print the nginx config that would be written; never touch disk, + # nginx -t, or reload. Implies a single cycle. + dry_run: bool = False + + def ann(self, suffix: str) -> str: + return f"{self.annotation_prefix}/{suffix}" + + +def parse_args(argv: Optional[List[str]] = None) -> Settings: + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--kubeconfig", action="append", default=[], + help="Path to a kubeconfig (repeatable). Also KUBECONFIGS=path1:path2.") + p.add_argument("--poll-interval", type=float, default=float(_env("POLL_INTERVAL", "30"))) + p.add_argument("--connect-timeout", type=float, default=float(_env("CONNECT_TIMEOUT", "5"))) + p.add_argument("--read-timeout", type=float, default=float(_env("READ_TIMEOUT", "15"))) + p.add_argument("--proxy", default=_env("PROXY", "") or _env("HTTPS_PROXY", ""), + help="Proxy for all API traffic, e.g. socks5h://127.0.0.1:1080 " + "for a SOCKS5 SSH tunnel. Also PROXY / HTTPS_PROXY env.") + p.add_argument("--output-dir", default=_env("OUTPUT_DIR", "/etc/nginx/k8s-revproxy")) + p.add_argument("--ssl-certificate", default=_env("SSL_CERTIFICATE", "/etc/ssl/certs/wildcard.crt")) + p.add_argument("--ssl-certificate-key", default=_env("SSL_CERTIFICATE_KEY", "/etc/ssl/private/wildcard.key")) + p.add_argument("--proxy-domain", default=_env("PROXY_DOMAIN", ""), + help="Domain for published vhosts (also PROXY_DOMAIN env). " + "Hostnames are -..") + p.add_argument("--upstream-scheme", default=_env("UPSTREAM_SCHEME", "http"), choices=["http", "https"]) + p.add_argument("--nginx-bin", default=_env("NGINX_BIN", "nginx")) + p.add_argument("--reload-cmd", default=_env("RELOAD_CMD", "systemctl reload nginx"), + help="Shell-word list, e.g. 'systemctl reload nginx' or 'nginx -s reload'.") + p.add_argument("--annotation-prefix", default=_env("ANNOTATION_PREFIX", "revproxy.hackinsdn.io")) + p.add_argument("--opt-in", action="store_true", default=_env("OPT_IN", "").lower() in ("1", "true", "yes")) + p.add_argument("--no-require-ready", dest="require_ready", action="store_false", + default=_env("REQUIRE_READY", "true").lower() in ("1", "true", "yes")) + p.add_argument("--namespace", action="append", default=[], + help="Namespace to watch (repeatable). Also NAMESPACES=ns1,ns2 (or NAMESPACE=ns).") + p.add_argument("--log-level", default=_env("LOG_LEVEL", "INFO")) + p.add_argument("--once", action="store_true", help="Run a single reconcile and exit (for testing).") + p.add_argument("--dry-run", action="store_true", + default=_env("DRY_RUN", "").lower() in ("1", "true", "yes"), + help="Print the nginx config that would be written and exit; " + "no disk writes, no nginx -t, no reload.") + a = p.parse_args(argv) + + kubeconfigs = list(a.kubeconfig) + if not kubeconfigs and _env("KUBECONFIGS", ""): + kubeconfigs = [x for x in _env("KUBECONFIGS", "").split(os.pathsep) if x] + if not kubeconfigs: + p.error("at least one --kubeconfig (or KUBECONFIGS env) is required") + + namespaces = list(a.namespace) + if not namespaces: + env_ns = _env("NAMESPACES", "") or _env("NAMESPACE", "") + namespaces = [x.strip() for x in re.split(r"[,\s]+", env_ns) if x.strip()] + if not namespaces: + p.error("at least one --namespace (or NAMESPACE/NAMESPACES env) is required") + + if not a.proxy_domain: + p.error("--proxy-domain (or PROXY_DOMAIN env) is required") + + s = Settings( + kubeconfigs=kubeconfigs, + poll_interval=a.poll_interval, + connect_timeout=a.connect_timeout, + read_timeout=a.read_timeout, + proxy=a.proxy, + output_dir=a.output_dir, + ssl_certificate=a.ssl_certificate, + ssl_certificate_key=a.ssl_certificate_key, + proxy_domain=a.proxy_domain, + default_upstream_scheme=a.upstream_scheme, + nginx_bin=a.nginx_bin, + reload_cmd=a.reload_cmd.split(), + annotation_prefix=a.annotation_prefix, + opt_in=a.opt_in, + require_ready=a.require_ready, + namespaces=namespaces, + dry_run=a.dry_run, + ) + logging.basicConfig( + level=getattr(logging, a.log_level.upper(), logging.INFO), + format="%(asctime)s %(levelname)s %(name)s: %(message)s", + ) + s._once = a.once # type: ignore[attr-defined] + return s + + +# --------------------------------------------------------------------------- # +# Kubernetes client pool (round-robin over kubeconfigs on failure) +# --------------------------------------------------------------------------- # +class KubeClientPool: + """Builds a CoreV1Api from the current kubeconfig and rotates to the next + one whenever :meth:`rotate` is called (i.e. the current endpoint failed).""" + + def __init__(self, kubeconfigs: List[str], proxy: str = ""): + if not kubeconfigs: + raise ValueError("no kubeconfigs provided") + self._kubeconfigs = kubeconfigs + self._proxy = proxy + self._idx = 0 + self._api: Optional[client.CoreV1Api] = None + + @property + def current_path(self) -> str: + return self._kubeconfigs[self._idx] + + def _build(self) -> client.CoreV1Api: + cfg = client.Configuration() + # Isolate each kubeconfig into its own Configuration/ApiClient so + # rotating never leaks state (host, token, CA) between endpoints. + config.load_kube_config(config_file=self.current_path, client_configuration=cfg) + + use_socks = bool(self._proxy) and self._proxy.lower().startswith("socks") + if self._proxy and not use_socks: + # Plain http/https proxy: the client handles this natively. + cfg.proxy = self._proxy + + api_client = client.ApiClient(cfg) + if use_socks: + # The kubernetes client passes Configuration.proxy to + # urllib3.ProxyManager, which rejects socks schemes + # (ProxySchemeUnknown). Swap in a SOCKSProxyManager instead, + # mirroring the TLS settings the client would have used. + self._install_socks_pool_manager(api_client, cfg) + + api = client.CoreV1Api(api_client) + log.info("Using kubeconfig %s (host=%s)%s", self.current_path, cfg.host, + f" via proxy {self._proxy}" if self._proxy else "") + return api + + def _install_socks_pool_manager(self, api_client, cfg) -> None: + import ssl as _ssl + try: + from urllib3.contrib.socks import SOCKSProxyManager + except ImportError as exc: # pragma: no cover + raise RuntimeError( + "SOCKS proxy requested but PySocks is missing; " + "install it with: pip install 'urllib3[socks]'" + ) from exc + + maxsize = cfg.connection_pool_maxsize or 4 + pool_args = { + "num_pools": maxsize, + "maxsize": maxsize, + "cert_reqs": _ssl.CERT_REQUIRED if cfg.verify_ssl else _ssl.CERT_NONE, + "ca_certs": cfg.ssl_ca_cert, + "cert_file": cfg.cert_file, + "key_file": cfg.key_file, + } + if getattr(cfg, "assert_hostname", None) is not None: + pool_args["assert_hostname"] = cfg.assert_hostname + + rest = api_client.rest_client + try: + rest.pool_manager.clear() # drop the direct-connection pool + except Exception: # noqa: BLE001 + pass + rest.pool_manager = SOCKSProxyManager(self._proxy, **pool_args) + + def api(self) -> client.CoreV1Api: + if self._api is None: + self._api = self._build() + return self._api + + def rotate(self) -> None: + prev = self.current_path + self._idx = (self._idx + 1) % len(self._kubeconfigs) + self._api = None # rebuilt lazily on next api() + if len(self._kubeconfigs) > 1: + log.warning("Rotating kubeconfig %s -> %s", prev, self.current_path) + + +# --------------------------------------------------------------------------- # +# Domain model +# --------------------------------------------------------------------------- # +@dataclass +class VHost: + hostname: str + upstream_name: str + scheme: str + endpoints: List[str] # ["10.0.0.5:31234", ...] + + def sort_key(self) -> str: + return self.hostname + + +_SAFE = re.compile(r"[^a-zA-Z0-9]+") + + +def _safe_name(*parts: str) -> str: + return _SAFE.sub("_", "_".join(parts)).strip("_").lower() + + +def _sanitize_host(name: str) -> str: + # DNS label(s): lowercase, only [a-z0-9.-]. + return re.sub(r"[^a-z0-9.-]+", "-", name.lower()).strip("-.") + + +def _pod_matches_selector(pod, selector: Dict[str, str]) -> bool: + labels = pod.metadata.labels or {} + return all(labels.get(k) == v for k, v in selector.items()) + + +def _pod_is_ready(pod) -> bool: + conds = (pod.status.conditions or []) if pod.status else [] + for c in conds: + if c.type == "Ready": + return c.status == "True" + return False + + +def _annotation(obj, key: str) -> Optional[str]: + anns = (obj.metadata.annotations or {}) if obj.metadata else {} + return anns.get(key) + + +# --------------------------------------------------------------------------- # +# Reconciler +# --------------------------------------------------------------------------- # +class Reconciler: + def __init__(self, settings: Settings): + self.s = settings + self.pool = KubeClientPool(settings.kubeconfigs, proxy=settings.proxy) + self._last_written_hash: Optional[str] = None + + # ---- fetch ---------------------------------------------------------- # + def _list_with_rotation(self, fn, *, what: str, **kwargs): + """Call an API list function; on failure rotate kubeconfig and raise so + the caller aborts this cycle without touching nginx.""" + timeout = (self.s.connect_timeout, self.s.read_timeout) + try: + return fn(_request_timeout=timeout, **kwargs) + except (ApiException, Urllib3HTTPError, OSError) as exc: + log.error("Failed to list %s via %s: %s", what, self.pool.current_path, exc) + self.pool.rotate() + raise + + def fetch_vhosts(self) -> List[VHost]: + api = self.pool.api() + services = [] + pods = [] + for ns in self.s.namespaces: + services += self._list_with_rotation( + api.list_namespaced_service, what=f"services in {ns}", namespace=ns + ).items + pods += self._list_with_rotation( + api.list_namespaced_pod, what=f"pods in {ns}", namespace=ns + ).items + + # Index running pods by namespace for quick selector matching. + pods_by_ns: Dict[str, list] = {} + for pod in pods: + if not pod.status or pod.status.phase != "Running": + continue + if not pod.status.host_ip: + continue + if self.s.require_ready and not _pod_is_ready(pod): + continue + pods_by_ns.setdefault(pod.metadata.namespace, []).append(pod) + + vhosts: Dict[str, VHost] = {} + for svc in services: + vh = self._service_to_vhosts(svc, pods_by_ns) + for v in vh: + if v.hostname in vhosts: + log.warning("Hostname collision for %s -- keeping first, " + "skipping service %s/%s", v.hostname, + svc.metadata.namespace, svc.metadata.name) + continue + vhosts[v.hostname] = v + return sorted(vhosts.values(), key=VHost.sort_key) + + def _service_to_vhosts(self, svc, pods_by_ns) -> List[VHost]: + s = self.s + ns = svc.metadata.namespace + name = svc.metadata.name + + if not svc.spec or svc.spec.type != "NodePort": + return [] + + enable = _annotation(svc, s.ann("enable")) + if s.opt_in: + if (enable or "").lower() != "true": + return [] + else: + if (enable or "").lower() == "false": + log.debug("Service %s/%s opted out via annotation", ns, name) + return [] + + selector = svc.spec.selector or {} + if not selector: + log.debug("Service %s/%s has no selector; skipping", ns, name) + return [] + + # Ports that actually have a nodePort assigned. + node_ports = [p for p in (svc.spec.ports or []) if p.node_port] + if not node_ports: + return [] + + # Optional per-service port restriction. + want_port = _annotation(svc, s.ann("port")) + if want_port: + node_ports = [ + p for p in node_ports + if str(p.port) == want_port or (p.name and p.name == want_port) + or str(p.node_port) == want_port + ] + if not node_ports: + log.warning("Service %s/%s: annotated port %r has no nodePort", + ns, name, want_port) + return [] + + # Matching, ready pods -> the set of node IPs to route to. + matched = [pod for pod in pods_by_ns.get(ns, []) + if _pod_matches_selector(pod, selector)] + if not matched: + log.debug("Service %s/%s has no ready backing pods; skipping", ns, name) + return [] + host_ips = sorted({pod.status.host_ip for pod in matched}) + + # Per-service scheme/hostname overrides (optional). + ann_scheme = (_annotation(svc, s.ann("scheme")) or "").lower() + ann_host = _annotation(svc, s.ann("hostname")) + + out: List[VHost] = [] + for p in node_ports: + portname = p.name or str(p.port) + proto = (p.protocol or "TCP").upper() + + # Only TCP can be reverse-proxied at L7 (http/https); skip the rest. + if proto != "TCP": + log.info("Service %s/%s port %s uses protocol %s; skipping " + "(only TCP is supported)", ns, name, portname, proto) + continue + + # hostname = -., annotation wins. + hostname = ann_host or f"{p.port}-{name}.{s.proxy_domain}" + hostname = _sanitize_host(hostname) + + # scheme: http by default; https if the port name starts with + # "https"; annotation overrides everything. + if ann_scheme in ("http", "https"): + scheme = ann_scheme + elif portname.lower().startswith("https"): + scheme = "https" + else: + scheme = s.default_upstream_scheme + + endpoints = [f"{ip}:{p.node_port}" for ip in host_ips] + out.append(VHost( + hostname=hostname, + upstream_name=_safe_name(ns, name, portname), + scheme=scheme, + endpoints=endpoints, + )) + return out + + # ---- render --------------------------------------------------------- # + def render(self, vhosts: List[VHost]) -> Dict[str, str]: + """Return {filename: content} for the managed output directory.""" + files: Dict[str, str] = {} + # A single map file (needs http{} context, once) for websocket upgrades. + files["00-maps.conf"] = ( + "# Managed by k8s_nginx_revproxy.py -- do not edit.\n" + "map $http_upgrade $revproxy_connection_upgrade {\n" + " default upgrade;\n" + " '' close;\n" + "}\n" + ) + for v in vhosts: + files[f"{v.hostname}.conf"] = self._render_vhost(v) + return files + + def _render_vhost(self, v: VHost) -> str: + s = self.s + servers = "\n".join(f" server {ep} max_fails=3 fail_timeout=15s;" + for ep in v.endpoints) + ssl_verify = "" if v.scheme == "http" else " proxy_ssl_verify off;\n" + return f"""# Managed by k8s_nginx_revproxy.py -- do not edit. +upstream {v.upstream_name} {{ +{servers} +}} + +server {{ + listen 80; + listen [::]:80; + server_name {v.hostname}; + return 301 https://$host$request_uri; +}} + +server {{ + listen 443 ssl http2; + listen [::]:443 ssl http2; + server_name {v.hostname}; + + ssl_certificate {s.ssl_certificate}; + ssl_certificate_key {s.ssl_certificate_key}; + + location / {{ + proxy_pass {v.scheme}://{v.upstream_name}; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $revproxy_connection_upgrade; +{ssl_verify} proxy_connect_timeout 5s; + proxy_next_upstream error timeout http_502 http_503 http_504; + }} +}} +""" + + # ---- apply ---------------------------------------------------------- # + @staticmethod + def _hash(files: Dict[str, str]) -> str: + h = hashlib.sha256() + for name in sorted(files): + h.update(name.encode()) + h.update(b"\0") + h.update(files[name].encode()) + h.update(b"\0") + return h.hexdigest() + + def _read_current(self) -> Dict[str, str]: + current: Dict[str, str] = {} + d = self.s.output_dir + if not os.path.isdir(d): + return current + for fn in os.listdir(d): + if fn.endswith(".conf"): + try: + with open(os.path.join(d, fn), "r", encoding="utf-8") as fh: + current[fn] = fh.read() + except OSError as exc: + log.warning("Could not read %s: %s", fn, exc) + return current + + def _write_files(self, files: Dict[str, str]) -> None: + d = self.s.output_dir + os.makedirs(d, exist_ok=True) + # Write/replace desired files atomically. + for fn, content in files.items(): + path = os.path.join(d, fn) + tmp = f"{path}.tmp.{os.getpid()}" + with open(tmp, "w", encoding="utf-8") as fh: + fh.write(content) + os.replace(tmp, path) + # Remove stale files we no longer manage. + for fn in os.listdir(d): + if fn.endswith(".conf") and fn not in files: + try: + os.remove(os.path.join(d, fn)) + except OSError as exc: + log.warning("Could not remove stale %s: %s", fn, exc) + + def _nginx_test(self) -> bool: + try: + r = subprocess.run([self.s.nginx_bin, "-t"], + capture_output=True, text=True, timeout=30) + except (OSError, subprocess.TimeoutExpired) as exc: + log.error("nginx -t failed to run: %s", exc) + return False + if r.returncode != 0: + log.error("nginx -t rejected config:\n%s", r.stderr.strip()) + return False + return True + + def _reload(self) -> bool: + try: + r = subprocess.run(self.s.reload_cmd, capture_output=True, + text=True, timeout=30) + except (OSError, subprocess.TimeoutExpired) as exc: + log.error("nginx reload failed to run: %s", exc) + return False + if r.returncode != 0: + log.error("nginx reload failed:\n%s", r.stderr.strip()) + return False + return True + + def apply(self, files: Dict[str, str]) -> None: + desired_hash = self._hash(files) + if desired_hash == self._last_written_hash: + log.debug("No configuration change (%d vhosts).", len(files) - 1) + return + + previous = self._read_current() + if self._hash(previous) == desired_hash: + # On-disk already matches (e.g. first run after restart). + self._last_written_hash = desired_hash + log.info("Configuration already up to date (%d vhosts).", len(files) - 1) + return + + log.info("Applying configuration: %d vhost(s).", len(files) - 1) + added = sorted(k for k in files if k not in previous and k != "00-maps.conf") + removed = sorted(k for k in previous if k not in files and k != "00-maps.conf") + for fn in added: + log.info("New vhost: %s", fn[:-len(".conf")] if fn.endswith(".conf") else fn) + for fn in removed: + log.info("Removed vhost: %s", fn[:-len(".conf")] if fn.endswith(".conf") else fn) + self._write_files(files) + if not self._nginx_test(): + log.error("Rolling back to previous configuration.") + self._write_files(previous) # restore exact previous state + self._nginx_test() # best-effort validation of rollback + return + if self._reload(): + self._last_written_hash = desired_hash + log.info("nginx reloaded successfully.") + else: + log.error("Reload failed; leaving new files in place for inspection.") + + def dry_run(self, files: Dict[str, str]) -> None: + """Print the nginx config that would be written, without touching disk, + nginx -t, or reload.""" + sys.stdout.write( + f"# ===== dry-run: {len(files)} file(s) would be written to " + f"{self.s.output_dir} =====\n" + ) + for fn in sorted(files): + sys.stdout.write(f"\n# ----- {os.path.join(self.s.output_dir, fn)} -----\n") + sys.stdout.write(files[fn]) + sys.stdout.flush() + + # ---- one cycle ------------------------------------------------------ # + def reconcile_once(self) -> None: + vhosts = self.fetch_vhosts() # raises on API failure (nginx untouched) + files = self.render(vhosts) + if self.s.dry_run: + self.dry_run(files) + return + self.apply(files) + + +# --------------------------------------------------------------------------- # +# Main loop +# --------------------------------------------------------------------------- # +_stop = False + + +def _handle_signal(signum, _frame): + global _stop + log.info("Received signal %s, shutting down.", signum) + _stop = True + + +def main(argv: Optional[List[str]] = None) -> int: + settings = parse_args(argv) + signal.signal(signal.SIGTERM, _handle_signal) + signal.signal(signal.SIGINT, _handle_signal) + + rec = Reconciler(settings) + once = getattr(settings, "_once", False) or settings.dry_run + backoff = settings.poll_interval + + log.info("Starting with %d kubeconfig(s), namespaces=%s, domain=%s, poll=%ss, output=%s", + len(settings.kubeconfigs), ",".join(settings.namespaces), + settings.proxy_domain, settings.poll_interval, settings.output_dir) + + while not _stop: + start = time.monotonic() + try: + rec.reconcile_once() + backoff = settings.poll_interval # reset after a good cycle + except (ApiException, Urllib3HTTPError, OSError) as exc: + # fetch_vhosts already logged + rotated the kubeconfig. + log.warning("Reconcile cycle failed (%s); nginx config unchanged.", type(exc).__name__) + except Exception: # noqa: BLE001 keep the daemon alive; systemd is the safety net + log.exception("Unexpected error during reconcile; continuing.") + + if once: + break + + elapsed = time.monotonic() - start + sleep_for = max(1.0, backoff - elapsed) + # Interruptible sleep so signals are handled promptly. + deadline = time.monotonic() + sleep_for + while not _stop and time.monotonic() < deadline: + time.sleep(min(1.0, deadline - time.monotonic())) + + log.info("Stopped.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/k8s_nginx_revproxy/requirements.txt b/scripts/k8s_nginx_revproxy/requirements.txt new file mode 100644 index 0000000..daf8d17 --- /dev/null +++ b/scripts/k8s_nginx_revproxy/requirements.txt @@ -0,0 +1,5 @@ +# Python dependencies for k8s_nginx_revproxy.py +kubernetes==34.1.0 +# PySocks is only needed when using a SOCKS5 proxy (--proxy socks5h://...). +# Installed via the urllib3[socks] extra so it matches the client's urllib3. +urllib3[socks]==2.3.0 diff --git a/scripts/k8s_nginx_revproxy/ssh-socks-tunnel.service b/scripts/k8s_nginx_revproxy/ssh-socks-tunnel.service new file mode 100644 index 0000000..a13f5a7 --- /dev/null +++ b/scripts/k8s_nginx_revproxy/ssh-socks-tunnel.service @@ -0,0 +1,16 @@ +[Unit] +Description=SSH SOCKS5 Proxy Tunnel +After=network.target + +[Service] +Type=simple +# Ensure you replace username and your-bastion-host.com with your actual details +ExecStart=/usr/bin/ssh -D 1080 -q -N -o ServerAliveInterval=60 -o ExitOnForwardFailure=yes yourusername@yourjumphost + +# Automatically restart the tunnel on failure +Restart=always +# Wait 10 seconds before attempting a restart +RestartSec=10 + +[Install] +WantedBy=default.target