From 2005e797f7e245cec045b6b482ac6e90aa4969a4 Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Mon, 19 Oct 2026 15:57:09 -0600 Subject: [PATCH] deploy_monitoring.sh: exit non-zero when the monitoring Helm release is not deployed Fail when helm upgrade fails or the release ends up in any state other than deployed, and retry a release left failed instead of skipping the install. Add a stubbed-helm test. --- scripts/k8s/deploy_monitoring.sh | 16 +++- .../tests/test_deploy_monitoring.py | 96 +++++++++++++++++++ 2 files changed, 109 insertions(+), 3 deletions(-) create mode 100644 scripts/validation/tests/test_deploy_monitoring.py diff --git a/scripts/k8s/deploy_monitoring.sh b/scripts/k8s/deploy_monitoring.sh index a11ed4ef9..dd549df2e 100755 --- a/scripts/k8s/deploy_monitoring.sh +++ b/scripts/k8s/deploy_monitoring.sh @@ -142,8 +142,9 @@ function setup_prom_monitoring() { if ! kubectl get ns monitoring >/dev/null 2>&1 ; then kubectl create ns monitoring fi - if ! helm status -n monitoring kube-prometheus-stack >/dev/null 2>&1 ; then - helm upgrade --install \ + # Install when the release is missing or not healthy; a failed release is retried. + if ! helm status -n monitoring kube-prometheus-stack -o json 2>/dev/null | grep -Eq '"status": *"deployed"' ; then + if ! helm upgrade --install \ kube-prometheus-stack \ prometheus-community/kube-prometheus-stack \ --version "${HELM_PROMETHEUS_CHART_VERSION}" \ @@ -154,7 +155,16 @@ function setup_prom_monitoring() { --set grafana.ingress.hosts[0]="grafana-${ingress_ip_string}" \ --timeout 1200s \ ${helm_prom_oper_args} \ - ${helm_kube_prom_args} + ${helm_kube_prom_args} ; then + echo "ERROR: helm upgrade --install of kube-prometheus-stack failed" >&2 + exit 1 + fi + fi + # Never report success while the release is not deployed + if ! helm status -n monitoring kube-prometheus-stack -o json 2>/dev/null | grep -Eq '"status": *"deployed"' ; then + echo "ERROR: Helm release kube-prometheus-stack in namespace monitoring is not in 'deployed' state" >&2 + echo "Inspect with: helm status -n monitoring kube-prometheus-stack" >&2 + exit 1 fi } diff --git a/scripts/validation/tests/test_deploy_monitoring.py b/scripts/validation/tests/test_deploy_monitoring.py new file mode 100644 index 000000000..588e5b656 --- /dev/null +++ b/scripts/validation/tests/test_deploy_monitoring.py @@ -0,0 +1,96 @@ +"""Tests deploy_monitoring.sh exit codes using stubbed helm/kubectl (no cluster). + +Run with: python3 -m unittest discover scripts/validation/tests +""" + +import os +import shutil +import stat +import subprocess +import tempfile +import unittest + +REPO = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) +SCRIPT = os.path.join(REPO, "scripts", "k8s", "deploy_monitoring.sh") + +HELM = r"""#!/bin/bash +echo "helm $*" >> "$STUB_LOG" +case "$1" in + repo) [ "$2" = list ] && echo prometheus-community; exit 0 ;; + status) + n=$(cat "$STUB_STATE" 2>/dev/null || echo 0) + [ "$n" -lt 1 ] && [ -z "$STATUS_BEFORE" ] && exit 1 + if [ "$n" -ge 1 ]; then s="$STATUS_AFTER"; else s="$STATUS_BEFORE"; fi + echo "{\"info\": {\"status\": \"$s\"}}"; exit 0 ;; + upgrade) echo 1 > "$STUB_STATE"; exit "${UPGRADE_RC:-0}" ;; +esac +exit 0 +""" +KUBECTL = r"""#!/bin/bash +[ "$1 $2" = "get storageclass" ] && echo "standard (default)" +exit 0 +""" + + +class TestDeployMonitoringExit(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp() + k8s = os.path.join(self.tmp, "scripts", "k8s") + os.makedirs(k8s) + os.makedirs(os.path.join(self.tmp, "config")) + self.bin = os.path.join(self.tmp, "bin") + os.makedirs(self.bin) + shutil.copy(SCRIPT, k8s) + self.write(os.path.join(self.tmp, "scripts", "common.sh"), "") + for name in ("deploy_ingress.sh", "install_helm.sh"): + self.write(os.path.join(k8s, name), "#!/bin/bash\nexit 0\n") + self.write(os.path.join(self.bin, "helm"), HELM) + self.write(os.path.join(self.bin, "kubectl"), KUBECTL) + self.log = os.path.join(self.tmp, "log") + self.state = os.path.join(self.tmp, "state") + + def tearDown(self): + shutil.rmtree(self.tmp) + + @staticmethod + def write(path, text): + with open(path, "w") as f: + f.write(text) + os.chmod(path, os.stat(path).st_mode | stat.S_IXUSR) + + def run_script(self, **env): + e = dict(os.environ, PATH=self.bin + ":" + os.environ["PATH"], + STUB_LOG=self.log, STUB_STATE=self.state, **env) + r = subprocess.run(["bash", os.path.join(self.tmp, "scripts", "k8s", "deploy_monitoring.sh")], + cwd=self.tmp, env=e, capture_output=True, text=True, timeout=60) + calls = open(self.log).read() if os.path.exists(self.log) else "" + return r, calls + + def test_fresh_install_succeeds(self): + r, calls = self.run_script(STATUS_AFTER="deployed") + self.assertEqual(r.returncode, 0, r.stderr) + self.assertIn("helm upgrade", calls) + + def test_already_deployed_skips_install(self): + r, calls = self.run_script(STATUS_BEFORE="deployed", STATUS_AFTER="deployed") + self.assertEqual(r.returncode, 0, r.stderr) + self.assertNotIn("helm upgrade", calls) + + def test_helm_upgrade_failure_exits_nonzero(self): + r, _ = self.run_script(UPGRADE_RC="1", STATUS_AFTER="failed") + self.assertNotEqual(r.returncode, 0) + self.assertIn("failed", r.stderr) + + def test_release_left_failed_exits_nonzero(self): + r, _ = self.run_script(STATUS_AFTER="failed") + self.assertNotEqual(r.returncode, 0) + self.assertIn("not in 'deployed' state", r.stderr) + + def test_existing_failed_release_is_retried(self): + r, calls = self.run_script(STATUS_BEFORE="failed", STATUS_AFTER="deployed") + self.assertEqual(r.returncode, 0, r.stderr) + self.assertIn("helm upgrade", calls) + + +if __name__ == "__main__": + unittest.main()