feat: add deploy gate silent-fail watchdog

.github/dependabot.yml

@@ -11,3 +11,30 @@ updates:
       all-github-actions:
         patterns:
           - "*"
+
+  # actionlint is pinned as a tool directive in tools/actionlint/go.mod precisely so this
+  # entry can bump it. A version pinned where Dependabot cannot see it is a version that
+  # stays where it was put.
+  - package-ecosystem: gomod
+    directory: /tools/actionlint
+    schedule:
+      interval: weekly
+    commit-message:
+      prefix: "fix"
+      include: "scope"
+    groups:
+      all-go-tools:
+        patterns:
+          - "*"
+
+  - package-ecosystem: pip
+    directory: /tests
+    schedule:
+      interval: weekly
+    commit-message:
+      prefix: "fix"
+      include: "scope"
+    groups:
+      all-test-deps:
+        patterns:
+          - "*"

.github/workflows/actions-watchdog-deploy-gate.yaml

@@ -0,0 +1,207 @@
+name: Deploy gate watchdog
+
+# A "deploy gate" is the workflow whose success is what actually publishes an artifact:
+# docker-build-push-jfrog and friends. When that gate goes red on the release branch,
+# nothing is pushed, the deployer has nothing new to sync, and production quietly keeps
+# serving the last image that made it through. Nothing in GitHub makes that loud — a red
+# run on the default branch is a grey cross in a commit list nobody is watching — so the
+# branch can sit unshipped for days while looking merged and done.
+#
+# This workflow asks the only question that matters, on a schedule: is the head of the
+# release branch represented by a *successful* run of the gate? If not, it files one
+# deduplicated issue and fails, and it closes that issue again when the branch ships.
+#
+# It is deliberately a watchdog rather than an on-failure hook. A hook is a strictly
+# weaker signal: it can only fire when the gate ran and lost. It cannot fire when the
+# gate never ran at all — workflow disabled, trigger deleted, runner outage, a push that
+# matched no path filter — and those are the failures that stay hidden longest.
+
+on:
+  workflow_call:
+    inputs:
+      workflow_file:
+        description: "File name of the gating workflow, e.g. build-images.yaml. The file name rather than the display name, so renaming the workflow's `name:` cannot silently detach the watchdog."
+        type: string
+        required: true
+      branch:
+        description: "Branch whose head must be shipped."
+        type: string
+        required: false
+        default: "main"
+      grace_minutes:
+        description: "How long a commit may go unshipped before it counts as stale. Must cover a normal end-to-end run of the gate, or every push trips the watchdog while it is still building."
+        type: number
+        required: false
+        default: 90
+      issue_label:
+        description: "Label used to find and deduplicate the watchdog's issue. One open issue per label, reused across consecutive failures."
+        type: string
+        required: false
+        default: "deploy-gate-stale"
+      issue_assignees:
+        description: "Comma-separated GitHub usernames to assign the issue to. Empty leaves it unassigned."
+        type: string
+        required: false
+        default: ""
+
+permissions:
+  contents: read
+  # Reading workflow run history is the whole check; writing issues is the whole alert.
+  actions: read
+  issues: write
+
+jobs:
+  watchdog:
+    name: Check deploy gate
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check gate freshness
+        id: check
+        env:
+          GH_TOKEN: ${{ github.token }}
+          REPO: ${{ github.repository }}
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          INPUT_WORKFLOW_FILE: ${{ inputs.workflow_file }}
+          INPUT_BRANCH: ${{ inputs.branch }}
+          INPUT_GRACE_MINUTES: ${{ inputs.grace_minutes }}
+          INPUT_ISSUE_LABEL: ${{ inputs.issue_label }}
+          INPUT_ISSUE_ASSIGNEES: ${{ inputs.issue_assignees }}
+        run: |
+          set -euo pipefail
+
+          # Exit codes are distinct so the tests can assert *which* thing happened rather
+          # than only "non-zero": 0 shipped or still inside the grace window, 1 stale,
+          # 2 called with arguments this cannot act on.
+          die() { echo "::error::$*"; exit 2; }
+          summary() { echo "$*" >> "${GITHUB_STEP_SUMMARY:-/dev/null}"; }
+
+          # Validate before touching the API. Allowlists, not blocklists: every one of
+          # these values is interpolated into a REST path, a query string or a gh
+          # argument, and "reject what looks wrong" is how a traversal or an injected
+          # query parameter gets through.
+          [[ "${INPUT_WORKFLOW_FILE}" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*\.(yml|yaml)$ ]] ||
+            die "workflow_file must be a workflow file name such as build-images.yaml, got: ${INPUT_WORKFLOW_FILE}"
+          [[ "${INPUT_BRANCH}" =~ ^[A-Za-z0-9][A-Za-z0-9._/-]*$ ]] ||
+            die "branch must be a plain branch name, got: ${INPUT_BRANCH}"
+          [[ "${INPUT_GRACE_MINUTES}" =~ ^[0-9]+$ ]] ||
+            die "grace_minutes must be a non-negative integer, got: ${INPUT_GRACE_MINUTES}"
+          [[ "${INPUT_ISSUE_LABEL}" =~ ^[A-Za-z0-9][A-Za-z0-9._:-]{0,48}$ ]] ||
+            die "issue_label must be 1-49 characters of [A-Za-z0-9._:-], got: ${INPUT_ISSUE_LABEL}"
+          [[ "${INPUT_ISSUE_ASSIGNEES}" =~ ^([A-Za-z0-9-]+(,[A-Za-z0-9-]+)*)?$ ]] ||
+            die "issue_assignees must be comma-separated GitHub usernames, got: ${INPUT_ISSUE_ASSIGNEES}"
+
+          work="$(mktemp -d)"
+
+          # --- observe -------------------------------------------------------------
+
+          gh api "repos/${REPO}/commits/${INPUT_BRANCH}" > "${work}/head.json"
+          head_sha="$(jq -r '.sha' "${work}/head.json")"
+          head_date="$(jq -r '.commit.committer.date' "${work}/head.json")"
+
+          # `status` on this endpoint takes conclusions as well as statuses, so
+          # status=success means "concluded successfully". Worth knowing: an unrecognised
+          # value there returns an empty list rather than a 4xx, so if that enum ever
+          # changes under us this reports a stale branch instead of silently reporting a
+          # healthy one. Wrong in the direction that gets looked at.
+          gh api "repos/${REPO}/actions/workflows/${INPUT_WORKFLOW_FILE}/runs?branch=${INPUT_BRANCH}&status=success&per_page=1" \
+            > "${work}/last_ok.json"
+          last_ok_sha="$(jq -r '.workflow_runs[0].head_sha // ""' "${work}/last_ok.json")"
+          last_ok_url="$(jq -r '.workflow_runs[0].html_url // ""' "${work}/last_ok.json")"
+
+          echo "branch head:       ${head_sha} (${head_date})"
+          echo "last shipped head: ${last_ok_sha:-<none on record>}"
+
+          # `gh issue list --label` errors on a label that does not exist yet rather than
+          # returning an empty list, so the label is created first. Already-exists is the
+          # normal case and is not a failure.
+          gh label create "${INPUT_ISSUE_LABEL}" --repo "${REPO}" --color B60205 \
+            --description "Release branch is not represented by a successful deploy gate run" \
+            >/dev/null 2>&1 || true
+
+          issue="$(gh issue list --repo "${REPO}" --label "${INPUT_ISSUE_LABEL}" \
+            --state open --limit 1 --json number --jq '.[0].number // empty')"
+
+          # --- shipped: close anything the watchdog opened earlier -------------------
+
+          if [[ -n "${last_ok_sha}" && "${last_ok_sha}" == "${head_sha}" ]]; then
+            echo "gate is current"
+            summary "### Deploy gate current"
+            summary "\`${INPUT_BRANCH}\` head \`${head_sha}\` shipped by [${INPUT_WORKFLOW_FILE}](${last_ok_url})."
+
+            if [[ -n "${issue}" ]]; then
+              # shellcheck disable=SC2016  # backticks here are markdown code spans
+              {
+                printf 'Recovered. `%s` head `%s` was shipped by [%s](%s).\n\n' \
+                  "${INPUT_BRANCH}" "${head_sha}" "${INPUT_WORKFLOW_FILE}" "${last_ok_url}"
+                printf 'Closed by the [deploy gate watchdog](%s).\n' "${RUN_URL}"
+              } > "${work}/recovered.md"
+              gh issue comment "${issue}" --repo "${REPO}" --body-file "${work}/recovered.md"
+              gh issue close "${issue}" --repo "${REPO}" --reason completed
+              echo "closed issue #${issue}"
+            fi
+            exit 0
+          fi
+
+          # --- unshipped, but possibly still building -------------------------------
+
+          # python3 rather than `date -d`: the same parse then works on a developer's
+          # macOS while running the tests, so the tested code and the shipped code are
+          # not two different implementations.
+          head_epoch="$(python3 -c 'import datetime, sys; print(int(datetime.datetime.strptime(sys.argv[1], "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=datetime.timezone.utc).timestamp()))' "${head_date}")"
+          age_minutes=$(( ( $(date -u +%s) - head_epoch ) / 60 ))
+
+          if (( age_minutes < INPUT_GRACE_MINUTES )); then
+            echo "head is ${age_minutes}m old, inside the ${INPUT_GRACE_MINUTES}m grace window"
+            summary "### Deploy gate pending"
+            summary "\`${head_sha}\` is ${age_minutes}m old, inside the ${INPUT_GRACE_MINUTES}m grace window."
+            exit 0
+          fi
+
+          # --- stale ---------------------------------------------------------------
+
+          gh api "repos/${REPO}/actions/workflows/${INPUT_WORKFLOW_FILE}/runs?head_sha=${head_sha}&per_page=1" \
+            > "${work}/head_run.json"
+          run_status="$(jq -r '.workflow_runs[0].status // "never ran"' "${work}/head_run.json")"
+          run_conclusion="$(jq -r '.workflow_runs[0].conclusion // "none"' "${work}/head_run.json")"
+          gate_run_url="$(jq -r '.workflow_runs[0].html_url // ""' "${work}/head_run.json")"
+
+          # shellcheck disable=SC2016  # backticks here are markdown code spans
+          {
+            printf '`%s` is at `%s`, committed %s minutes ago, and no successful run of `%s` has that commit as its head. Whatever consumes this gate is still serving the previous artifact.\n\n' \
+              "${INPUT_BRANCH}" "${head_sha}" "${age_minutes}" "${INPUT_WORKFLOW_FILE}"
+            printf '| | |\n|---|---|\n'
+            printf '| Branch head | `%s` |\n' "${head_sha}"
+            printf '| Head commit age | %s minutes |\n' "${age_minutes}"
+            printf '| Gate run for that head | %s / %s %s |\n' "${run_status}" "${run_conclusion}" "${gate_run_url}"
+            printf '| Last head that shipped | %s %s |\n\n' "${last_ok_sha:-none on record}" "${last_ok_url}"
+            printf 'Filed by the [deploy gate watchdog](%s). It closes this issue by itself once `%s` ships again.\n' \
+              "${RUN_URL}" "${INPUT_BRANCH}"
+          } > "${work}/stale.md"
+
+          if [[ -z "${issue}" ]]; then
+            assignees=()
+            if [[ -n "${INPUT_ISSUE_ASSIGNEES}" ]]; then
+              assignees=(--assignee "${INPUT_ISSUE_ASSIGNEES}")
+            fi
+            gh issue create --repo "${REPO}" \
+              --title "Deploy gate stale: ${INPUT_WORKFLOW_FILE} has not succeeded for ${INPUT_BRANCH}" \
+              --label "${INPUT_ISSUE_LABEL}" ${assignees[@]+"${assignees[@]}"} \
+              --body-file "${work}/stale.md"
+            echo "opened an issue"
+          else
+            # Already reported for this exact head? The watchdog runs hourly, and an
+            # hourly comment about one unchanged commit trains people to mute the issue,
+            # which rebuilds the silence this exists to remove.
+            gh issue view "${issue}" --repo "${REPO}" --json body,comments \
+              --jq '.body, (.comments[].body)' > "${work}/reported.txt"
+            if grep -qF "${head_sha}" "${work}/reported.txt"; then
+              echo "issue #${issue} already reports ${head_sha}"
+            else
+              gh issue comment "${issue}" --repo "${REPO}" --body-file "${work}/stale.md"
+              echo "commented on issue #${issue}"
+            fi
+          fi
+
+          summary "### Deploy gate stale"
+          summary "\`${INPUT_BRANCH}\` head \`${head_sha}\` is ${age_minutes}m old and unshipped. Gate run: ${run_status} / ${run_conclusion}."
+          exit 1

.github/workflows/ci.yaml

@@ -0,0 +1,50 @@
+name: CI
+
+# This repository ships workflows that other repositories run, so a mistake here lands in
+# everyone's pipeline at once and shows up as their build breaking, not ours. Both jobs
+# below are the same commands a developer runs locally — tests/lint_workflows.sh and
+# tests/deploy-gate-watchdog/run_tests.sh — so there is nothing CI checks that cannot be
+# reproduced from a terminal, and nothing a terminal checks that CI skips.
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  lint:
+    name: Lint workflows
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      # actionlint's version is a tool directive in tools/actionlint/go.mod rather than a
+      # string in this file, because a version written here is invisible to Dependabot and
+      # rots in place. shellcheck comes from the runner image; actionlint finds it and runs
+      # it over every run: block.
+      - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
+        with:
+          go-version-file: tools/actionlint/go.mod
+
+      - run: tests/lint_workflows.sh
+
+  watchdog-tests:
+    name: Deploy gate watchdog
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: "3.13"
+
+      - run: pip install -r tests/requirements.txt
+
+      - run: tests/deploy-gate-watchdog/run_tests.sh

README.md

@@ -52,6 +52,22 @@ Read about it in the [examples/docker/README.md](examples/docker/README.md) file
 
 Read about it in the [examples/ai-review/README.md](examples/ai-review/README.md) file.
 
+## Deploy Gate Watchdog
+
+Catches the case where a build-and-push workflow has been failing on the release branch
+and nobody noticed, so production is still serving an old image. Read about it in the
+[examples/watchdog/README.md](examples/watchdog/README.md) file.
+
+## Working on this repository
+
+```shell
+tests/lint_workflows.sh                      # actionlint over the workflows
+tests/deploy-gate-watchdog/run_tests.sh      # watchdog behaviour
+```
+
+Both are exactly what CI runs. `tests/lint_workflows.sh` needs Go; the watchdog tests
+need python3 with the pin in [tests/requirements.txt](tests/requirements.txt).
+
 
 ## License
 

examples/watchdog/README.md

@@ -0,0 +1,77 @@
+# Deploy Gate Watchdog
+
+**Implementation:** [`.github/workflows/actions-watchdog-deploy-gate.yaml`](../../.github/workflows/actions-watchdog-deploy-gate.yaml)
+
+**Example:** [`examples/watchdog/deploy-gate-watchdog.yml`](./deploy-gate-watchdog.yml)
+
+## The failure this exists for
+
+A build-and-push workflow is a deploy gate: when it succeeds an image lands in the
+registry and the deployer picks it up, and when it fails nothing lands and the deployer
+has nothing new to sync. That second case is silent. Production keeps serving the last
+image that made it through, the branch still looks merged and green in the PR list, and
+the only signal is a grey cross on a commit page nobody opens.
+
+It happened on `angkor-platform-frontend`: the Trivy stage of the build started failing
+before the push step, main went unshipped for days, and production stayed on an image
+with known auth-bypass CVEs. It surfaced only because someone shipped an unrelated fix
+and noticed it never appeared.
+
+This workflow asks, on a schedule, the question that was never being asked: **is the head
+of the release branch represented by a successful run of the gate?** If not, it opens one
+issue and fails. When the branch ships again it closes the issue by itself.
+
+It is deliberately a watchdog and not an on-failure notification. A failure hook is a
+strictly weaker signal — it can only fire when the gate ran and lost. It cannot fire when
+the gate never ran at all: workflow disabled, trigger deleted or path-filtered away,
+runner outage, a schedule that stopped. Those are the failures that stay hidden longest,
+and they are the ones a watchdog catches for free.
+
+## Usage
+
+Copy [`deploy-gate-watchdog.yml`](./deploy-gate-watchdog.yml) into `.github/workflows/`
+and set `workflow_file` to the workflow that publishes your artifact.
+
+The caller must grant `actions: read` (to read run history) and `issues: write` (to file
+the alert). No secrets, no webhooks: it runs on the built-in `GITHUB_TOKEN`.
+
+## Inputs
+
+| Input | Default | Notes |
+|---|---|---|
+| `workflow_file` | *required* | File name of the gating workflow, e.g. `build-images.yaml`. The file name and not the display name, so renaming the workflow's `name:` cannot silently detach the watchdog. |
+| `branch` | `main` | Branch whose head must be shipped. |
+| `grace_minutes` | `90` | How long a commit may go unshipped before it counts as stale. Must comfortably exceed an end-to-end run of the gate, or every push trips the watchdog while it is still building. |
+| `issue_label` | `deploy-gate-stale` | Used to find and deduplicate the issue. One open issue per label. |
+| `issue_assignees` | *(none)* | Comma-separated usernames to assign the issue to. |
+
+Every input is validated against an allowlist before any API call, and a rejected input
+fails the run rather than being cleaned up and used.
+
+## Behaviour
+
+| Situation | Result |
+|---|---|
+| Branch head has a successful gate run | Succeeds. Closes the watchdog's issue if one is open. |
+| Branch head unshipped, younger than `grace_minutes` | Succeeds quietly. The build is presumed to still be running. |
+| Branch head unshipped and older than `grace_minutes` | Fails, and opens an issue naming the head, its age, and what the gate did for that commit. |
+| Same head still stale on the next run | Fails, comments nothing. An hourly comment about one unchanged commit is how an issue gets muted. |
+| A newer head is also stale | Fails, and comments on the existing issue with the new head. |
+
+Exit codes are distinct on purpose: `0` shipped or within grace, `1` stale, `2` called
+with arguments it cannot act on.
+
+## Choosing `grace_minutes`
+
+Set it above the p95 wall-clock time of the gate, measured rather than guessed, and leave
+room for queueing. Too low and the watchdog files an issue against a build that was going
+to succeed five minutes later, which is the fastest way to teach people to ignore it. Too
+high and a genuinely broken main stays quiet for that long. The 90 minute default suits a
+multi-arch image build with a scan stage.
+
+## Tests
+
+`tests/deploy-gate-watchdog/run_tests.sh`, runnable locally with nothing but bash and
+python3. The suite runs the step body extracted from the workflow file itself against a
+stubbed `gh`, so there is one copy of the logic and the tests are pinned to the text that
+actually ships.