From 425335bb2a277dade9a2a2633c8032784c54ab26 Mon Sep 17 00:00:00 2001 From: ErenAri Date: Sun, 18 Oct 2026 23:33:27 +0300 Subject: [PATCH 1/5] readiness: date the Falco run from its first target, not the report production-readiness-report.sh required the Falco report's run.started_at to postdate the candidate's generated_at. The runner writes run.started_at when it assembles the report, after every target ran (runner.go:308-311); in each preserved real report it is minutes later than the first target's start. A run that began before the candidate was built could therefore be accepted as starting after it, contradicting the written rule that the Falco run must start after the candidate was built. The run now starts at the earliest of run.started_at and every target's started_at, and a report with a target that records no start time is refused. The manifest still names the report's run.started_at, unchanged. Signed-off-by: ErenAri --- scripts/production-readiness-report.sh | 10 ++++++++++ scripts/production-readiness-report_test.sh | 11 +++++++++++ 2 files changed, 21 insertions(+) diff --git a/scripts/production-readiness-report.sh b/scripts/production-readiness-report.sh index e8078e6..3ee3770 100755 --- a/scripts/production-readiness-report.sh +++ b/scripts/production-readiness-report.sh @@ -350,6 +350,16 @@ falco_report_started_at="$(jq -r '.run.started_at // ""' "$falco_report")" [[ "$(jq -r '.falco.started_at' "$input")" == "$falco_report_started_at" ]] || fail "Falco started_at does not match the report's run.started_at (${falco_report_started_at:-missing})" falco_started_epoch="$(iso_epoch "$falco_report_started_at" "Falco report run.started_at")" +# run.started_at is written when the report is assembled, after the targets +# ran, so it can postdate the candidate while the run itself began before it. +# The run started at the earliest of run.started_at and every target's +# started_at, and every target must record one. +jq -e '[.targets[] | select((.started_at // "") == "")] | length == 0' "$falco_report" >/dev/null || + fail "Falco report has a target without started_at, so when the run started cannot be established" +while IFS= read -r target_started_at; do + target_epoch="$(iso_epoch "$target_started_at" "Falco report target started_at")" + ((target_epoch < falco_started_epoch)) && falco_started_epoch="$target_epoch" +done < <(jq -r '.targets[].started_at' "$falco_report") [[ "$falco_started_epoch" -gt "$candidate_generated_epoch" ]] || fail "Falco run started before release candidate ${candidate_tag} was built" # Written policy does not say the Falco run must execute the candidate's own diff --git a/scripts/production-readiness-report_test.sh b/scripts/production-readiness-report_test.sh index 390cd25..3357881 100755 --- a/scripts/production-readiness-report_test.sh +++ b/scripts/production-readiness-report_test.sh @@ -23,6 +23,7 @@ make_report() { profile_id: ., required: true, status: "pass", + started_at: $started_at, duration_ms: 1000 }], summary: {status: "pass"}, @@ -478,6 +479,16 @@ jq 'del(.run.started_at)' "$falco_report.keep" >"$falco_report" rehash "$falco_report" .falco.report_sha256 "$tmp/bad-falco-nostart.json" expect_refused "a Falco report without a run start" "$tmp/bad-falco-nostart.json" \ "Falco started_at does not match the report's run.started_at" +# run.started_at is written when the report is assembled, after the targets +# ran: a run whose first target predates the candidate must not pass on it. +jq '.targets[0].started_at = "2026-07-30T13:59:59Z"' "$falco_report.keep" >"$falco_report" +rehash "$falco_report" .falco.report_sha256 "$tmp/bad-falco-early-target.json" +expect_refused "a Falco run whose first target predates the candidate" "$tmp/bad-falco-early-target.json" \ + "Falco run started before release candidate" +jq 'del(.targets[2].started_at)' "$falco_report.keep" >"$falco_report" +rehash "$falco_report" .falco.report_sha256 "$tmp/bad-falco-target-nostart.json" +expect_refused "a Falco report with a target that has no start time" "$tmp/bad-falco-target-nostart.json" \ + "Falco report has a target without started_at" restore "$falco_report" # Rollback: alias, drilled digest and recovery are this candidate's. From df47e0fce3e9ceee9b86c5a42e2eb7bf7d21ac65 Mon Sep 17 00:00:00 2001 From: ErenAri Date: Sun, 18 Oct 2026 23:51:31 +0300 Subject: [PATCH 2/5] readiness: ignore unbooted optional profiles when dating the Falco run Review of the previous commit: the runner writes no started_at for a target it never runs, such as an optional profile the default runner cannot boot (status unsupported; fail in v0.3.7), and the shipped expanded matrix has three. Refusing every target without started_at would refuse such a legitimate report. Now every required or passing target must record started_at, and the earliest recorded start is the run's start. The runbook describes the rule, and the test gains a positive case shaped like a real report (targets start before run.started_at) with an unbooted optional profile. Signed-off-by: ErenAri --- docs/production-release-process.md | 7 +++++-- scripts/production-readiness-report.sh | 10 ++++++---- scripts/production-readiness-report_test.sh | 18 ++++++++++++++++-- 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/docs/production-release-process.md b/docs/production-release-process.md index f15d6bd..ed43d8d 100644 --- a/docs/production-release-process.md +++ b/docs/production-release-process.md @@ -166,8 +166,11 @@ aggregator binds every piece of evidence to the attested candidate (`generated_at`, tag, commit, image digest, evidence SHA-256): - **Falco.** `falco.started_at` must equal the report's own `run.started_at`, - and, as for the campaigns, that run must start after the candidate's - `generated_at`: evidence must postdate the candidate it graduates. + and the run must start after the candidate's `generated_at`: evidence must + postdate the candidate it graduates. The runner writes `run.started_at` when + it assembles the report, so the run's start is the earliest of that and every + target's `started_at`; a required or passing target without `started_at` is + refused, while an optional profile the runner never booted does not count. `workflow_run_id` and `commit_sha` are not recorded in the report and remain operator-asserted (`falco.run_identity` in the output). Written policy does not require the Falco lane to run the candidate's own bpfcompat, so this is diff --git a/scripts/production-readiness-report.sh b/scripts/production-readiness-report.sh index 3ee3770..cdb0ea8 100755 --- a/scripts/production-readiness-report.sh +++ b/scripts/production-readiness-report.sh @@ -353,13 +353,15 @@ falco_started_epoch="$(iso_epoch "$falco_report_started_at" "Falco report run.st # run.started_at is written when the report is assembled, after the targets # ran, so it can postdate the candidate while the run itself began before it. # The run started at the earliest of run.started_at and every target's -# started_at, and every target must record one. -jq -e '[.targets[] | select((.started_at // "") == "")] | length == 0' "$falco_report" >/dev/null || - fail "Falco report has a target without started_at, so when the run started cannot be established" +# started_at. Every required or passing target must record one; an optional +# profile the runner never booted (no started_at, not passing) does not count. +jq -e '[.targets[] | select((.required == true or .status == "pass") and (.started_at // "") == "")] | length == 0' \ + "$falco_report" >/dev/null || + fail "Falco report has a target that ran without started_at, so when the run started cannot be established" while IFS= read -r target_started_at; do target_epoch="$(iso_epoch "$target_started_at" "Falco report target started_at")" ((target_epoch < falco_started_epoch)) && falco_started_epoch="$target_epoch" -done < <(jq -r '.targets[].started_at' "$falco_report") +done < <(jq -r '.targets[] | .started_at // empty' "$falco_report") [[ "$falco_started_epoch" -gt "$candidate_generated_epoch" ]] || fail "Falco run started before release candidate ${candidate_tag} was built" # Written policy does not say the Falco run must execute the candidate's own diff --git a/scripts/production-readiness-report_test.sh b/scripts/production-readiness-report_test.sh index 3357881..44c8fba 100755 --- a/scripts/production-readiness-report_test.sh +++ b/scripts/production-readiness-report_test.sh @@ -487,8 +487,22 @@ expect_refused "a Falco run whose first target predates the candidate" "$tmp/bad "Falco run started before release candidate" jq 'del(.targets[2].started_at)' "$falco_report.keep" >"$falco_report" rehash "$falco_report" .falco.report_sha256 "$tmp/bad-falco-target-nostart.json" -expect_refused "a Falco report with a target that has no start time" "$tmp/bad-falco-target-nostart.json" \ - "Falco report has a target without started_at" +expect_refused "a Falco report whose required target has no start time" "$tmp/bad-falco-target-nostart.json" \ + "Falco report has a target that ran without started_at" +# Shaped like a real report: targets start minutes before run.started_at, all +# after the candidate; an optional profile the runner never booted +# ("unsupported", no started_at) does not count. +jq '.targets |= map(.started_at = "2026-08-03T05:50:00Z") | + .targets += [{profile_id: "talos-1.7", required: false, status: "unsupported", duration_ms: 0}]' \ + "$falco_report.keep" >"$falco_report" +rehash "$falco_report" .falco.report_sha256 "$tmp/good-falco-real-shape.json" +if BPFCOMPAT_SKIP_READINESS_ATTESTATION=1 "$script" "$tmp/good-falco-real-shape.json" "$tmp/good.md" >"$tmp/falco-real-shape.log" 2>&1; then + echo "[production-readiness-test] accepted: a real-shaped Falco report with an unbooted optional profile" +else + cat "$tmp/falco-real-shape.log" >&2 + echo "[production-readiness-test] a real-shaped Falco report with an unbooted optional profile was refused" >&2 + exit 1 +fi restore "$falco_report" # Rollback: alias, drilled digest and recovery are this candidate's. From e033fe6be8c3b9e7128b9391e1c5305c328fc4af Mon Sep 17 00:00:00 2001 From: ErenAri Date: Sun, 18 Oct 2026 23:54:57 +0300 Subject: [PATCH 3/5] readiness: look up the drill and incident runs it is given Written policy requires the rollback evidence to be the note rollback-drill.yml writes and the incident note to link the deliberately rejected promote-release run. The readiness report checked only that each was a well-formed run ID, so (verified by reproduction) a run ID that does not exist, or a successful ci.yml run, was accepted with attestations verified. With attestation verification on, the drill run must now be a successful rollback-drill.yml run on main and the incident run a completed promote-release.yml run that did not succeed; an unreadable run fails. Which rejection qualifies stays the operator's choice; a successful or unrelated run never does. The test-only bypass is unchanged. Signed-off-by: ErenAri --- docs/production-release-process.md | 8 +++-- scripts/production-readiness-report.sh | 18 ++++++++++ scripts/production-readiness-report_test.sh | 39 +++++++++++++++++++++ 3 files changed, 63 insertions(+), 2 deletions(-) diff --git a/docs/production-release-process.md b/docs/production-release-process.md index ed43d8d..3852a77 100644 --- a/docs/production-release-process.md +++ b/docs/production-release-process.md @@ -198,8 +198,12 @@ aggregator binds every piece of evidence to the attested candidate - **Incident.** The note must contain `[bpfcompat-promotion-incident:v1]`, name `promote-release`, and link the deliberately rejected run, and only that run, as - `https://github.com/Kernel-Guard/bpfcompat/actions/runs/`. Written - policy does not require the incident drill to target the current candidate, + `https://github.com/Kernel-Guard/bpfcompat/actions/runs/`. With + attestation verification on, that run must be a completed `promote-release.yml` + run that did not succeed, and the rollback evidence's workflow run must be a + successful `rollback-drill.yml` run on `main`; both are looked up on GitHub. + Written policy does not require the incident drill to target the current + candidate, so an earlier candidate's fail-closed drill is accepted; the output records `incident.promotion_run_id` and `incident.candidate_bound`. - **Operator.** The confirmation must contain diff --git a/scripts/production-readiness-report.sh b/scripts/production-readiness-report.sh index cdb0ea8..0ffd9eb 100755 --- a/scripts/production-readiness-report.sh +++ b/scripts/production-readiness-report.sh @@ -545,6 +545,24 @@ incident_run_ids="$( [[ "$(wc -l <<<"$incident_run_ids")" -eq 1 ]] || fail "incident evidence links more than one workflow run; it must link only the rejected promote-release run" incident_run_id="$incident_run_ids" +# Written policy: the rollback evidence is the note rollback-drill.yml writes, +# and the incident links the deliberately rejected promote-release run. With +# attestation verification on, both run IDs must be real runs of those +# workflows here: the drill a successful run on main, the incident a completed +# promote-release run that did not succeed. (Which rejection qualifies stays +# the operator's call; a successful or unrelated run never does.) +if [[ "${BPFCOMPAT_SKIP_READINESS_ATTESTATION:-0}" != "1" ]]; then + rollback_run="$(gh api "repos/Kernel-Guard/bpfcompat/actions/runs/${rollback_run_id}" 2>/dev/null)" || + fail "rollback drill run ${rollback_run_id} cannot be read from GitHub" + jq -e '.path == ".github/workflows/rollback-drill.yml" and .head_branch == "main" and + .status == "completed" and .conclusion == "success"' <<<"$rollback_run" >/dev/null || + fail "rollback drill run ${rollback_run_id} is not a successful rollback-drill.yml run on main" + incident_run="$(gh api "repos/Kernel-Guard/bpfcompat/actions/runs/${incident_run_id}" 2>/dev/null)" || + fail "incident run ${incident_run_id} cannot be read from GitHub" + jq -e '.path == ".github/workflows/promote-release.yml" and + .status == "completed" and .conclusion != "success"' <<<"$incident_run" >/dev/null || + fail "incident run ${incident_run_id} is not a rejected promote-release.yml run" +fi incident_candidate_bound=false if grep -qE "(^|[^[:alnum:].-])${candidate_tag//./\\.}([^[:alnum:]]|$)" "$incident"; then incident_candidate_bound=true diff --git a/scripts/production-readiness-report_test.sh b/scripts/production-readiness-report_test.sh index 44c8fba..8b80bf3 100755 --- a/scripts/production-readiness-report_test.sh +++ b/scripts/production-readiness-report_test.sh @@ -626,6 +626,21 @@ mkdir -p "$tmp/bin" cat >"$tmp/bin/gh" <<'STUB' #!/usr/bin/env bash [[ "$1" == attestation && "$2" == --help ]] && exit 0 +# Workflow run lookups: run 5000 is the rollback drill and 6000 the rejected +# promotion; STUB_* variables override their fields, any other run is a 404. +if [[ "$1" == api && "$2" =~ ^repos/Kernel-Guard/bpfcompat/actions/runs/([0-9]+)$ ]]; then + printf '%s\n' "$*" >>"$STUB_LOG" + case "${BASH_REMATCH[1]}" in + 5000) jq -n --arg p "${STUB_ROLLBACK_PATH:-.github/workflows/rollback-drill.yml}" \ + --arg b "${STUB_ROLLBACK_BRANCH:-main}" --arg c "${STUB_ROLLBACK_CONCLUSION:-success}" \ + '{path: $p, head_branch: $b, status: "completed", conclusion: $c}' ;; + 6000) jq -n --arg p "${STUB_INCIDENT_PATH:-.github/workflows/promote-release.yml}" \ + --arg c "${STUB_INCIDENT_CONCLUSION:-failure}" \ + '{path: $p, head_branch: "v0.4.0-rc.1", status: "completed", conclusion: $c}' ;; + *) echo "gh: Not Found (HTTP 404)" >&2; exit 1 ;; + esac + exit 0 +fi [[ "$1 $2" == "attestation verify" ]] || { echo "unexpected gh $*" >&2; exit 99; } printf '%s\n' "$*" >>"$STUB_LOG" [[ -n "${STUB_FAIL:-}" && "$3" == *"$STUB_FAIL"* ]] && exit 1 @@ -661,6 +676,30 @@ if PATH="$tmp/bin:$PATH" STUB_LOG="$tmp/gh.log" STUB_EVENT=workflow_dispatch "$s echo "[production-readiness-test] a dispatched run relabelled as scheduled was accepted" >&2; exit 1 fi echo "[production-readiness-test] refused: evidence claiming schedule whose attestation shows workflow_dispatch" +grep -Fq "api repos/Kernel-Guard/bpfcompat/actions/runs/5000" "$tmp/gh.log" && + grep -Fq "api repos/Kernel-Guard/bpfcompat/actions/runs/6000" "$tmp/gh.log" || { + echo "[production-readiness-test] the drill and incident runs were not looked up" >&2; exit 1; } +# Written policy: the rollback note comes from a rollback-drill.yml run and the +# incident links the rejected promote-release run. Both must be real runs. +for case in \ + "STUB_ROLLBACK_PATH=.github/workflows/ci.yml|a rollback drill run that is another workflow" \ + "STUB_ROLLBACK_CONCLUSION=failure|a rollback drill run that did not succeed" \ + "STUB_ROLLBACK_BRANCH=feature|a rollback drill run not on main" \ + "STUB_INCIDENT_PATH=.github/workflows/ci.yml|an incident linking a run of another workflow" \ + "STUB_INCIDENT_CONCLUSION=success|an incident linking a promotion that succeeded"; do + if env "${case%%|*}" PATH="$tmp/bin:$PATH" STUB_LOG="$tmp/gh.log" "$script" "$tmp/input.json" "$tmp/bad.md" >/dev/null 2>&1; then + echo "[production-readiness-test] ${case#*|} was accepted" >&2; exit 1 + fi + echo "[production-readiness-test] refused: ${case#*|}" +done +sed 's#/actions/runs/6000#/actions/runs/6999#' "$incident_note" >"$tmp/incident-404.md" +cp "$incident_note" "$incident_note.keep"; cp "$tmp/incident-404.md" "$incident_note" +rehash "$incident_note" .incident.evidence_sha256 "$tmp/bad-incident-404.json" +if PATH="$tmp/bin:$PATH" STUB_LOG="$tmp/gh.log" "$script" "$tmp/bad-incident-404.json" "$tmp/bad.md" >/dev/null 2>&1; then + echo "[production-readiness-test] an incident linking a run that does not exist was accepted" >&2; exit 1 +fi +echo "[production-readiness-test] refused: an incident linking a run that does not exist" +mv "$incident_note.keep" "$incident_note" # --- Readiness output v0.2 and the real stable gate ---------------------------- jq -e --arg c "$(printf 'c%.0s' {1..40})" ' From 7f7a4effe2860cc4ee4219b5641d14635afc7b8d Mon Sep 17 00:00:00 2001 From: ErenAri Date: Mon, 19 Oct 2026 00:15:39 +0300 Subject: [PATCH 4/5] readiness: bind the looked-up drill run to this repository, dispatch and time Review of e033fe6: the drill lookup accepted a fork's pull-request run from a branch named main, any old rollback-drill run, and a note whose Completed time was unrelated to the run (2099). The drill run must now be a dispatched run of this repository that started after the candidate was built, with the note's Completed time inside it; the incident run must be a dispatched promote-release run of this repository. The runbook says the lookups need the network and an authenticated gh. Signed-off-by: ErenAri --- docs/production-release-process.md | 15 +++++++++------ scripts/production-readiness-report.sh | 14 ++++++++++++-- scripts/production-readiness-report_test.sh | 16 +++++++++++++--- 3 files changed, 34 insertions(+), 11 deletions(-) diff --git a/docs/production-release-process.md b/docs/production-release-process.md index 3852a77..2c924dc 100644 --- a/docs/production-release-process.md +++ b/docs/production-release-process.md @@ -194,15 +194,17 @@ aggregator binds every piece of evidence to the attested candidate `ghcr.io/kernel-guard/bpfcompat:rollback-drill-vX.Y.Z-rc.N`, `- Candidate digest:` is the candidate image digest, `- Final digest:` equals a distinct `- Known-good digest:`, and `- Completed:` is after - `generated_at`. A drill of an earlier candidate is refused. + `generated_at`. A drill of an earlier candidate is refused. With attestation + verification on, `- Workflow run:` must be a successful, dispatched + `rollback-drill.yml` run on this repository's `main` that started after + `generated_at`, and `- Completed:` must fall inside that run. - **Incident.** The note must contain `[bpfcompat-promotion-incident:v1]`, name `promote-release`, and link the deliberately rejected run, and only that run, as `https://github.com/Kernel-Guard/bpfcompat/actions/runs/`. With - attestation verification on, that run must be a completed `promote-release.yml` - run that did not succeed, and the rollback evidence's workflow run must be a - successful `rollback-drill.yml` run on `main`; both are looked up on GitHub. - Written policy does not require the incident drill to target the current + attestation verification on, that run must be a completed, dispatched + `promote-release.yml` run on this repository that did not succeed. Written + policy does not require the incident drill to target the current candidate, so an earlier candidate's fail-closed drill is accepted; the output records `incident.promotion_run_id` and `incident.candidate_bound`. @@ -226,7 +228,8 @@ The command writes both Markdown and JSON. Add the reviewed outputs to request. The stable release workflow validates, checksums, attests, and publishes those files; a stable tag fails if they are absent or not `ready`. The online candidate check requires an authenticated GitHub CLI version that -provides `gh attestation`; an older CLI fails closed. +provides `gh attestation`, and the drill and incident runs are read through the +GitHub API; an older or unauthenticated CLI, or no network, fails closed. Do not use `BPFCOMPAT_SKIP_READINESS_ATTESTATION=1` outside the regression test; production evidence must verify the candidate attestation online. diff --git a/scripts/production-readiness-report.sh b/scripts/production-readiness-report.sh index 0ffd9eb..6b47f08 100755 --- a/scripts/production-readiness-report.sh +++ b/scripts/production-readiness-report.sh @@ -555,13 +555,23 @@ if [[ "${BPFCOMPAT_SKIP_READINESS_ATTESTATION:-0}" != "1" ]]; then rollback_run="$(gh api "repos/Kernel-Guard/bpfcompat/actions/runs/${rollback_run_id}" 2>/dev/null)" || fail "rollback drill run ${rollback_run_id} cannot be read from GitHub" jq -e '.path == ".github/workflows/rollback-drill.yml" and .head_branch == "main" and + .event == "workflow_dispatch" and .head_repository.full_name == "Kernel-Guard/bpfcompat" and .status == "completed" and .conclusion == "success"' <<<"$rollback_run" >/dev/null || - fail "rollback drill run ${rollback_run_id} is not a successful rollback-drill.yml run on main" + fail "rollback drill run ${rollback_run_id} is not a successful, dispatched rollback-drill.yml run on this repository's main" + # The run itself started after the candidate was built, and the note's + # Completed time falls inside it. + rollback_run_started="$(iso_epoch "$(jq -r '.run_started_at // ""' <<<"$rollback_run")" "rollback drill run start")" + rollback_run_updated="$(iso_epoch "$(jq -r '.updated_at // ""' <<<"$rollback_run")" "rollback drill run end")" + [[ "$rollback_run_started" -gt "$candidate_generated_epoch" ]] || + fail "rollback drill run ${rollback_run_id} started before release candidate ${candidate_tag} was built" + [[ "$rollback_completed_epoch" -ge "$rollback_run_started" && "$rollback_completed_epoch" -le "$rollback_run_updated" ]] || + fail "rollback drill note's Completed time is outside run ${rollback_run_id}" incident_run="$(gh api "repos/Kernel-Guard/bpfcompat/actions/runs/${incident_run_id}" 2>/dev/null)" || fail "incident run ${incident_run_id} cannot be read from GitHub" jq -e '.path == ".github/workflows/promote-release.yml" and + .event == "workflow_dispatch" and .head_repository.full_name == "Kernel-Guard/bpfcompat" and .status == "completed" and .conclusion != "success"' <<<"$incident_run" >/dev/null || - fail "incident run ${incident_run_id} is not a rejected promote-release.yml run" + fail "incident run ${incident_run_id} is not a rejected, dispatched promote-release.yml run on this repository" fi incident_candidate_bound=false if grep -qE "(^|[^[:alnum:].-])${candidate_tag//./\\.}([^[:alnum:]]|$)" "$incident"; then diff --git a/scripts/production-readiness-report_test.sh b/scripts/production-readiness-report_test.sh index 8b80bf3..013e698 100755 --- a/scripts/production-readiness-report_test.sh +++ b/scripts/production-readiness-report_test.sh @@ -633,10 +633,15 @@ if [[ "$1" == api && "$2" =~ ^repos/Kernel-Guard/bpfcompat/actions/runs/([0-9]+) case "${BASH_REMATCH[1]}" in 5000) jq -n --arg p "${STUB_ROLLBACK_PATH:-.github/workflows/rollback-drill.yml}" \ --arg b "${STUB_ROLLBACK_BRANCH:-main}" --arg c "${STUB_ROLLBACK_CONCLUSION:-success}" \ - '{path: $p, head_branch: $b, status: "completed", conclusion: $c}' ;; + --arg e "${STUB_ROLLBACK_EVENT:-workflow_dispatch}" --arg r "${STUB_ROLLBACK_REPO:-Kernel-Guard/bpfcompat}" \ + --arg s "${STUB_ROLLBACK_STARTED:-2026-07-30T15:55:00Z}" --arg u "${STUB_ROLLBACK_UPDATED:-2026-07-30T16:01:00Z}" \ + '{path: $p, head_branch: $b, event: $e, head_repository: {full_name: $r}, + run_started_at: $s, updated_at: $u, status: "completed", conclusion: $c}' ;; 6000) jq -n --arg p "${STUB_INCIDENT_PATH:-.github/workflows/promote-release.yml}" \ - --arg c "${STUB_INCIDENT_CONCLUSION:-failure}" \ - '{path: $p, head_branch: "v0.4.0-rc.1", status: "completed", conclusion: $c}' ;; + --arg c "${STUB_INCIDENT_CONCLUSION:-failure}" --arg e "${STUB_INCIDENT_EVENT:-workflow_dispatch}" \ + --arg r "${STUB_INCIDENT_REPO:-Kernel-Guard/bpfcompat}" \ + '{path: $p, head_branch: "v0.4.0-rc.1", event: $e, head_repository: {full_name: $r}, + status: "completed", conclusion: $c}' ;; *) echo "gh: Not Found (HTTP 404)" >&2; exit 1 ;; esac exit 0 @@ -685,6 +690,11 @@ for case in \ "STUB_ROLLBACK_PATH=.github/workflows/ci.yml|a rollback drill run that is another workflow" \ "STUB_ROLLBACK_CONCLUSION=failure|a rollback drill run that did not succeed" \ "STUB_ROLLBACK_BRANCH=feature|a rollback drill run not on main" \ + "STUB_ROLLBACK_REPO=someone/bpfcompat|a rollback drill run from a fork" \ + "STUB_ROLLBACK_EVENT=pull_request|a rollback drill run that was not dispatched" \ + "STUB_ROLLBACK_STARTED=2026-07-30T13:00:00Z|a rollback drill run that started before the candidate" \ + "STUB_ROLLBACK_UPDATED=2026-07-30T15:59:00Z|a rollback note whose completion is outside its run" \ + "STUB_INCIDENT_REPO=someone/bpfcompat|an incident linking a run from a fork" \ "STUB_INCIDENT_PATH=.github/workflows/ci.yml|an incident linking a run of another workflow" \ "STUB_INCIDENT_CONCLUSION=success|an incident linking a promotion that succeeded"; do if env "${case%%|*}" PATH="$tmp/bin:$PATH" STUB_LOG="$tmp/gh.log" "$script" "$tmp/input.json" "$tmp/bad.md" >/dev/null 2>&1; then From fdae550d4d24c920f19e7f6f3a26290bcdc184d1 Mon Sep 17 00:00:00 2001 From: ErenAri Date: Mon, 19 Oct 2026 21:20:47 +0300 Subject: [PATCH 5/5] readiness: look up the canary runs, their attempts and the release Written policy says a canary is a run of the temporary release-candidate-canary workflow, the timed ones from its dated schedule, and that its plan job admits T+24h only 20-30 and T+72h only 68-78 whole hours after publication. Readiness checked only fields the canary evidence reports about itself and made no run lookup. A ci.yml, branch, fork, dispatched, failed or nonexistent run was accepted, and a partial re-run of a failed in-window run, which keeps the plan job's earlier decision, could write evidence long after the window closed (F6). With attestation verification on, readiness now reads publication (T0) from the candidate release, which must be published, not a draft, and not before generated_at. For each canary it reads the run, its attempts and the chosen attempt's jobs: - the run is a successful release-candidate-canary.yml run of this repository from workflow_dispatch (manual) or schedule (timed, on main); - the evidence's completed_at falls inside a successful attempt, in which the "Assemble RC canary evidence" job itself ran and succeeded; - that attempt started at or after T0, and a timed attempt started before its window closed (T0+31h, T0+79h); a first attempt, which ran its own plan job, also completed after the window opened (T0+20h, T0+68h). The attempt, its start, the run's branch and T0 are recorded in the JSON and Markdown. No written rule names the manual dispatch's branch, so it is recorded, not enforced. A re-run started inside the window is accepted, as the windows are written; whether re-runs should count stays an open operator decision. The bypass mode is unchanged. The test's gh stub answers the new lookups in this same commit, because the test runs in ci.yml and in the release quality gates. The runbook no longer says the plan job alone bounds re-runs and describes the new checks. Signed-off-by: ErenAri --- docs/production-release-process.md | 33 ++- scripts/production-readiness-report.sh | 132 +++++++++- scripts/production-readiness-report_test.sh | 274 +++++++++++++++++++- 3 files changed, 425 insertions(+), 14 deletions(-) diff --git a/docs/production-release-process.md b/docs/production-release-process.md index 2c924dc..febc55f 100644 --- a/docs/production-release-process.md +++ b/docs/production-release-process.md @@ -67,8 +67,12 @@ digest, and the external Inspektor Gadget/KubeArmor/Falco workflow dispatched at the immutable RC tag. Run it once immediately after publication. Its two dated schedules run just after T+24 hours and T+72 hours and emit `[bpfcompat-rc-canary:v1]` evidence; any out-of-window invocation exits after -the planning job. Remove or retarget these temporary schedules after -graduation. +the planning job. A partial re-run ("Re-run failed jobs", or re-running one +job) is not a new invocation: it does not run the planning job again, keeps +the first attempt's window decision, and can write evidence after the window +has closed. The planning job alone therefore does not bound re-runs; the +readiness report checks each canary run's attempts (see **Canaries** below). +Remove or retarget these temporary schedules after graduation. Every job in `release-candidate-canary`, `external-consumer-canary`, and `consumer-canary` runs on the pinned `ubuntu-24.04` image, not the moving @@ -181,9 +185,25 @@ aggregator binds every piece of evidence to the attested candidate every canary must bind the same RC tag, commit, and image digest as the attested candidate. Completion times must be in milestone order and at least 0, 20, and 68 hours after `generated_at`. These are floors derived from the - plan job's 20-30h and 68-78h windows after publication: publication time is - not in the attested evidence, but it never precedes `generated_at`. The plan - job itself enforces the upper bounds. Each canary's components must be + plan job's 20-30h and 68-78h windows after publication (T0): publication time + is not in the attested evidence, but it never precedes `generated_at`. With + attestation verification on, T0 is the release's `published_at`; the release + must be published, not a draft, and not before `generated_at`. Each canary's + `workflow_run_id` must be a successful `release-candidate-canary.yml` run of + this repository from `workflow_dispatch` (manual) or `schedule` (timed, on + `main`). Its `completed_at` must fall inside a successful attempt of that + run in which the `Assemble RC canary evidence` job itself ran and succeeded, + and that attempt must have started at or after T0. A timed canary's attempt + must have started before its window closed (T0+31h or T0+79h), and a first + attempt, which ran its own plan job, must have completed after the window + opened (T0+20h or T0+68h). The output records each canary's + `workflow_run_attempt`, `attempt_started_at` and `head_branch`, and the + release's `published_at`; no written rule names the manual dispatch's + branch, so it is recorded, not enforced. A re-run started inside the window + is accepted, as the windows are written. Whether a re-run should count is an + open operator decision: counting first attempts only would tighten this, and + counting a re-run started after the window closed would relax the windows + and must be written here first. Each canary's components must be artifacts of that canary's own run (`*canary-component--/`), and the `.json` record of each of the five checks must be present, match its recorded SHA-256, pass, and name the candidate tag and commit; the @@ -228,7 +248,8 @@ The command writes both Markdown and JSON. Add the reviewed outputs to request. The stable release workflow validates, checksums, attests, and publishes those files; a stable tag fails if they are absent or not `ready`. The online candidate check requires an authenticated GitHub CLI version that -provides `gh attestation`, and the drill and incident runs are read through the +provides `gh attestation`, and the candidate release, the canary runs with +their attempts and jobs, and the drill and incident runs are read through the GitHub API; an older or unauthenticated CLI, or no network, fails closed. Do not use `BPFCOMPAT_SKIP_READINESS_ATTESTATION=1` outside the regression diff --git a/scripts/production-readiness-report.sh b/scripts/production-readiness-report.sh index 6b47f08..f832007 100755 --- a/scripts/production-readiness-report.sh +++ b/scripts/production-readiness-report.sh @@ -377,6 +377,21 @@ fi : >"$tmp/canaries.ndjson" : >"$tmp/canary_run_ids" previous_canary_epoch=0 +# The canary plan job measures its windows from publication (T0), which the +# canary designation records as CANDIDATE_PUBLISHED_AT. With attestation +# verification on, T0 is read from the release itself: it must be published, +# not a draft, and cannot precede the candidate evidence it publishes. +candidate_published_at="" +if [[ "${BPFCOMPAT_SKIP_READINESS_ATTESTATION:-0}" != "1" ]]; then + candidate_release="$(gh api "repos/Kernel-Guard/bpfcompat/releases/tags/${candidate_tag}" 2>/dev/null)" || + fail "release ${candidate_tag} cannot be read from GitHub" + jq -e '.draft == false' <<<"$candidate_release" >/dev/null || + fail "release ${candidate_tag} is a draft, not a published release" + candidate_published_at="$(jq -r '.published_at // ""' <<<"$candidate_release")" + candidate_published_epoch="$(iso_epoch "$candidate_published_at" "release ${candidate_tag} published_at")" + [[ "$candidate_published_epoch" -ge "$candidate_generated_epoch" ]] || + fail "release ${candidate_tag} was published at ${candidate_published_at}, before its candidate evidence was generated" +fi while IFS= read -r canary; do milestone="$(jq -r '.milestone' <<<"$canary")" evidence_rel="$(jq -r '.evidence' <<<"$canary")" @@ -426,7 +441,8 @@ while IFS= read -r canary; do # hours, and T+72h only 68-78 whole hours, after CANDIDATE_PUBLISHED_AT. # Publication time is not part of the attested candidate evidence, but it # cannot precede generated_at, so generated_at + 0/20/68h is a floor every - # authentic observation meets. Upper bounds are left to the plan job. + # authentic observation meets. The plan job does not bound a partial re-run, + # which keeps its earlier decision; the run lookup below does. case "$milestone" in manual) floor_hours=0 ;; t-plus-24h) floor_hours=20 ;; @@ -439,6 +455,92 @@ while IFS= read -r canary; do fail "RC canaries must complete in manual, T+24h, T+72h order" previous_canary_epoch="$canary_completed_epoch" + # Written policy: a canary is a run of the temporary release-candidate-canary + # workflow, the timed ones from its dated schedule, and its plan job admits + # T+24h only 20-30 and T+72h only 68-78 whole hours after T0. A partial + # re-run ("Re-run failed jobs") does not run the plan job again: it keeps + # the earlier decision and writes new evidence later. With attestation + # verification on, the run is looked up. It must be a successful run of that + # workflow on this repository from the expected event, and the evidence must + # come from the successful attempt whose span contains completed_at, with + # the evidence job itself run in that attempt. That attempt started after + # T0 and, for a timed milestone, before its window closed (T0+31h, T0+79h); + # a first attempt ran its own plan job, so its evidence also completed after + # the window opened (T0+20h, T0+68h). Whether a re-run started inside the + # window should count is an open operator decision; this applies the windows + # as written. The attempt is recorded. + canary_attempt=null + canary_attempt_started_at="" + canary_head_branch="" + if [[ "${BPFCOMPAT_SKIP_READINESS_ATTESTATION:-0}" != "1" ]]; then + canary_run="$(gh api "repos/Kernel-Guard/bpfcompat/actions/runs/${canary_run_id}" 2>/dev/null)" || + fail "RC canary ${milestone} run ${canary_run_id} cannot be read from GitHub" + jq -e --arg event "$expected_event" ' + .path == ".github/workflows/release-candidate-canary.yml" and + .event == $event and + .head_repository.full_name == "Kernel-Guard/bpfcompat" and + .status == "completed" and .conclusion == "success" + ' <<<"$canary_run" >/dev/null || + fail "RC canary ${milestone} run ${canary_run_id} is not a successful ${expected_event} run of release-candidate-canary.yml on this repository" + canary_head_branch="$(jq -r '.head_branch // ""' <<<"$canary_run")" + # Scheduled runs execute on the default branch. Written policy names no + # branch for the manual dispatch, so its branch is recorded, not enforced. + if [[ "$milestone" != manual ]]; then + [[ "$canary_head_branch" == main ]] || + fail "RC canary ${milestone} run ${canary_run_id} is not on main" + fi + canary_last_attempt="$(jq -r '.run_attempt' <<<"$canary_run")" + [[ "$canary_last_attempt" =~ ^[1-9][0-9]*$ ]] || + fail "RC canary ${milestone} run ${canary_run_id} has no valid run_attempt" + for ((attempt = 1; attempt <= canary_last_attempt; attempt++)); do + attempt_run="$(gh api "repos/Kernel-Guard/bpfcompat/actions/runs/${canary_run_id}/attempts/${attempt}" 2>/dev/null)" || + fail "RC canary ${milestone} run ${canary_run_id} attempt ${attempt} cannot be read from GitHub" + jq -e '.conclusion == "success"' <<<"$attempt_run" >/dev/null || continue + attempt_started_at="$(jq -r '.run_started_at // ""' <<<"$attempt_run")" + attempt_started_epoch="$(iso_epoch "$attempt_started_at" "RC canary run ${canary_run_id} attempt ${attempt} start")" + attempt_updated_epoch="$(iso_epoch "$(jq -r '.updated_at // ""' <<<"$attempt_run")" "RC canary run ${canary_run_id} attempt ${attempt} end")" + if [[ "$canary_completed_epoch" -ge "$attempt_started_epoch" && "$canary_completed_epoch" -le "$attempt_updated_epoch" ]]; then + canary_attempt="$attempt" + canary_attempt_started_at="$attempt_started_at" + canary_attempt_started_epoch="$attempt_started_epoch" + break + fi + done + [[ "$canary_attempt" != null ]] || + fail "RC canary ${milestone} completed_at ${canary_completed_at} is not inside a successful attempt of run ${canary_run_id}" + # An out-of-window or late "Re-run all jobs" attempt also concludes success, + # with every check and the evidence job skipped, and a re-run attempt lists + # carried-over jobs with their earlier start. The jobs API names a job by + # its display name, which release-candidate-canary.yml gives its evidence + # job; a renamed job fails closed. + canary_jobs="$(gh api "repos/Kernel-Guard/bpfcompat/actions/runs/${canary_run_id}/attempts/${canary_attempt}/jobs?per_page=100" 2>/dev/null)" || + fail "RC canary ${milestone} run ${canary_run_id} attempt ${canary_attempt} jobs cannot be read from GitHub" + evidence_job_started_at="$(jq -r ' + [(.jobs // [])[] | select(.name == "Assemble RC canary evidence")] | + if length == 1 and .[0].conclusion == "success" then .[0].started_at // "" else "" end + ' <<<"$canary_jobs")" + [[ -n "$evidence_job_started_at" ]] || + fail "RC canary ${milestone} run ${canary_run_id} attempt ${canary_attempt} has no successful \"Assemble RC canary evidence\" job" + evidence_job_started_epoch="$(iso_epoch "$evidence_job_started_at" "RC canary run ${canary_run_id} evidence job start")" + [[ "$evidence_job_started_epoch" -ge "$canary_attempt_started_epoch" ]] || + fail "RC canary ${milestone} run ${canary_run_id}: its evidence job was carried over into attempt ${canary_attempt}, not run in it" + [[ "$canary_attempt_started_epoch" -ge "$candidate_published_epoch" ]] || + fail "RC canary ${milestone} attempt ${canary_attempt} of run ${canary_run_id} started before release ${candidate_tag} was published" + case "$milestone" in + t-plus-24h) window_open=20 window_close=31 ;; + t-plus-72h) window_open=68 window_close=79 ;; + *) window_open="" window_close="" ;; + esac + if [[ -n "$window_close" ]]; then + [[ "$canary_attempt_started_epoch" -lt $((candidate_published_epoch + window_close * 3600)) ]] || + fail "RC canary ${milestone} attempt ${canary_attempt} of run ${canary_run_id} started ${window_close}h or more after release ${candidate_tag} was published" + if [[ "$canary_attempt" -eq 1 ]]; then + [[ "$canary_completed_epoch" -ge $((candidate_published_epoch + window_open * 3600)) ]] || + fail "RC canary ${milestone} completed less than ${window_open}h after release ${candidate_tag} was published" + fi + fi + fi + # Components: each is an artifact of this exact canary run (the workflow # downloads rc*-canary-component--), every check is # covered, and each check's own record -- shipped in the same artifact -- @@ -475,17 +577,29 @@ while IFS= read -r canary; do done printf '%s\n' "$canary_run_id" >>"$tmp/canary_run_ids" - printf "| %s | \`%s\` | %s |\n" \ - "$milestone" "$canary_run_id" "$canary_completed_at" >>"$tmp/canary_rows" + if [[ "$canary_attempt" == null ]]; then + printf "| %s | \`%s\` | not looked up | not looked up | not looked up | %s |\n" \ + "$milestone" "$canary_run_id" "$canary_completed_at" >>"$tmp/canary_rows" + else + printf "| %s | \`%s\` | %s | %s | %s | %s |\n" \ + "$milestone" "$canary_run_id" "$canary_attempt" "$canary_attempt_started_at" \ + "$canary_head_branch" "$canary_completed_at" >>"$tmp/canary_rows" + fi jq -n \ --arg milestone "$milestone" \ --argjson workflow_run_id "$canary_run_id" \ + --argjson attempt "$canary_attempt" \ + --arg attempt_started_at "$canary_attempt_started_at" \ + --arg head_branch "$canary_head_branch" \ --arg completed_at "$canary_completed_at" \ --arg evidence_sha256 "$canary_sha" \ --argjson minutes "$(((canary_completed_epoch - candidate_generated_epoch) / 60))" \ '{ milestone: $milestone, workflow_run_id: $workflow_run_id, + workflow_run_attempt: $attempt, + attempt_started_at: (if $attempt_started_at == "" then null else $attempt_started_at end), + head_branch: (if $head_branch == "" then null else $head_branch end), completed_at: $completed_at, minutes_after_candidate_generated: $minutes, components_verified: true, @@ -627,8 +741,12 @@ mkdir -p "$(dirname "$output_json")" echo echo "## Release-Candidate Canaries" echo - echo "| Milestone | Workflow run | Completed (UTC) |" - echo "|---|---|---|" + if [[ -n "$candidate_published_at" ]]; then + echo "Windows are measured from the release's publication at ${candidate_published_at}; each run, attempt and evidence job was read from GitHub." + echo + fi + echo "| Milestone | Workflow run | Attempt | Attempt started (UTC) | Branch | Completed (UTC) |" + echo "|---|---|---|---|---|---|" cat "$tmp/canary_rows" echo echo "## Required External Evidence" @@ -682,6 +800,7 @@ jq -n \ --arg candidate_commit "$candidate_commit" \ --arg candidate_image "$candidate_image" \ --arg candidate_sha "$candidate_sha" \ + --arg candidate_published_at "$candidate_published_at" \ --arg rollback_sha "$rollback_sha" \ --arg incident_sha "$incident_sha" \ --arg operator "$operator" \ @@ -735,7 +854,8 @@ jq -n \ tag: $candidate_tag, commit_sha: $candidate_commit, image: $candidate_image, - evidence_sha256: $candidate_sha + evidence_sha256: $candidate_sha, + published_at: (if $candidate_published_at == "" then null else $candidate_published_at end) }, canaries: $canaries, rollback: { diff --git a/scripts/production-readiness-report_test.sh b/scripts/production-readiness-report_test.sh index 013e698..1adab37 100755 --- a/scripts/production-readiness-report_test.sh +++ b/scripts/production-readiness-report_test.sh @@ -626,8 +626,18 @@ mkdir -p "$tmp/bin" cat >"$tmp/bin/gh" <<'STUB' #!/usr/bin/env bash [[ "$1" == attestation && "$2" == --help ]] && exit 0 +# serve : answer a read from $STUB_API_DIR/.json (the canary runs, their attempts and jobs, and the candidate +# release); a missing file is a 404. +serve() { + local file="${STUB_API_DIR:?}/${1#repos/Kernel-Guard/bpfcompat/}" + file="${file%%\?*}.json" + [[ -f "$file" ]] || { echo "gh: Not Found (HTTP 404)" >&2; exit 1; } + cat "$file" +} # Workflow run lookups: run 5000 is the rollback drill and 6000 the rejected -# promotion; STUB_* variables override their fields, any other run is a 404. +# promotion; STUB_* variables override their fields. Any other run is served +# from $STUB_API_DIR, else a 404. if [[ "$1" == api && "$2" =~ ^repos/Kernel-Guard/bpfcompat/actions/runs/([0-9]+)$ ]]; then printf '%s\n' "$*" >>"$STUB_LOG" case "${BASH_REMATCH[1]}" in @@ -642,10 +652,15 @@ if [[ "$1" == api && "$2" =~ ^repos/Kernel-Guard/bpfcompat/actions/runs/([0-9]+) --arg r "${STUB_INCIDENT_REPO:-Kernel-Guard/bpfcompat}" \ '{path: $p, head_branch: "v0.4.0-rc.1", event: $e, head_repository: {full_name: $r}, status: "completed", conclusion: $c}' ;; - *) echo "gh: Not Found (HTTP 404)" >&2; exit 1 ;; + *) serve "$2" ;; esac exit 0 fi +if [[ "$1" == api && "$2" == repos/Kernel-Guard/bpfcompat/* ]]; then + printf '%s\n' "$*" >>"$STUB_LOG" + serve "$2" + exit 0 +fi [[ "$1 $2" == "attestation verify" ]] || { echo "unexpected gh $*" >&2; exit 99; } printf '%s\n' "$*" >>"$STUB_LOG" [[ -n "${STUB_FAIL:-}" && "$3" == *"$STUB_FAIL"* ]] && exit 1 @@ -660,6 +675,51 @@ if [[ "$*" == *latest-kernel-compatibility.yml* ]]; then fi STUB chmod +x "$tmp/bin/gh" +# GitHub API answers for the canary lookups, one file per endpoint. The +# candidate release was published (T0) at 14:50Z, 50 minutes after +# generated_at. Canary run 300 has one successful attempt, from the +# workflow_dispatch (manual) or schedule event, whose evidence job ran in it +# just before the canary's completed_at. +api="$tmp/api" +# api_put [jq args...]: write one answer. +api_put() { + local dir="$1" path="$2" program="$3" + shift 3 + mkdir -p "$(dirname "$dir/$path")" + jq -n "$@" "$program" >"$dir/$path.json" +} +# canary_attempt +# : one attempt of +# a canary run and its jobs. The run itself shows its latest attempt, so the +# last attempt written also becomes the run. +# shellcheck disable=SC2016 # the single-quoted arguments are jq programs +canary_attempt() { + api_put "$1" "actions/runs/$2/attempts/$3" \ + '{id: $run, path: ".github/workflows/release-candidate-canary.yml", head_branch: "main", + event: $e, head_repository: {full_name: "Kernel-Guard/bpfcompat"}, run_attempt: $n, + status: "completed", conclusion: $c, run_started_at: $s, updated_at: $u}' \ + --argjson run "$2" --argjson n "$3" --arg e "$4" --arg c "$5" --arg s "$6" --arg u "$7" + api_put "$1" "actions/runs/$2/attempts/$3/jobs" \ + '{total_count: 2, jobs: [ + {name: "Select RC observation window", run_attempt: $n, status: "completed", + conclusion: "success", started_at: $s, completed_at: $s}, + {name: "Assemble RC canary evidence", run_attempt: $n, status: "completed", + conclusion: $ec, started_at: $es, completed_at: $u}]}' \ + --argjson n "$3" --arg s "$6" --arg u "$7" --arg ec "$8" --arg es "$9" + cp "$1/actions/runs/$2/attempts/$3.json" "$1/actions/runs/$2.json" +} +api_put "$api" releases/tags/v0.4.0-rc.1 \ + '{tag_name: "v0.4.0-rc.1", draft: false, prerelease: true, published_at: "2026-07-30T14:50:00Z"}' +for index in 0 1 2; do + day=$((index * (index + 1) / 2)) + event=schedule + [[ "$index" == 0 ]] && event=workflow_dispatch + canary_attempt "$api" "$((3000 + index))" 1 "$event" success \ + "$(date -u -d "2026-07-30 14:59:00 UTC +${day} days" +%Y-%m-%dT%H:%M:%SZ)" \ + "$(date -u -d "2026-07-30 15:06:00 UTC +${day} days" +%Y-%m-%dT%H:%M:%SZ)" success \ + "$(date -u -d "2026-07-30 15:05:20 UTC +${day} days" +%Y-%m-%dT%H:%M:%SZ)" +done +export STUB_API_DIR="$api" : >"$tmp/gh.log" PATH="$tmp/bin:$PATH" STUB_LOG="$tmp/gh.log" "$script" "$tmp/input.json" "$tmp/attested.md" >/dev/null grep -Fq -- "--source-digest $(printf 'c%.0s' {1..40}) --source-ref refs/tags/v0.4.0-rc.1" "$tmp/gh.log" || { @@ -711,6 +771,216 @@ fi echo "[production-readiness-test] refused: an incident linking a run that does not exist" mv "$incident_note.keep" "$incident_note" +# --- Canary runs: identity, attempt and observation windows -------------------- +# Written policy: a canary is a run of the temporary release-candidate-canary +# workflow, the timed ones from its dated schedule, and its plan job admits +# T+24h only 20-30 and T+72h only 68-78 whole hours after publication (T0). A +# partial re-run keeps the plan job's earlier decision and writes new evidence +# later, so with attestation verification on every canary run is looked up. +for path in releases/tags/v0.4.0-rc.1 \ + actions/runs/3000 actions/runs/3000/attempts/1 "actions/runs/3000/attempts/1/jobs?per_page=100" \ + actions/runs/3001 actions/runs/3001/attempts/1 "actions/runs/3001/attempts/1/jobs?per_page=100" \ + actions/runs/3002 actions/runs/3002/attempts/1 "actions/runs/3002/attempts/1/jobs?per_page=100"; do + grep -Fxq "api repos/Kernel-Guard/bpfcompat/$path" "$tmp/gh.log" || { + echo "[production-readiness-test] canary lookup $path was not made" >&2; exit 1; } +done +jq -e ' + [.canaries[].workflow_run_attempt] == [1, 1, 1] and + [.canaries[].attempt_started_at] == ["2026-07-30T14:59:00Z", "2026-07-31T14:59:00Z", "2026-08-02T14:59:00Z"] and + [.canaries[].head_branch] == ["main", "main", "main"] and + .candidate.published_at == "2026-07-30T14:50:00Z" +' "$tmp/attested.json" >/dev/null || { + echo "[production-readiness-test] attested readiness output does not record the canary attempts" >&2; exit 1; } +grep -Fq "| t-plus-24h | \`3001\` | 1 | 2026-07-31T14:59:00Z | main | 2026-07-31T15:05:34Z |" "$tmp/attested.md" || { + echo "[production-readiness-test] attested readiness Markdown does not show the canary attempts" >&2; exit 1; } +jq -e '([.canaries[] | .workflow_run_attempt == null and .attempt_started_at == null and .head_branch == null] | all) and + .candidate.published_at == null' "$tmp/readiness.json" >/dev/null || { + echo "[production-readiness-test] bypass readiness output claims canary lookups it did not make" >&2; exit 1; } +echo "[production-readiness-test] attested: the release and every canary run, attempt and job list were looked up and recorded" +# The jobs API names a job only by its display name. While the temporary +# canary workflow exists, its evidence job must carry the name readiness +# looks for. +rcc="${ROOT_DIR}/.github/workflows/release-candidate-canary.yml" +grep -Fq 'select(.name == "Assemble RC canary evidence")' "$script" || { + echo "[production-readiness-test] readiness no longer looks up the evidence job by name" >&2; exit 1; } +if [[ -f "$rcc" ]]; then + awk '/^ evidence:$/ {getline; found = ($0 == " name: Assemble RC canary evidence")} END {exit !found}' "$rcc" || { + echo "[production-readiness-test] release-candidate-canary.yml does not name its evidence job \"Assemble RC canary evidence\"" >&2; exit 1; } + echo "[production-readiness-test] release-candidate-canary.yml names its evidence job as readiness expects" +fi + +c="$tmp/api-case" +# api_case: a fresh copy of the default API answers for one case. +api_case() { rm -rf "$c"; cp -R "$api" "$c"; } +# api_edit : change one answer of the current case. +api_edit() { jq "$2" "$c/$1.json" >"$c/$1.json.new" && mv "$c/$1.json.new" "$c/$1.json"; } +# canary_completed : a manifest pinning +# canary evidence that records another completed_at; canary_restore +# puts the original evidence back. +canary_completed() { + local file="$tmp/canary-$1/release-candidate-canary.json" + cp "$file" "$file.orig" + jq --arg t "$2" '.completed_at = $t' "$file.orig" >"$file" + rehash "$file" ".canaries[$1].evidence_sha256" "$3" +} +canary_restore() { mv "$tmp/canary-$1/release-candidate-canary.json.orig" "$tmp/canary-$1/release-candidate-canary.json"; } +# canary_refused