diff --git a/.github/workflows/actions.lock b/.github/workflows/actions.lock index dc919a66..c099b647 100644 --- a/.github/workflows/actions.lock +++ b/.github/workflows/actions.lock @@ -124,6 +124,8 @@ workflows: - 'gitleaks/gitleaks-action@v3.0.0' - 'taiki-e/install-action@v2.90.3' - 'trufflesecurity/trufflehog@v3.99.8' + '.github/workflows/sq-map-source-check.yml': + - 'actions/checkout@v7.0.1' '.github/workflows/standards-integration.yml': [] '.github/workflows/tests.yml': - 'actions/cache@v6.1.0' diff --git a/.github/workflows/sq-map-source-check.yml b/.github/workflows/sq-map-source-check.yml new file mode 100644 index 00000000..ad321a5f --- /dev/null +++ b/.github/workflows/sq-map-source-check.yml @@ -0,0 +1,98 @@ +# SPDX-License-Identifier: MPL-2.0 +# This workflow is managed by gh actions-lock. +# +# SQ map source check. +# +# Every hypatia/ci_failure_triage rule, and each of the two ADR 0004 +# workflow_audit ids (actions_lock_stale, mirror_without_visibility_guard), +# needs a row in the Hypatia to squabbler rule map, proven/registry/sq-map.tsv +# in hyperpolymath/proven-cicd-and-workflows. This job fails a change here that +# adds such a rule without that row, and names the row, so the author adds it in +# a proven-cicd PR first and then re-runs this job. +# +# What is checked: the registry the rule sources emit now, from +# `scripts/gen-rules-praxis.sh --stdout`, not the committed +# hypatia_rules_praxis.deed. A rule added without regenerating the deed is +# therefore still seen. Whether the committed deed is stale is quality.yml's +# check ("Rule registry (hypatia_rules_praxis.deed) matches the code and lints"). +# +# proven-cicd is read at main, not at a pin: the question is whether the rule is +# mapped now. The verdict is proven-cicd's own tests/registry-sq-map.sh. A failed +# checkout, generator or test is NOT RUN and fails the job; it is never a pass. +# +# Not a required check: making it one is a ruleset change for the owner. + +name: SQ map source check + +on: + pull_request: + paths: + - hypatia_rules_praxis.deed + - scripts/gen-rules-praxis.sh + - scripts/check-sq-map.sh + - tests/check-sq-map-test.sh + - lib/rules/** + - lib/pattern_registry.ex + - lib/hypatia/cli.ex + - lib/hypatia/scanner_suppression.ex + - lib/scorecard_ingestor.ex + - .github/workflows/sq-map-source-check.yml + push: + branches: [main] + paths: + - hypatia_rules_praxis.deed + - scripts/gen-rules-praxis.sh + - scripts/check-sq-map.sh + - tests/check-sq-map-test.sh + - lib/rules/** + - lib/pattern_registry.ex + - lib/hypatia/cli.ex + - lib/hypatia/scanner_suppression.ex + - lib/scorecard_ingestor.ex + - .github/workflows/sq-map-source-check.yml + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + sq-map: + name: Triage rules have a row in proven-cicd sq-map.tsv + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - name: Checkout + uses: actions/checkout@v7.0.1 + with: + persist-credentials: false + + # Public repository: the default token reads it; nothing is written. + - name: Checkout proven-cicd-and-workflows at main (the map and its test) + id: proven + uses: actions/checkout@v7.0.1 + with: + repository: hyperpolymath/proven-cicd-and-workflows + ref: main + path: proven-cicd + sparse-checkout: | + tests/registry-sq-map.sh + proven/registry/sq-map.tsv + sparse-checkout-cone-mode: false + persist-credentials: false + + - name: Say NOT RUN when proven-cicd could not be checked out + if: failure() && steps.proven.outcome == 'failure' + run: | + echo "::error title=SQ map check NOT RUN::proven-cicd-and-workflows could not be checked out at main. This is not a pass: re-run the job." + printf '### SQ map check: NOT RUN\n\nproven-cicd-and-workflows could not be checked out at main. A check that did not run is not a pass.\n' >> "$GITHUB_STEP_SUMMARY" + exit 1 + + - name: Controls for the check + run: bash tests/check-sq-map-test.sh proven-cicd + + - name: Every triage id the rule sources emit has a row + run: bash scripts/check-sq-map.sh proven-cicd diff --git a/scripts/check-sq-map.sh b/scripts/check-sq-map.sh new file mode 100755 index 00000000..7e4b0056 --- /dev/null +++ b/scripts/check-sq-map.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +# check-sq-map.sh: does every hypatia/ci_failure_triage id this tree emits, and +# each of the two ADR 0004 workflow_audit ids, have a row in the Hypatia to +# squabbler rule map, proven/registry/sq-map.tsv in +# hyperpolymath/proven-cicd-and-workflows? +# +# scripts/check-sq-map.sh PROVEN_CICD_DIR [DEED] +# +# PROVEN_CICD_DIR is a checkout of proven-cicd-and-workflows (CI checks out its +# main branch, because the question is whether the rule is mapped now). +# +# DEED defaults to a fresh run of `scripts/gen-rules-praxis.sh --stdout`: the +# registry this tree's rule sources emit now, so a rule added without +# regenerating hypatia_rules_praxis.deed is still seen here. Whether the +# committed deed is stale is a separate check (quality.yml, "Rule registry +# (hypatia_rules_praxis.deed) matches the code and lints"). Pass a DEED to check +# that file instead. +# +# The verdict is proven-cicd's own tests/registry-sq-map.sh, run with +# HYPATIA_PRAXIS=DEED. This script adds only the cure: the exact row to add for +# each id the test names, and where to add it. +# +# Exit: 0 every required id has a row; 1 the map lacks a row or a row is wrong; +# 2 NOT RUN (the checkout, the generator or the test is missing or failed). +# CI fails on 2 as well as on 1: a check that did not run is never a pass. +set -euo pipefail +cd "$(dirname "$0")/.." + +summary=${GITHUB_STEP_SUMMARY:-/dev/null} +notrun() { + echo "::error title=SQ map check NOT RUN::$*" + printf '### SQ map check: NOT RUN\n\n%s\n\nA check that did not run is not a pass.\n' "$*" >>"$summary" + exit 2 +} + +proven=${1:-} +[[ -n $proven ]] || notrun "usage: scripts/check-sq-map.sh PROVEN_CICD_DIR [DEED]" +test=$proven/tests/registry-sq-map.sh +map=$proven/proven/registry/sq-map.tsv +[[ -f $test && -f $map ]] || + notrun "$proven has no tests/registry-sq-map.sh or proven/registry/sq-map.tsv (a failed or partial checkout of proven-cicd-and-workflows)" + +work=$(mktemp -d) +trap 'rm -rf -- "$work"' EXIT +if [[ -n ${2:-} ]]; then + deed=$2 + [[ -f $deed ]] || notrun "$deed missing" + what="$deed" +else + deed=$work/hypatia_rules_praxis.deed + bash scripts/gen-rules-praxis.sh --stdout >"$deed" || notrun "scripts/gen-rules-praxis.sh --stdout failed" + [[ -s $deed ]] || notrun "scripts/gen-rules-praxis.sh --stdout printed nothing" + what="the registry the rule sources emit now (scripts/gen-rules-praxis.sh --stdout)" +fi +echo "checking $what against $map" + +# SQ_MAP and SQUABBLER_RULES are the test's other inputs: unset, so it reads +# the checkout's own map and judges only the Hypatia direction. +rc=0 +env -u SQ_MAP -u SQUABBLER_RULES HYPATIA_PRAXIS="$deed" bash "$test" >"$work/out" 2>&1 || rc=$? +cat "$work/out" +case $rc in + 0) printf '### SQ map check: every required id has a row\n\nChecked %s.\n' "$what" >>"$summary"; exit 0 ;; + 1) grep -q '^FAIL ' "$work/out" || + notrun "proven-cicd tests/registry-sq-map.sh exited 1 without naming a broken rule (it could not judge)" ;; + *) notrun "proven-cicd tests/registry-sq-map.sh exited $rc" ;; +esac + +# The test failed. Name the cure for each line it printed. +note="in the praxis registry, not in ADR0004 mapping table; SQ rule is the squabbler owner's decision" +missing=$(sed -n 's/^FAIL triage ids in the registry with no row[^:]*: //p' "$work/out" | tr ' ' '\n' | grep . || true) +{ + echo "### SQ map check: the map in proven-cicd-and-workflows needs a change" + echo + echo "Add the row(s) below to \`proven/registry/sq-map.tsv\` in a pull request on" + echo "hyperpolymath/proven-cicd-and-workflows first (keep the rows in id order and" + echo "the counts in \`proven/registry/README.adoc\` true). Once it is merged, re-run" + echo "this job here: it reads proven-cicd at main." + echo +} >"$work/cure" +if [[ -n $missing ]]; then + echo "Rows to add (tab-separated: a5_id, sq_id, status, praxis, note):" >>"$work/cure" + echo '```' >>"$work/cure" + while IFS= read -r id; do + if [[ $id == hypatia/ci_failure_triage/* ]]; then + printf '%s\tunassigned\tunmapped\tpresent\t%s\n' "$id" "$note" >>"$work/cure" + else + # The ADR 0004 workflow_audit ids are mapped rows; one is missing only + # when someone deleted it. Its SQ id comes from ADR 0004, not from here. + printf '%s\t\tmapped\tpresent\tADR0004 mapping table\n' "$id" >>"$work/cure" + fi + done <<<"$missing" + echo '```' >>"$work/cure" + echo >>"$work/cure" + echo "\`unmapped\` with \`unassigned\` is right until the squabbler owner gives the id an SQ rule." >>"$work/cure" +fi +other=$(grep '^FAIL ' "$work/out" | grep -v '^FAIL triage ids in the registry with no row' || true) +if [[ -n $other ]]; then + { + echo + echo "Other rows the test refused (fix or drop each in the same proven-cicd PR; a" + echo "praxis mismatch means a rule was added or removed here):" + echo '```' + echo "$other" + echo '```' + } >>"$work/cure" +fi +cat "$work/cure" +cat "$work/cure" >>"$summary" +while IFS= read -r id; do + [[ -n $id ]] && echo "::error title=No row in proven-cicd sq-map.tsv::$id has no row in proven/registry/sq-map.tsv; add it in proven-cicd-and-workflows first, then re-run" +done <<<"$missing" +exit 1 diff --git a/tests/check-sq-map-test.sh b/tests/check-sq-map-test.sh new file mode 100755 index 00000000..0772361a --- /dev/null +++ b/tests/check-sq-map-test.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +# Controls for scripts/check-sq-map.sh, against a checkout of +# hyperpolymath/proven-cicd-and-workflows (its test and map are copied, so the +# controls do not depend on what main's map holds today): +# * a registry whose ids all have rows passes (0); +# * a new ci_failure_triage id fails (1) and the cure names its exact row; +# * a deleted ADR 0004 workflow_audit row fails (1); +# * a missing or partial checkout, a missing deed, and a test that cannot +# judge are NOT RUN (2), never a pass; +# * with no deed the generator runs, and its registry is judged (0 or 1). +# +# tests/check-sq-map-test.sh PROVEN_CICD_DIR +set -euo pipefail +cd "$(dirname "$0")/.." +src=${1:?usage: tests/check-sq-map-test.sh PROVEN_CICD_DIR} +[[ -f $src/tests/registry-sq-map.sh && -f $src/proven/registry/sq-map.tsv ]] || + { echo "check-sq-map-test: $src is not a proven-cicd checkout; NOT RUN" >&2; exit 2; } +work=$(mktemp -d) +trap 'rm -rf -- "$work"' EXIT +pass=0 fail=0 +ok() { pass=$((pass + 1)); echo " ok $1"; } +bad() { fail=$((fail + 1)); echo " FAIL $1" >&2; } +fresh() { # -> $work/p, a copy of the test and the map + rm -rf -- "$work/p"; mkdir -p "$work/p/tests" "$work/p/proven/registry" + cp "$src/tests/registry-sq-map.sh" "$work/p/tests/" + cp "$src/proven/registry/sq-map.tsv" "$work/p/proven/registry/" +} +expect() { # NAME WANT ARGS... + local name=$1 want=$2 rc=0; shift 2 + env -u GITHUB_STEP_SUMMARY bash scripts/check-sq-map.sh "$@" >"$work/out" 2>&1 || rc=$? + if [[ $rc == "$want" ]]; then ok "$name"; else bad "$name: wanted $want, got $rc"; sed 's/^/ /' "$work/out" | tail -12 >&2; fi +} + +fresh +# A registry of exactly the map's own ids: every required id has a row. +grep -oE 'hypatia/[a-z0-9_]+/[A-Za-z0-9_]+' "$work/p/proven/registry/sq-map.tsv" | sort -u >"$work/own.deed" +expect 'a registry whose ids all have rows passes' 0 "$work/p" "$work/own.deed" + +{ cat "$work/own.deed"; echo hypatia/ci_failure_triage/brand_new_rule; } >"$work/new.deed" +expect 'a new ci_failure_triage id with no row fails' 1 "$work/p" "$work/new.deed" +row=$(printf 'hypatia/ci_failure_triage/brand_new_rule\tunassigned\tunmapped\tpresent\t') +grep -qF -- "$row" "$work/out" && ok 'the cure names the exact unmapped row' || bad 'the cure does not name the row' +grep -q 'proven-cicd-and-workflows first' "$work/out" && ok 'the cure says where to add it' || bad 'the cure does not say where' +# The printed row, appended to the map, makes the same registry pass. +grep -F -- "$row" "$work/out" | head -n 1 >>"$work/p/proven/registry/sq-map.tsv" +expect 'adding the printed row makes it pass' 0 "$work/p" "$work/new.deed" + +fresh +grep -v '^hypatia/workflow_audit/actions_lock_stale ' "$src/proven/registry/sq-map.tsv" >"$work/p/proven/registry/sq-map.tsv" +expect 'a deleted ADR 0004 workflow_audit row fails' 1 "$work/p" "$work/own.deed" +grep -q '^hypatia/workflow_audit/actions_lock_stale mapped' "$work/out" && + ok 'the cure asks for the mapped row back' || bad 'the cure does not name the mapped row' + +fresh +expect 'a missing checkout is NOT RUN' 2 "$work/absent" "$work/own.deed" +rm "$work/p/tests/registry-sq-map.sh" +expect 'a checkout without the test is NOT RUN' 2 "$work/p" "$work/own.deed" +fresh +expect 'a missing deed is NOT RUN' 2 "$work/p" "$work/absent.deed" +echo 'no ids here' >"$work/empty.deed" +expect 'a registry the test cannot read (its exit 2) is NOT RUN' 2 "$work/p" "$work/empty.deed" +expect 'no PROVEN_CICD_DIR is NOT RUN' 2 + +fresh +rc=0; env -u GITHUB_STEP_SUMMARY bash scripts/check-sq-map.sh "$work/p" >"$work/out" 2>&1 || rc=$? +if [[ $rc == 0 || $rc == 1 ]] && grep -q 'gen-rules-praxis.sh --stdout' "$work/out"; then + ok "with no deed the generator's registry is judged (exit $rc)" +else bad "generator path: exit $rc"; tail -8 "$work/out" >&2; fi + +echo "check-sq-map-test: $pass passed, $fail failed" +[[ $fail == 0 ]]