diff --git a/.github/workflows/audit-release-gate-tests.yml b/.github/workflows/audit-release-gate-tests.yml new file mode 100644 index 00000000..a1baaa8f --- /dev/null +++ b/.github/workflows/audit-release-gate-tests.yml @@ -0,0 +1,52 @@ +name: Audit Release Gate Tests + +on: + pull_request: + paths: + - '.github/workflows/audit-release-gate-tests.yml' + - '.github/workflows/publish.yml' + - '.github/workflows/release-datadog-serverless-compat.yml' + - '.github/workflows/build-datadog-serverless-compat.yml' + - 'Cargo.lock' + - '.cargo/audit.toml' + - 'scripts/cargo-audit-release-gate.sh' + - 'scripts/cargo-audit-report-guard.sh' + - 'scripts/tests/audit-release-gate/**' + workflow_dispatch: + inputs: + network_smoke: + description: 'Also run the opt-in real-scanner network smoke suite' + required: false + type: boolean + default: false + +# Read-only: this check never publishes, signs, or uploads anything. +permissions: + contents: read + +jobs: + offline: + name: Offline gate tests + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 + with: + python-version: '3.12' + - name: Install test-only dependencies + run: python3 -m pip install --disable-pip-version-check 'PyYAML==6.0.2' + - name: Run deterministic offline gate tests + run: | + python3 -m unittest discover -v -s scripts/tests/audit-release-gate -p 'test_*.py' + + network-smoke: + name: Opt-in real-scanner smoke + if: ${{ github.event_name == 'workflow_dispatch' && inputs.network_smoke }} + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions-rust-lang/setup-rust-toolchain@9399c7bb15d4c7d47b27263d024f0a4978346ba4 # v1.11.0 + with: + cache: false + - name: Run real scanner download, checksum, refresh, and scan + run: python3 scripts/tests/audit-release-gate/network_smoke.py diff --git a/.github/workflows/build-datadog-serverless-compat.yml b/.github/workflows/build-datadog-serverless-compat.yml index 070938fe..2b2aea22 100644 --- a/.github/workflows/build-datadog-serverless-compat.yml +++ b/.github/workflows/build-datadog-serverless-compat.yml @@ -17,6 +17,8 @@ jobs: RUSTC_WRAPPER: "sccache" steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ github.sha }} - uses: actions-rust-lang/setup-rust-toolchain@9399c7bb15d4c7d47b27263d024f0a4978346ba4 # v1.11.0 with: cache: false @@ -27,6 +29,8 @@ jobs: runs-on: ${{ inputs.runner }} steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ github.sha }} - name: Install Protoc Binary shell: bash run: chmod +x ./scripts/install-protoc.sh && ./scripts/install-protoc.sh $HOME @@ -35,7 +39,7 @@ jobs: run: | sudo apt-get update rustup target add x86_64-unknown-linux-musl && sudo apt-get install -y musl-tools - cargo build --release -p datadog-serverless-compat --target x86_64-unknown-linux-musl + cargo build --release -p datadog-serverless-compat --target x86_64-unknown-linux-musl --locked - if: ${{ inputs.runner == 'ubuntu-24.04' }} uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # 4.6.2 with: @@ -47,7 +51,7 @@ jobs: run: | sudo apt-get update rustup target add aarch64-unknown-linux-musl && sudo apt-get install -y musl-tools - cargo build --release -p datadog-serverless-compat --target aarch64-unknown-linux-musl + cargo build --release -p datadog-serverless-compat --target aarch64-unknown-linux-musl --locked - if: ${{ inputs.runner == 'ubuntu-24.04-arm' }} uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # 4.6.2 with: @@ -56,7 +60,7 @@ jobs: retention-days: 3 - if: ${{ inputs.runner == 'windows-2022' }} shell: bash - run: cargo build --release -p datadog-serverless-compat --features windows-pipes,windows-enhanced-metrics + run: cargo build --release -p datadog-serverless-compat --features windows-pipes,windows-enhanced-metrics --locked - if: ${{ inputs.runner == 'windows-2022' }} uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # 4.6.2 with: @@ -69,7 +73,8 @@ jobs: rustup target add i686-pc-windows-msvc cargo build --release -p datadog-serverless-compat \ --target i686-pc-windows-msvc \ - --features windows-pipes,windows-enhanced-metrics + --features windows-pipes,windows-enhanced-metrics \ + --locked - if: ${{ inputs.runner == 'windows-2022' }} uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # 4.6.2 with: @@ -78,7 +83,7 @@ jobs: retention-days: 3 - if: ${{ inputs.runner == 'macos-14' }} shell: bash - run: cargo build --release -p datadog-serverless-compat + run: cargo build --release -p datadog-serverless-compat --locked - if: ${{ inputs.runner == 'macos-14' }} uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # 4.6.2 with: diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 73954695..e011799f 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -17,6 +17,7 @@ on: type: string permissions: + contents: read # Read-only checkout of the audited source revision id-token: write # For OIDC npm publishing with provenance jobs: @@ -35,6 +36,7 @@ jobs: steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: + ref: ${{ github.sha }} sparse-checkout: npm sparse-checkout-cone-mode: false - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # 4.3.0 @@ -109,6 +111,11 @@ jobs: environment: Production needs: package steps: + - name: Check out the audited source revision + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ github.sha }} + path: source - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # 4.3.0 with: name: npm-packages @@ -117,6 +124,13 @@ jobs: with: node-version: "22.x" registry-url: 'https://registry.npmjs.org' + - name: Install Rust toolchain for cargo-audit + uses: actions-rust-lang/setup-rust-toolchain@9399c7bb15d4c7d47b27263d024f0a4978346ba4 # v1.11.0 + with: + cache: false + - name: Strict cargo-audit release gate + working-directory: source + run: ./scripts/cargo-audit-release-gate.sh "$GITHUB_SHA" - name: Publish npm packages run: | npm publish ./npm/datadog-serverless-compat-linux-x64 --provenance --access public diff --git a/.github/workflows/release-datadog-serverless-compat.yml b/.github/workflows/release-datadog-serverless-compat.yml index ae17f253..ae38ab04 100644 --- a/.github/workflows/release-datadog-serverless-compat.yml +++ b/.github/workflows/release-datadog-serverless-compat.yml @@ -19,12 +19,24 @@ jobs: permissions: contents: write steps: + - name: Check out the audited source revision + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ github.sha }} + path: source - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # 4.3.0 with: path: target/bin - run: | chmod +x target/bin/linux-amd64/datadog-serverless-compat chmod +x target/bin/windows-amd64/datadog-serverless-compat.exe + - name: Install Rust toolchain for cargo-audit + uses: actions-rust-lang/setup-rust-toolchain@9399c7bb15d4c7d47b27263d024f0a4978346ba4 # v1.11.0 + with: + cache: false + - name: Strict cargo-audit release gate + working-directory: source + run: ./scripts/cargo-audit-release-gate.sh "$GITHUB_SHA" - run: zip -r datadog-serverless-compat.zip ./* working-directory: target - uses: softprops/action-gh-release@da05d552573ad5aba039eaac05058a918a7bf631 # v2.2.2 diff --git a/scripts/cargo-audit-release-gate.sh b/scripts/cargo-audit-release-gate.sh new file mode 100755 index 00000000..39128793 --- /dev/null +++ b/scripts/cargo-audit-release-gate.sh @@ -0,0 +1,202 @@ +#!/usr/bin/env bash +# +# Strict cargo-audit release gate for datadog-serverless-compat publication. +# +# Usage: scripts/cargo-audit-release-gate.sh +# +# Run this gate inside a protected publication job, immediately before its first +# publication side effect. It performs downloads and a dependency scan only: +# it never publishes, signs, uploads, or pushes anything. +# +# The gate fails closed unless every one of these holds: +# * the repository root is the expected commit (usually $GITHUB_SHA); +# * the working-tree Cargo.lock and .cargo/audit.toml byte-match that commit, +# so the audited lockfile and advisory policy cannot be swapped out; +# * a working Rust toolchain is present, because the pinned prebuilt +# cargo-audit still shells out to `cargo -V`; +# * the pinned official cargo-audit 0.22.2 (Linux x86_64 musl) archive +# downloads (bounded by curl's --connect-timeout 15 and --max-time 180) and +# matches its reviewed SHA-256; +# * the installed binary reports exactly the expected version; +# * a fresh RustSec advisory database and a fresh Cargo cache are created in a +# private temporary directory, so a warm cache can never turn a failed +# advisory refresh into an offline success; +# * the scanner runs under coreutils timeout (--kill-after=10s, 5m) and exits +# zero, the terminal log shows the full Fetching/Loaded/Updating/Scanning +# sequence with nonzero advisory and dependency counts for the expected +# lockfile, and the log carries none of the crates.io index diagnostics the +# pinned binary can otherwise hide behind a zero exit status. See +# scripts/cargo-audit-report-guard.sh. +# +# The archive URL, digest, scanner version, and advisory database URL are fixed +# reviewed constants. This script deliberately honors no environment variable +# that could point it at a different archive, digest, or advisory source, and it +# passes no --no-fetch, --stale, or extra ignore flags. All exceptions come from +# the repository's reviewed .cargo/audit.toml. +set -euo pipefail + +readonly SCANNER_VERSION_LINE="cargo-audit-audit 0.22.2" +readonly ARCHIVE_URL="https://github.com/rustsec/rustsec/releases/download/cargo-audit/v0.22.2/cargo-audit-x86_64-unknown-linux-musl-v0.22.2.tgz" +readonly ARCHIVE_SHA256="7fb9497f8594b389e5fce5ef9b92db08432996895b2e0c5a0167a69ed445c428" +readonly ADVISORY_DB_URL="https://github.com/RustSec/advisory-db.git" + +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +readonly script_dir +readonly GUARD_SCRIPT="${script_dir}/cargo-audit-report-guard.sh" + +die() { + printf 'cargo-audit release gate: ERROR: %s\n' "$*" >&2 + exit 2 +} + +fail() { + printf 'cargo-audit release gate: FAILED: %s\n' "$*" >&2 + exit 1 +} + +if [ "$#" -ne 1 ]; then + die "usage: $0 " +fi + +expected_sha="$1" +if ! [[ "$expected_sha" =~ ^([0-9a-f]{40}|[0-9a-f]{64})$ ]]; then + die "expected commit must be a 40- or 64-character lowercase hex id" +fi + +[ -x "$GUARD_SCRIPT" ] || fail "report guard is missing or not executable: $GUARD_SCRIPT" + +for tool in git cargo curl tar sha256sum cut timeout; do + command -v "$tool" >/dev/null 2>&1 || die "required tool not found on PATH: $tool" +done + +cargo_version="$(cargo -V 2>&1)" || die "cargo -V failed; a working Rust toolchain must be installed before this gate" + +repo_root="$(git rev-parse --show-toplevel 2>/dev/null)" || die "not inside a git work tree" +cd "$repo_root" + +head_sha="$(git rev-parse HEAD)" || die "cannot resolve HEAD" + +lockfile="$repo_root/Cargo.lock" +policy="$repo_root/.cargo/audit.toml" +[ -f "$lockfile" ] || fail "missing Cargo.lock at repository root" +[ -f "$policy" ] || fail "missing .cargo/audit.toml at repository root" + +if [ "$head_sha" != "$expected_sha" ]; then + fail "HEAD ($head_sha) does not match expected release commit ($expected_sha)" +fi + +# Bind the audited inputs to the expected commit, not to the working tree alone. +if ! expected_lock_hash="$(git show "${expected_sha}:Cargo.lock" | sha256sum | cut -d' ' -f1)"; then + fail "cannot read Cargo.lock from expected commit $expected_sha" +fi +actual_lock_hash="$(sha256sum "$lockfile" | cut -d' ' -f1)" +if [ "$expected_lock_hash" != "$actual_lock_hash" ]; then + fail "working-tree Cargo.lock does not match the expected commit" +fi + +if ! expected_policy_hash="$(git show "${expected_sha}:.cargo/audit.toml" | sha256sum | cut -d' ' -f1)"; then + fail "cannot read .cargo/audit.toml from expected commit $expected_sha" +fi +actual_policy_hash="$(sha256sum "$policy" | cut -d' ' -f1)" +if [ "$expected_policy_hash" != "$actual_policy_hash" ]; then + fail "working-tree .cargo/audit.toml does not match the expected commit" +fi + +work="$(mktemp -d "${TMPDIR:-/tmp}/cargo-audit-release-gate.XXXXXXXX")" || die "cannot create a temporary directory" +readonly work +# Single quotes defer expansion to EXIT so the cleanup always targets this run's +# directory without interpolating it into a fragile command string. +trap 'rm -rf -- "$work"' EXIT + +archive="$work/cargo-audit.tgz" +install_dir="$work/install" +mkdir -p "$install_dir" + +if ! curl --fail --silent --show-error --location --proto '=https' \ + --connect-timeout 15 --max-time 180 --output "$archive" "$ARCHIVE_URL"; then + fail "failed to download the pinned cargo-audit archive" +fi + +archive_hash="$(sha256sum "$archive" | cut -d' ' -f1)" +if [ "$archive_hash" != "$ARCHIVE_SHA256" ]; then + fail "pinned cargo-audit archive checksum mismatch" +fi + +if ! tar -xzf "$archive" -C "$install_dir" --strip-components=1; then + fail "failed to extract the pinned cargo-audit archive" +fi + +scanner="$install_dir/cargo-audit" +[ -x "$scanner" ] || fail "extracted archive does not contain an executable cargo-audit" + +scanner_version="$("$scanner" audit --version 2>&1)" || fail "cannot execute the installed cargo-audit" +if [ "$scanner_version" != "$SCANNER_VERSION_LINE" ]; then + fail "unexpected cargo-audit version: '$scanner_version' (expected '$SCANNER_VERSION_LINE')" +fi + +# Fresh advisory database and fresh Cargo cache on every execution. Nothing from +# a previous run, job, or retry is reused, so a stale cache cannot authorize a +# release when the required refresh fails. +cargo_home="$work/cargo-home" +mkdir -p "$cargo_home" +db="$work/rustsec-db" +scan_log="$work/cargo-audit.log" + +# The scan is bounded by coreutils timeout: 5 minutes of runtime, then a 10 +# second grace before SIGKILL. A killed scan exits 124 (SIGTERM) or 137 +# (SIGKILL); both are nonzero and are rejected by the report guard below. +scan_rc=0 +CARGO_HOME="$cargo_home" timeout --kill-after=10s 5m "$scanner" audit \ + --file "$lockfile" \ + --db "$db" \ + --url "$ADVISORY_DB_URL" \ + --format terminal \ + --color never >"$scan_log" 2>&1 || scan_rc=$? + +db_origin="unavailable" +db_revision="unavailable" +if [ -d "$db/.git" ]; then + db_origin="$(git -C "$db" remote get-url origin 2>/dev/null || printf 'unavailable')" + db_revision="$(git -C "$db" rev-parse HEAD 2>/dev/null || printf 'unavailable')" +fi + +printf 'cargo-audit release gate\n' +printf 'expected-sha: %s\n' "$expected_sha" +printf 'head-sha: %s\n' "$head_sha" +printf 'cargo-lock-sha256: %s\n' "$actual_lock_hash" +printf 'audit-toml-sha256: %s\n' "$actual_policy_hash" +printf 'cargo-version: %s\n' "$cargo_version" +printf 'scanner-version: %s\n' "$scanner_version" +printf 'advisory-db-url: %s\n' "$ADVISORY_DB_URL" +printf 'advisory-db-origin: %s\n' "$db_origin" +printf 'advisory-db-revision: %s\n' "$db_revision" +printf 'scanner-exit: %s\n' "$scan_rc" + +printf '%s\n' '----- begin cargo-audit output -----' +if ! cat "$scan_log"; then + fail "cannot read the cargo-audit scan log: $scan_log" +fi +printf '%s\n' '----- end cargo-audit output -----' + +guard_rc=0 +"$GUARD_SCRIPT" "$scan_log" "$scan_rc" "$lockfile" || guard_rc=$? +case "$guard_rc" in + 0 | 1) ;; + *) fail "report guard returned an unexpected status ($guard_rc); treating as failure" ;; +esac + +if [ "$guard_rc" -ne 0 ]; then + printf 'cargo-audit release gate: result: FAIL (scanner-exit=%s)\n' "$scan_rc" >&2 + if [ "$scan_rc" -ne 0 ]; then + # Preserve the scanner's own status for rejected findings and errors. + exit "$scan_rc" + fi + exit 1 +fi + +if [ "$db_origin" != "$ADVISORY_DB_URL" ]; then + fail "unexpected advisory database origin: '$db_origin'" +fi + +printf 'cargo-audit release gate: result: PASS\n' +exit 0 diff --git a/scripts/cargo-audit-report-guard.sh b/scripts/cargo-audit-report-guard.sh new file mode 100755 index 00000000..3052f132 --- /dev/null +++ b/scripts/cargo-audit-report-guard.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash +# +# Pure verdict guard for a cargo-audit release scan. +# +# Usage: scripts/cargo-audit-report-guard.sh +# +# The guard exits 0 only when all of the following hold: +# * the scanner exit code is a non-negative integer and equals zero; +# * the scan log contains the complete Fetching/Loaded/Updating/Scanning +# progress sequence emitted by the pinned cargo-audit 0.22.2 binary, in that +# order, with a nonzero loaded-advisory count and a nonzero crate-dependency +# count for the expected lockfile; +# * the lockfile named on the Scanning line is exactly the expected lockfile +# path the gate passed to the scanner, so a scan of a different or +# substituted file cannot authorize a release; +# * the scan log contains none of the pinned binary's crates.io index +# diagnostics ("couldn't update crates.io index", "couldn't open +# crates.io index", "couldn't check if the package is yanked"). +# +# cargo-audit 0.22.2 can exit zero while hiding an internal crates.io index +# failure behind JSON output. Exit status and JSON alone are therefore not +# enough to authorize a release, which is why the human-readable terminal log +# is parsed here. +# +# The marker shapes this guard requires are pinned to the scanner version's +# terminal output. Structured lines are matched rather than bare marker words, so +# a log that merely mentions "Loaded" or "Scanning" without the substantive +# payload cannot pass. Re-review this guard and its regression tests whenever the +# pinned version changes. +# +# This script performs no network, download, installation, or publication work, +# so it can be exercised entirely offline. +set -euo pipefail + +if [ "$#" -ne 3 ]; then + printf 'cargo-audit report guard: usage: %s \n' "$0" >&2 + exit 2 +fi + +log="$1" +rc="$2" +expected_lockfile="$3" + +reject() { + printf 'cargo-audit report guard: %s\n' "$*" >&2 + exit 1 +} + +if [ -z "$expected_lockfile" ]; then + printf 'cargo-audit report guard: expected lockfile path must not be empty\n' >&2 + exit 2 +fi + +if [ ! -f "$log" ]; then + printf 'cargo-audit report guard: scan log not found: %s\n' "$log" >&2 + exit 2 +fi + +case "$rc" in + '' | *[!0-9]*) + printf 'cargo-audit report guard: scanner exit code is not a non-negative integer: %s\n' "$rc" >&2 + exit 2 + ;; +esac + +if [ "$rc" -ne 0 ]; then + printf 'cargo-audit report guard: scanner exited %s; release blocked\n' "$rc" >&2 + exit 1 +fi + +if grep -Eq "couldn't (update|open) crates\.io index|couldn't check if the package is yanked" "$log"; then + reject "scan log shows an incomplete crates.io index check; release blocked" +fi + +fetch_line=0 +loaded_line=0 +updating_line=0 +scanning_line=0 +scanned_lockfile="" +line_number=0 +while IFS= read -r line || [ -n "$line" ]; do + line_number=$((line_number + 1)) + + if [ "$fetch_line" -eq 0 ] && [[ "$line" =~ ^[[:space:]]*Fetching[[:space:]]+advisory[[:space:]]+database[[:space:]]+from[[:space:]] ]]; then + fetch_line=$line_number + continue + fi + + if [ "$loaded_line" -eq 0 ] && [[ "$line" =~ ^[[:space:]]*Loaded[[:space:]]+([1-9][0-9]*)[[:space:]]+security[[:space:]]+advisories ]]; then + loaded_line=$line_number + continue + fi + + if [ "$updating_line" -eq 0 ] && [[ "$line" =~ ^[[:space:]]*Updating[[:space:]]+crates\.io[[:space:]]+index[[:space:]]*$ ]]; then + updating_line=$line_number + continue + fi + + if [ "$scanning_line" -eq 0 ] && [[ "$line" =~ ^[[:space:]]*Scanning[[:space:]]+(.+)[[:space:]]+for[[:space:]]+vulnerabilities[[:space:]]+\(([1-9][0-9]*)[[:space:]]+crate[[:space:]]+dependencies\)[[:space:]]*$ ]]; then + scanning_line=$line_number + scanned_lockfile="${BASH_REMATCH[1]}" + continue + fi +done <"$log" + +[ "$fetch_line" -ne 0 ] || reject "scan log is missing the advisory database fetch progress line" +[ "$loaded_line" -ne 0 ] || reject "scan log does not report a nonzero count of loaded security advisories" +[ "$updating_line" -ne 0 ] || reject "scan log is missing the crates.io index update progress line" +[ "$scanning_line" -ne 0 ] || reject "scan log does not report scanning a lockfile with a nonzero crate dependency count" + +if [ "$fetch_line" -ge "$loaded_line" ] || + [ "$loaded_line" -ge "$updating_line" ] || + [ "$updating_line" -ge "$scanning_line" ]; then + reject "scan log progress lines are out of order" +fi + +if [ "$scanned_lockfile" != "$expected_lockfile" ]; then + reject "scan log scanned '$scanned_lockfile' but the gate expected '$expected_lockfile'" +fi + +exit 0 diff --git a/scripts/tests/audit-release-gate/README.md b/scripts/tests/audit-release-gate/README.md new file mode 100644 index 00000000..4c175f2e --- /dev/null +++ b/scripts/tests/audit-release-gate/README.md @@ -0,0 +1,61 @@ +# Audit release-gate tests + +Exercise the real release gate offline with disposable git repositories and +fixture-only scanner/download doubles. No test publishes, signs or uploads. + +## Run + +Activate a disposable virtual environment outside the checkout and install +`PyYAML==6.0.2`. Use that environment's Python for the offline suite: + +```sh +PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -v -s scripts/tests/audit-release-gate -p 'test_*.py' +# Opt-in: requires cargo and network access, uses isolated fixture lockfiles. +PYTHONDONTWRITEBYTECODE=1 python3 scripts/tests/audit-release-gate/network_smoke.py +``` + +| File | Coverage | +|---|---| +| `test_release_gate.py` | Identity, installation, verdicts, sentinel blocking, fresh state/retry, cleanup, pins and command bounds | +| `test_report_guard.py` | Captured verdicts, synthetic incomplete scans, ordered markers, counts, exact paths and status/usage errors | +| `test_workflows.py` | Inline ordering, approvals, permissions, source/artifact binding and locked builds, with representative mutations | +| `network_smoke.py` | Real clean/retry, findings/committed ignore, informational warning, and scanner/database identity | + +## Captured log provenance + +`fixtures/logs/` contains terminal output from the official cargo-audit 0.22.2 +Linux x86_64 musl archive, SHA-256 +`7fb9497f8594b389e5fce5ef9b92db08432996895b2e0c5a0167a69ed445c428`. +Capture combined stdout/stderr from a temporary fixture using `cargo-audit audit +--file Cargo.lock --db --url https://github.com/RustSec/advisory-db.git +--format terminal --color never`. Index failure was induced with a directory at +`CARGO_HOME/.package-cache`; configuration cases disabled fetching or yanked checks. +The relative captured path is expected as `Cargo.lock`; embedded scratch paths +are not assertions. Retained logs are unchanged from the original captures. + +| Log | Scanner status | Guard verdict | +|---|---|---| +| `clean-success.log` | 0 | pass | +| `warning-only-success.log` | 0 | pass | +| `vulnerable-failure.log` | 1 | reject | +| `index-error-rc0.log` | 0 | reject | +| `quiet-output-rc0.log` | 0 | reject | +| `disabled-yanked-rc0.log` | 0 | reject | +| `disabled-fetch-rc0.log` | 0 | reject | + +## Lint and limits + +```sh +shellcheck scripts/cargo-audit-release-gate.sh scripts/cargo-audit-report-guard.sh \ + scripts/tests/audit-release-gate/fixtures/fake-scanner/cargo-audit \ + scripts/tests/audit-release-gate/fixtures/stubs/{cargo,curl,sha256sum} +``` + +Captured logs are real output; in-memory variants are synthetic contract fixtures. +Only the opt-in smoke executes the real scanner and proves current output shapes, +nonzero counts and absolute scan paths. The guard remains coupled to 0.22.2. +Timeout statuses are replayed, not actual timer/signal tests; bounds are checked +on command tokens with comments excluded. Workflow checks cover publication and +build wiring, not the tests workflow itself or GitHub-runner integration. +The repository lockfile need not +pass: the smoke suite uses fixtures, not the current unresolved advisories. diff --git a/scripts/tests/audit-release-gate/fixtures/fake-scanner/cargo-audit b/scripts/tests/audit-release-gate/fixtures/fake-scanner/cargo-audit new file mode 100755 index 00000000..8afd3686 --- /dev/null +++ b/scripts/tests/audit-release-gate/fixtures/fake-scanner/cargo-audit @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# +# Replay double for the pinned cargo-audit binary, used only by +# test_release_gate.py and never by production. Fixture-only variables: +# FAKE_VERSION line printed for `audit --version` +# FAKE_RECORD file that receives one record block per scan +# FAKE_LOG captured scanner log to replay +# FAKE_RC exit status to return after replaying (default 0) +# FAKE_NO_DB when set, create no advisory database (fetch failure) +# FAKE_ORIGIN origin URL of the created advisory database +# FAKE_SCAN_PATH lockfile path shown on the Scanning line instead of --file +set -euo pipefail +export GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_SYSTEM=/dev/null + +if [ "$*" = "audit --version" ]; then + printf '%s\n' "${FAKE_VERSION:-cargo-audit-audit 0.22.2}" + exit 0 +fi + +db="" lockfile="" previous="" +for arg in "$@"; do + case "$previous" in + --db) db="$arg" ;; + --file) lockfile="$arg" ;; + esac + previous="$arg" +done + +# Record the state this scan inherited, before the database is created. +home="${CARGO_HOME:-}" +{ + printf -- '---\ndb=%s\nhome=%s\n' "$db" "$home" + if [ -e "$db" ]; then echo db_existed=yes; else echo db_existed=no; fi + if [ -d "$home" ] && [ -z "$(ls -A -- "$home")" ]; then echo home_empty=yes; else echo home_empty=no; fi + printf 'arg=%s\n' "$@" +} >>"$FAKE_RECORD" + +if [ -z "${FAKE_NO_DB:-}" ]; then + git init --quiet "$db" + git -C "$db" remote add origin "${FAKE_ORIGIN:-https://github.com/RustSec/advisory-db.git}" + git -C "$db" -c user.name=fixture -c user.email=fixture@example.invalid \ + commit --quiet --allow-empty -m 'fixture advisory database' +fi + +scan_path="${FAKE_SCAN_PATH:-$lockfile}" +while IFS= read -r line || [ -n "$line" ]; do + printf '%s\n' "${line/Scanning Cargo.lock for/"Scanning $scan_path for"}" +done <"$FAKE_LOG" +exit "${FAKE_RC:-0}" diff --git a/scripts/tests/audit-release-gate/fixtures/logs/clean-success.log b/scripts/tests/audit-release-gate/fixtures/logs/clean-success.log new file mode 100644 index 00000000..8b017d21 --- /dev/null +++ b/scripts/tests/audit-release-gate/fixtures/logs/clean-success.log @@ -0,0 +1,4 @@ + Fetching advisory database from `https://github.com/RustSec/advisory-db.git` + Loaded 1380 security advisories (from /tmp/cargo-release-preflight-vrpsnQ/rustsec-db) + Updating crates.io index + Scanning Cargo.lock for vulnerabilities (1 crate dependencies) diff --git a/scripts/tests/audit-release-gate/fixtures/logs/disabled-fetch-rc0.log b/scripts/tests/audit-release-gate/fixtures/logs/disabled-fetch-rc0.log new file mode 100644 index 00000000..6e5b30c7 --- /dev/null +++ b/scripts/tests/audit-release-gate/fixtures/logs/disabled-fetch-rc0.log @@ -0,0 +1,2 @@ + Loaded 1380 security advisories (from /tmp/cargo-release-preflight-vrpsnQ/rustsec-db) + Scanning Cargo.lock for vulnerabilities (1 crate dependencies) diff --git a/scripts/tests/audit-release-gate/fixtures/logs/disabled-yanked-rc0.log b/scripts/tests/audit-release-gate/fixtures/logs/disabled-yanked-rc0.log new file mode 100644 index 00000000..928d3a90 --- /dev/null +++ b/scripts/tests/audit-release-gate/fixtures/logs/disabled-yanked-rc0.log @@ -0,0 +1,3 @@ + Fetching advisory database from `https://github.com/RustSec/advisory-db.git` + Loaded 1380 security advisories (from /tmp/cargo-release-preflight-vrpsnQ/rustsec-db) + Scanning Cargo.lock for vulnerabilities (1 crate dependencies) diff --git a/scripts/tests/audit-release-gate/fixtures/logs/index-error-rc0.log b/scripts/tests/audit-release-gate/fixtures/logs/index-error-rc0.log new file mode 100644 index 00000000..5cb97983 --- /dev/null +++ b/scripts/tests/audit-release-gate/fixtures/logs/index-error-rc0.log @@ -0,0 +1,5 @@ + Fetching advisory database from `https://github.com/RustSec/advisory-db.git` + Loaded 1380 security advisories (from /tmp/cargo-release-preflight-vrpsnQ/rustsec-db) + Updating crates.io index +warning: couldn't update crates.io index: I/O operation failed: failed to obtain lock file '/tmp/gate-fixture-capture/index-home/.package-cache' + Scanning Cargo.lock for vulnerabilities (1 crate dependencies) diff --git a/scripts/tests/audit-release-gate/fixtures/logs/quiet-output-rc0.log b/scripts/tests/audit-release-gate/fixtures/logs/quiet-output-rc0.log new file mode 100644 index 00000000..e69de29b diff --git a/scripts/tests/audit-release-gate/fixtures/logs/vulnerable-failure.log b/scripts/tests/audit-release-gate/fixtures/logs/vulnerable-failure.log new file mode 100644 index 00000000..3ef98b63 --- /dev/null +++ b/scripts/tests/audit-release-gate/fixtures/logs/vulnerable-failure.log @@ -0,0 +1,62 @@ + Fetching advisory database from `https://github.com/RustSec/advisory-db.git` + Loaded 1380 security advisories (from /tmp/cargo-release-preflight-vrpsnQ/rustsec-db) + Updating crates.io index + Scanning Cargo.lock for vulnerabilities (2 crate dependencies) +Crate: openssl +Version: 0.10.38 +Title: `openssl` `X509VerifyParamRef::set_host` buffer over-read +Date: 2023-06-20 +ID: RUSTSEC-2023-0044 +URL: https://rustsec.org/advisories/RUSTSEC-2023-0044 +Solution: Upgrade to >=0.10.55 + +Crate: openssl +Version: 0.10.38 +Title: `openssl` `X509Extension::new` and `X509Extension::new_nid` null pointer dereference +Date: 2023-03-24 +ID: RUSTSEC-2023-0024 +URL: https://rustsec.org/advisories/RUSTSEC-2023-0024 +Solution: Upgrade to >=0.10.48 + +Crate: openssl +Version: 0.10.38 +Title: ssl::select_next_proto use after free +Date: 2025-02-02 +ID: RUSTSEC-2025-0004 +URL: https://rustsec.org/advisories/RUSTSEC-2025-0004 +Solution: Upgrade to >=0.10.70 + +Crate: openssl +Version: 0.10.38 +Title: `openssl` `SubjectAlternativeName` and `ExtendedKeyUsage::other` allow arbitrary file read +Date: 2023-03-24 +ID: RUSTSEC-2023-0023 +URL: https://rustsec.org/advisories/RUSTSEC-2023-0023 +Solution: Upgrade to >=0.10.48 + +Crate: openssl +Version: 0.10.38 +Title: `openssl` `X509NameBuilder::build` returned object is not thread safe +Date: 2023-03-24 +ID: RUSTSEC-2023-0022 +URL: https://rustsec.org/advisories/RUSTSEC-2023-0022 +Solution: Upgrade to >=0.10.48 + +Crate: openssl +Version: 0.10.38 +Title: `MemBio::get_buf` has undefined behavior with empty buffers +Date: 2024-07-21 +ID: RUSTSEC-2024-0357 +URL: https://rustsec.org/advisories/RUSTSEC-2024-0357 +Solution: Upgrade to >=0.10.66 + +Crate: openssl +Version: 0.10.38 +Warning: unsound +Title: `openssl` `X509StoreRef::objects` is unsound +Date: 2023-11-23 +ID: RUSTSEC-2023-0072 +URL: https://rustsec.org/advisories/RUSTSEC-2023-0072 + +error: 6 vulnerabilities found! +warning: 1 allowed warning found diff --git a/scripts/tests/audit-release-gate/fixtures/logs/warning-only-success.log b/scripts/tests/audit-release-gate/fixtures/logs/warning-only-success.log new file mode 100644 index 00000000..c65cd8af --- /dev/null +++ b/scripts/tests/audit-release-gate/fixtures/logs/warning-only-success.log @@ -0,0 +1,13 @@ + Fetching advisory database from `https://github.com/RustSec/advisory-db.git` + Loaded 1380 security advisories (from /tmp/cargo-release-preflight-vrpsnQ/rustsec-db) + Updating crates.io index + Scanning Cargo.lock for vulnerabilities (2 crate dependencies) +Crate: paste +Version: 1.0.15 +Warning: unmaintained +Title: paste - no longer maintained +Date: 2024-10-07 +ID: RUSTSEC-2024-0436 +URL: https://rustsec.org/advisories/RUSTSEC-2024-0436 + +warning: 1 allowed warning found diff --git a/scripts/tests/audit-release-gate/fixtures/stubs/cargo b/scripts/tests/audit-release-gate/fixtures/stubs/cargo new file mode 100755 index 00000000..4ae3e558 --- /dev/null +++ b/scripts/tests/audit-release-gate/fixtures/stubs/cargo @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# cargo double for test_release_gate.py: answers `cargo -V`, or simulates a +# missing toolchain when FAKE_CARGO_FAIL is set. +if [ -n "${FAKE_CARGO_FAIL:-}" ]; then + echo 'error: fixture cargo is unavailable' >&2 + exit 1 +fi +echo 'cargo 1.99.0 (fixture)' diff --git a/scripts/tests/audit-release-gate/fixtures/stubs/curl b/scripts/tests/audit-release-gate/fixtures/stubs/curl new file mode 100755 index 00000000..a6a90119 --- /dev/null +++ b/scripts/tests/audit-release-gate/fixtures/stubs/curl @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# curl double for test_release_gate.py: records argv (one arg per line), then +# copies FAKE_ARCHIVE to the --output path. FAKE_CURL_FAIL simulates a failure. +set -euo pipefail +{ printf -- '---\n'; printf 'arg=%s\n' "$@"; } >>"$FAKE_CURL_RECORD" +[ -z "${FAKE_CURL_FAIL:-}" ] || { echo 'curl: (22) fixture download failure' >&2; exit 22; } +output="" previous="" +for arg in "$@"; do + [ "$previous" != --output ] || output="$arg" + previous="$arg" +done +[ -n "$output" ] || { echo 'curl stub: missing --output' >&2; exit 2; } +cp -- "$FAKE_ARCHIVE" "$output" diff --git a/scripts/tests/audit-release-gate/fixtures/stubs/sha256sum b/scripts/tests/audit-release-gate/fixtures/stubs/sha256sum new file mode 100755 index 00000000..efd5e539 --- /dev/null +++ b/scripts/tests/audit-release-gate/fixtures/stubs/sha256sum @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# sha256sum double for test_release_gate.py: reports FAKE_ARCHIVE_SHA256 for the +# downloaded */cargo-audit.tgz only and delegates every other call (lockfile, +# policy, and stdin hashing) to the real binary at REAL_SHA256SUM. +set -euo pipefail +if [ "$#" -eq 1 ] && [[ "$1" == */cargo-audit.tgz ]]; then + printf '%s %s\n' "$FAKE_ARCHIVE_SHA256" "$1" + exit 0 +fi +exec "$REAL_SHA256SUM" "$@" diff --git a/scripts/tests/audit-release-gate/network_smoke.py b/scripts/tests/audit-release-gate/network_smoke.py new file mode 100644 index 00000000..822870c6 --- /dev/null +++ b/scripts/tests/audit-release-gate/network_smoke.py @@ -0,0 +1,93 @@ +"""Opt-in real-scanner tests; excluded from offline test_*.py discovery. + +Run from the repository root with cargo and network access. Each gate invocation +uses the real pinned download, checksum, fresh advisory DB and crates.io index. +Only temporary fixture lockfiles are scanned; nothing is published or uploaded. +""" +import os +import re +import subprocess +import tempfile +import unittest +from pathlib import Path + +GATE = Path(__file__).resolve().parents[2] / "cargo-audit-release-gate.sh" +VERSION = "cargo-audit-audit 0.22.2" +TRUSTED_DB = "https://github.com/RustSec/advisory-db.git" +LOCK = 'version = 3\n\n[[package]]\nname = "smoke-fixture"\nversion = "0.1.0"\n' +GIT_ENV = {**os.environ, "GIT_CONFIG_GLOBAL": os.devnull, "GIT_CONFIG_SYSTEM": os.devnull} + + +def git(repo, *args): + return subprocess.run(["git", "-C", str(repo), *args], check=True, + capture_output=True, text=True, env=GIT_ENV).stdout.strip() + + +def commit_all(repo, message): + git(repo, "add", "-A") + git(repo, "-c", "user.email=smoke@example.invalid", "-c", "user.name=smoke", + "commit", "--quiet", "-m", message) + return git(repo, "rev-parse", "HEAD") + + +def registry_package(name, version): + return (f'\n[[package]]\nname = "{name}"\nversion = "{version}"\n' + 'source = "registry+https://github.com/rust-lang/crates.io-index"\n') + + +def make_repo(root, lock): + repo = Path(root) / "repo" + (repo / ".cargo").mkdir(parents=True) + (repo / "Cargo.lock").write_text(lock) + (repo / ".cargo" / "audit.toml").write_text("[advisories]\nignore = []\n") + git(repo, "init", "--quiet", "--initial-branch=main") + return repo, commit_all(repo, "smoke fixture") + + +class NetworkSmoke(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory(prefix="audit-gate-smoke.") + self.addCleanup(tmp.cleanup) + self.root = tmp.name + + def assert_scanner_identity(self, output): + self.assertIn(f"scanner-version: {VERSION}", output) + self.assertIn(f"advisory-db-origin: {TRUSTED_DB}", output) + self.assertIn("Fetching advisory database", output) + self.assertRegex(output, r"(?m)^advisory-db-revision: (?:[0-9a-f]{40}|[0-9a-f]{64})$") + self.assertRegex(output, r"(?m)^\s*Loaded [1-9][0-9]* security advisories") + self.assertRegex( + output, r"(?m)^\s*Scanning /.*/Cargo\.lock for vulnerabilities " + r"\([1-9][0-9]* crate dependencies\)$", + ) + + def scan(self, repo, sha, accepted=True): + result = subprocess.run(["bash", str(GATE), sha], cwd=repo, capture_output=True, + text=True, timeout=900) + self.assertEqual(result.returncode == 0, accepted, result.stdout + result.stderr) + self.assertIn("result: PASS" if accepted else "result: FAIL", result.stdout + result.stderr) + self.assert_scanner_identity(result.stdout) + return result.stdout + + def test_clean_fixture_and_retry_refresh(self): + repo, sha = make_repo(self.root, LOCK) + self.scan(repo, sha) + # A retry must fetch again, not authorize from inherited state. + self.scan(repo, sha) + + def test_findings_block_until_ignore_policy_is_committed(self): + repo, sha = make_repo(self.root, LOCK + registry_package("openssl", "0.10.38")) + blocked = self.scan(repo, sha, accepted=False) + advisories = sorted(set(re.findall(r"RUSTSEC-\d{4}-\d{4}", blocked))) + self.assertTrue(advisories, blocked) + ignores = ",\n".join(f' "{advisory}"' for advisory in advisories) + (repo / ".cargo" / "audit.toml").write_text(f"[advisories]\nignore = [\n{ignores}\n]\n") + self.scan(repo, commit_all(repo, "ignore the fixture advisories")) + + def test_informational_warning_passes(self): + repo, sha = make_repo(self.root, LOCK + registry_package("paste", "1.0.15")) + self.assertIn("allowed warning found", self.scan(repo, sha)) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/scripts/tests/audit-release-gate/test_release_gate.py b/scripts/tests/audit-release-gate/test_release_gate.py new file mode 100644 index 00000000..3c5649d8 --- /dev/null +++ b/scripts/tests/audit-release-gate/test_release_gate.py @@ -0,0 +1,299 @@ +"""Offline control-flow tests for scripts/cargo-audit-release-gate.sh. + +Each test runs the real production gate inside a temporary git repository with a +tiny committed Cargo.lock and .cargo/audit.toml. The "downloaded" archive is a tar +of the replay scanner in fixtures/fake-scanner, and PATH doubles stand in only for +curl, sha256sum, and cargo; git, tar, cut, and coreutils timeout are real. The +doubles read fixture-only FAKE_* variables; the gate itself honors none of them. + +Scanner exit statuses 124 and 137 are returned by the replay scanner through the +real timeout binary. That proves the gate's exit-status handling for a killed scan, +not actual timer expiry or signal delivery; the bounds themselves are covered by +the static command test below. + +network_smoke.py is the opt-in runtime proof against the real scanner. +""" +import hashlib +import os +import re +import shlex +import shutil +import subprocess +import tarfile +import tempfile +import unittest +from pathlib import Path + +HERE = Path(__file__).resolve().parent +GATE = HERE.parents[1] / "cargo-audit-release-gate.sh" +FAKE_SCANNER = HERE / "fixtures" / "fake-scanner" / "cargo-audit" +STUBS = HERE / "fixtures" / "stubs" +LOGS = HERE / "fixtures" / "logs" + +# Reviewed pins, written out independently of the production script. +ARCHIVE_URL = ( + "https://github.com/rustsec/rustsec/releases/download/" + "cargo-audit/v0.22.2/cargo-audit-x86_64-unknown-linux-musl-v0.22.2.tgz" +) +ARCHIVE_SHA256 = "7fb9497f8594b389e5fce5ef9b92db08432996895b2e0c5a0167a69ed445c428" +SCANNER_VERSION = "cargo-audit-audit 0.22.2" +TRUSTED_DB = "https://github.com/RustSec/advisory-db.git" + +LOCK = 'version = 3\n\n[[package]]\nname = "gate-fixture"\nversion = "0.1.0"\n' +POLICY = "[advisories]\nignore = []\n" +GIT_ENV = {"GIT_CONFIG_GLOBAL": os.devnull, "GIT_CONFIG_SYSTEM": os.devnull} +REAL_SHA256SUM = shutil.which("sha256sum") +if REAL_SHA256SUM is None: + raise RuntimeError("sha256sum is required to run the offline release-gate tests") + + +def read_records(path): + """Parse '---'-separated blocks of key=value lines; 'arg' keys accumulate.""" + runs = [] + if path.exists(): + for line in path.read_text().splitlines(): + if line == "---": + runs.append({"arg": []}) + else: + key, _, value = line.partition("=") + if key == "arg": + runs[-1]["arg"].append(value) + else: + runs[-1][key] = value + return runs + + +def logical_commands(script_text): + """Join backslash continuations and shell-split each line, dropping comments.""" + joined = script_text.replace("\\\n", " ") + return [shlex.split(line, comments=True) for line in joined.splitlines()] + + +class ReleaseGateTests(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory(prefix="audit-gate-test.") + self.addCleanup(tmp.cleanup) + self.root = Path(tmp.name).resolve() + self.repo = self.root / "repo" + self.lock = self.repo / "Cargo.lock" + self.policy = self.repo / ".cargo" / "audit.toml" + self.tmpdir = self.root / "tmp" + self.tmpdir.mkdir() + self.sentinel = self.root / "sentinel" + self.scan_record = self.root / "scans.txt" + self.curl_record = self.root / "curl.txt" + + self.policy.parent.mkdir(parents=True) + self.write_inputs() + self.git("init", "--quiet") + self.git("add", "-A") + self.git("-c", "user.name=gate", "-c", "user.email=gate@example.invalid", + "commit", "--quiet", "-m", "fixture") + self.head = self.git("rev-parse", "HEAD") + + self.archive = self.root / "fixture.tgz" + with tarfile.open(self.archive, "w:gz") as tar: + tar.add(FAKE_SCANNER, arcname="cargo-audit-x86_64-unknown-linux-musl-v0.22.2/cargo-audit") + + def write_inputs(self): + self.lock.write_text(LOCK) + self.policy.write_text(POLICY) + + def git(self, *args): + return subprocess.run(["git", "-C", str(self.repo), *args], env={**os.environ, **GIT_ENV}, + capture_output=True, text=True, check=True).stdout.strip() + + def run_gate(self, args=None, script=GATE, log="clean-success.log", **fake_env): + """Run ` && touch sentinel` from fresh record and sentinel state.""" + for path in (self.sentinel, self.scan_record, self.curl_record): + path.unlink(missing_ok=True) + env = { + **os.environ, **GIT_ENV, + "PATH": f"{STUBS}{os.pathsep}{os.environ['PATH']}", + "TMPDIR": str(self.tmpdir), + "REAL_SHA256SUM": REAL_SHA256SUM, + "FAKE_ARCHIVE": str(self.archive), + "FAKE_ARCHIVE_SHA256": ARCHIVE_SHA256, + "FAKE_CURL_RECORD": str(self.curl_record), + "FAKE_RECORD": str(self.scan_record), + "FAKE_LOG": str(LOGS / log), + **fake_env, + } + args = [self.head] if args is None else args + command = 'sentinel="$1" gate="$2"; shift 2; "$gate" "$@" && touch "$sentinel"' + return subprocess.run(["bash", "-c", command, "run-gate", str(self.sentinel), str(script), *args], + cwd=self.repo, env=env, capture_output=True, text=True, timeout=120) + + def assert_blocked(self, proc, message, rc=1): + output = proc.stdout + proc.stderr + self.assertEqual(proc.returncode, rc, output) + self.assertFalse(self.sentinel.exists(), "gate failure reached the sentinel") + self.assertIn(message, output) + + def test_clean_scan_passes_with_pinned_identity_and_invocation(self): + proc = self.run_gate() + out = proc.stdout + self.assertEqual(proc.returncode, 0, out + proc.stderr) + self.assertTrue(self.sentinel.exists()) + self.assertIn("cargo-audit release gate: result: PASS", out) + for line in ( + f"expected-sha: {self.head}", + f"head-sha: {self.head}", + f"cargo-lock-sha256: {hashlib.sha256(LOCK.encode()).hexdigest()}", + f"audit-toml-sha256: {hashlib.sha256(POLICY.encode()).hexdigest()}", + f"scanner-version: {SCANNER_VERSION}", + f"advisory-db-origin: {TRUSTED_DB}", + "----- begin cargo-audit output -----", + f"Scanning {self.lock} for vulnerabilities (1 crate dependencies)", + "----- end cargo-audit output -----", + ): + self.assertIn(line, out) + self.assertRegex(out, r"(?m)^advisory-db-revision: [0-9a-f]{40}$") + + [scan] = read_records(self.scan_record) + self.assertEqual(scan["arg"], [ + "audit", "--file", str(self.lock), "--db", scan["db"], "--url", TRUSTED_DB, + "--format", "terminal", "--color", "never", + ]) + [curl] = read_records(self.curl_record) + pairs = list(zip(curl["arg"], curl["arg"][1:])) + for pair in (("--proto", "=https"), ("--connect-timeout", "15"), ("--max-time", "180")): + self.assertIn(pair, pairs) + self.assertEqual(curl["arg"][-1], ARCHIVE_URL) + + def test_installation_failures_block_before_scan(self): + corrupt = self.root / "corrupt.tgz" + corrupt.write_text("not a gzip archive\n") + cases = [ + ({"FAKE_CURL_FAIL": "1"}, "failed to download the pinned cargo-audit archive"), + ({"FAKE_ARCHIVE_SHA256": "0" * 64}, "pinned cargo-audit archive checksum mismatch"), + ({"FAKE_ARCHIVE": str(corrupt)}, "failed to extract the pinned cargo-audit archive"), + ({"FAKE_VERSION": "cargo-audit-audit 0.22.1"}, "unexpected cargo-audit version"), + ({"FAKE_VERSION": "cargo-audit-audit 0.22.2 extra"}, "unexpected cargo-audit version"), + ({"FAKE_CARGO_FAIL": "1"}, "cargo -V failed"), + ] + for fake_env, message in cases: + with self.subTest(fake_env=fake_env): + self.assert_blocked(self.run_gate(**fake_env), message, rc=2 if "FAKE_CARGO_FAIL" in fake_env else 1) + self.assertEqual(read_records(self.scan_record), []) + + def test_source_identity_failures_block_before_scan(self): + cases = [ + ("a" * 40, {}, "does not match expected release commit"), + (None, {self.lock: LOCK + "# tampered\n"}, "working-tree Cargo.lock does not match"), + (None, {self.policy: '[advisories]\nignore = ["RUSTSEC-0000-0000"]\n'}, + "working-tree .cargo/audit.toml does not match"), + (None, {self.lock: None}, "missing Cargo.lock at repository root"), + (None, {self.policy: None}, "missing .cargo/audit.toml at repository root"), + ] + for sha, edits, message in cases: + with self.subTest(message=message): + for path, text in edits.items(): + path.unlink() if text is None else path.write_text(text) + try: + self.assert_blocked(self.run_gate(args=[sha] if sha else None), message) + self.assertEqual(read_records(self.scan_record), []) + finally: + self.write_inputs() + + def test_usage_errors_exit_2_before_scan(self): + for args in ([], [self.head, self.head], ["not-a-commit"], [self.head.upper()]): + with self.subTest(args=args): + proc = self.run_gate(args=args) + self.assert_blocked(proc, "cargo-audit release gate: ERROR:", rc=2) + self.assertFalse(self.curl_record.exists() or self.scan_record.exists()) + + def test_scanner_verdicts(self): + fetch_failure = {"log": os.devnull, "FAKE_RC": "1", "FAKE_NO_DB": "1"} + cases = [ + ({"log": "warning-only-success.log"}, 0, "result: PASS"), + ({"log": "vulnerable-failure.log", "FAKE_RC": "1"}, 1, "scanner exited 1"), + ({"log": "index-error-rc0.log"}, 1, "incomplete crates.io index check"), + ({"log": "quiet-output-rc0.log"}, 1, "missing the advisory database fetch progress line"), + ({"log": "disabled-yanked-rc0.log"}, 1, "missing the crates.io index update progress line"), + ({"log": "disabled-fetch-rc0.log"}, 1, "missing the advisory database fetch progress line"), + (fetch_failure, 1, "advisory-db-origin: unavailable"), + ({"FAKE_ORIGIN": "https://example.invalid/advisory-db.git"}, 1, "unexpected advisory database origin"), + ({"FAKE_SCAN_PATH": "Cargo.lock"}, 1, f"scanned 'Cargo.lock' but the gate expected '{self.lock}'"), + ({"FAKE_RC": "124"}, 124, "scanner exited 124; release blocked"), + ({"FAKE_RC": "137"}, 137, "scanner exited 137; release blocked"), + ] + for fake_env, rc, message in cases: + with self.subTest(fake_env=fake_env): + proc = self.run_gate(**fake_env) + if rc == 0: + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + self.assertTrue(self.sentinel.exists()) + self.assertIn(message, proc.stdout) + else: + self.assert_blocked(proc, message, rc=rc) + self.assertEqual(len(read_records(self.scan_record)), 1) + + def test_retry_after_refresh_failure_uses_fresh_private_state(self): + warm_db = self.tmpdir / "rustsec-db" + warm_home = self.root / "warm-cargo-home" + for warm in (warm_db, warm_home): + (warm / "registry").mkdir(parents=True) + (warm / "registry" / "cached").write_text("warm\n") + warm_env = {"CARGO_HOME": str(warm_home)} + + failed = self.run_gate(log=os.devnull, FAKE_RC="1", FAKE_NO_DB="1", **warm_env) + self.assert_blocked(failed, "scanner exited 1") + scans = read_records(self.scan_record) + passed = self.run_gate(**warm_env) + self.assertEqual(passed.returncode, 0, passed.stdout + passed.stderr) + scans += read_records(self.scan_record) + + self.assertEqual(len(scans), 2, scans) + private = f"{self.tmpdir}/cargo-audit-release-gate." + for scan in scans: + self.assertTrue(scan["db"].startswith(private), scan) + self.assertTrue(scan["home"].startswith(private), scan) + self.assertEqual((scan["db_existed"], scan["home_empty"]), ("no", "yes"), scan) + self.assertNotIn(scan["db"], (str(warm_db), str(warm_home))) + self.assertNotIn(scan["home"], (str(warm_db), str(warm_home))) + self.assertNotEqual(scans[0]["db"], scans[1]["db"]) + self.assertNotEqual(scans[0]["home"], scans[1]["home"]) + # Each run removed its private directory; the warm state is untouched. + self.assertEqual([p.name for p in self.tmpdir.iterdir()], ["rustsec-db"]) + self.assertEqual((warm_home / "registry" / "cached").read_text(), "warm\n") + + def test_unexpected_guard_status_fails_closed(self): + copies = self.root / "scripts" + copies.mkdir() + gate_copy = copies / GATE.name + shutil.copy(GATE, gate_copy) + guard_copy = copies / "cargo-audit-report-guard.sh" + guard_copy.write_text("#!/usr/bin/env bash\nexit 127\n") + guard_copy.chmod(0o755) + self.assert_blocked(self.run_gate(script=gate_copy), "report guard returned an unexpected status (127)") + + def test_static_commands_keep_pins_and_bounds(self): + commands = logical_commands(GATE.read_text()) + tokens = [token for command in commands for token in command] + + assignments = {c[1] for c in commands if len(c) == 2 and c[0] == "readonly"} + for pin in (f"ARCHIVE_URL={ARCHIVE_URL}", f"ARCHIVE_SHA256={ARCHIVE_SHA256}", + f"SCANNER_VERSION_LINE={SCANNER_VERSION}", f"ADVISORY_DB_URL={TRUSTED_DB}"): + self.assertIn(pin, assignments) + + [curl] = [c for c in commands if "curl" in c and any("$ARCHIVE_URL" in t for t in c)] + curl_pairs = list(zip(curl, curl[1:])) + for flag, value in (("--proto", "=https"), ("--connect-timeout", "15"), ("--max-time", "180")): + self.assertEqual(curl.count(flag), 1, curl) + self.assertIn((flag, value), curl_pairs) + + [scan] = [c for c in commands if "timeout" in c and "audit" in c] + start = scan.index("timeout") + self.assertEqual(scan[start:start + 5], ["timeout", "--kill-after=10s", "5m", "$scanner", "audit"]) + scan_pairs = list(zip(scan, scan[1:])) + for pair in (("--file", "$lockfile"), ("--db", "$db"), ("--url", "$ADVISORY_DB_URL"), + ("--format", "terminal"), ("--color", "never")): + self.assertIn(pair, scan_pairs) + + forbidden = re.compile(r"^--(no-fetch|stale|ignore|deny)") + self.assertEqual([t for t in tokens if forbidden.match(t)], []) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/audit-release-gate/test_report_guard.py b/scripts/tests/audit-release-gate/test_report_guard.py new file mode 100644 index 00000000..b378a65c --- /dev/null +++ b/scripts/tests/audit-release-gate/test_report_guard.py @@ -0,0 +1,131 @@ +"""Offline tests for scripts/cargo-audit-report-guard.sh. + +The guard is pure: it takes a captured cargo-audit terminal log, the numeric +scanner exit code, and the lockfile path the gate expected the scan to cover, and +returns a verdict. None of these tests touch the network or install anything. + +Provenance: the logs under fixtures/logs are captured runtime output from the +real pinned cargo-audit 0.22.2 binary (see README.md), recorded +with a relative `--file Cargo.lock`, so `Cargo.lock` is their expected path. The +SYNTHETIC variants are contract fixtures derived in memory from the clean log to +isolate single rejection reasons; they are not scanner output. The runtime proof +for the absolute-path Scanning line is the opt-in network_smoke.py suite. +""" +import subprocess +import tempfile +import unittest +from pathlib import Path + +HERE = Path(__file__).resolve().parent +GUARD = HERE.parents[1] / "cargo-audit-report-guard.sh" +LOGS = HERE / "fixtures" / "logs" +CLEAN_LOG = LOGS / "clean-success.log" +CLEAN = CLEAN_LOG.read_text() +CAPTURED_LOCKFILE = "Cargo.lock" + +# (captured log, scanner exit code it was captured with, expected guard exit code) +CAPTURED = [ + ("clean-success", 0, 0), + ("warning-only-success", 0, 0), + ("vulnerable-failure", 1, 1), + ("index-error-rc0", 0, 1), + ("quiet-output-rc0", 0, 1), + ("disabled-yanked-rc0", 0, 1), + ("disabled-fetch-rc0", 0, 1), +] + + +def without(marker): + kept = [line for line in CLEAN.splitlines() if marker not in line] + if len(kept) == len(CLEAN.splitlines()): + raise LookupError(f"clean log has no '{marker}' line") + return "\n".join(kept) + "\n" + + +def replaced(old, new): + if old not in CLEAN: + raise LookupError(f"clean log does not contain '{old}'") + return CLEAN.replace(old, new) + + +def reordered(): + fetch, loaded, *rest = CLEAN.splitlines() + if "Fetching" not in fetch or "Loaded" not in loaded: + raise LookupError("clean log no longer starts with Fetching then Loaded") + return "\n".join([loaded, fetch, *rest]) + "\n" + + +ABSOLUTE_SCAN = "Scanning /tmp/absolute/path/Cargo.lock for vulnerabilities" + +# (name, log text, expected lockfile, expected guard exit code); scanner exit is 0. +SYNTHETIC = [ + *[(f"missing-{m}", without(m), CAPTURED_LOCKFILE, 1) for m in ("Fetching", "Loaded", "Updating", "Scanning")], + ("update-index-diagnostic", CLEAN + "warning: couldn't update crates.io index: fixture\n", CAPTURED_LOCKFILE, 1), + ("open-index-diagnostic", CLEAN + "warning: couldn't open crates.io index: fixture\n", CAPTURED_LOCKFILE, 1), + ("yanked-diagnostic", CLEAN + "error: couldn't check if the package is yanked: fixture\n", CAPTURED_LOCKFILE, 1), + ("zero-advisories", replaced("Loaded 1380 security", "Loaded 0 security"), CAPTURED_LOCKFILE, 1), + ("missing-advisory-count", replaced("Loaded 1380 security", "Loaded some security"), CAPTURED_LOCKFILE, 1), + ("zero-dependencies", replaced("(1 crate dependencies)", "(0 crate dependencies)"), CAPTURED_LOCKFILE, 1), + ("missing-dependency-count", replaced("(1 crate", "(several crate"), CAPTURED_LOCKFILE, 1), + ("empty-log", "", CAPTURED_LOCKFILE, 1), + ("bare-marker-words", "Fetching x\nLoaded a note\nUpdating x\nScanning x\n", CAPTURED_LOCKFILE, 1), + ("fetch-after-loaded", reordered(), CAPTURED_LOCKFILE, 1), + ("exact-wrong-path", CLEAN, "/expected/elsewhere/Cargo.lock", 1), + ("suffix-wrong-path", replaced("Scanning Cargo.lock", "Scanning /a/b/Cargo.lock"), "/b/Cargo.lock", 1), + ("absolute-valid-path", replaced("Scanning Cargo.lock for vulnerabilities", ABSOLUTE_SCAN), + "/tmp/absolute/path/Cargo.lock", 0), +] + +# (name, argv after the guard path); every case is a usage error (exit 2). +USAGE_ERRORS = [ + *[(f"status-{rc or 'empty'}", [str(CLEAN_LOG), rc, CAPTURED_LOCKFILE]) for rc in ("abc", "", "-1", "0x0", "1.0")], + ("missing-log", [str(LOGS / "does-not-exist.log"), "0", CAPTURED_LOCKFILE]), + ("empty-expected-path", [str(CLEAN_LOG), "0", ""]), + ("zero-args", []), + ("one-arg", [str(CLEAN_LOG)]), + ("two-args", [str(CLEAN_LOG), "0"]), + ("four-args", ["a", "b", "c", "d"]), +] + + +def guard(*args): + return subprocess.run(["bash", str(GUARD), *map(str, args)], capture_output=True, text=True, check=False) + + +class ReportGuardTests(unittest.TestCase): + def assert_exit(self, result, expected): + self.assertEqual(result.returncode, expected, result.stdout + result.stderr) + + def test_captured_logs(self): + for name, scanner_rc, expected in CAPTURED: + with self.subTest(name): + log = LOGS / f"{name}.log" + if name == "warning-only-success": + self.assertIn("allowed warning found", log.read_text()) + self.assert_exit(guard(log, scanner_rc, CAPTURED_LOCKFILE), expected) + + def test_synthetic_variants(self): + with tempfile.TemporaryDirectory() as tmp: + for name, text, expected_lockfile, expected in SYNTHETIC: + with self.subTest(name): + log = Path(tmp) / f"{name}.log" + log.write_text(text) + self.assert_exit(guard(log, 0, expected_lockfile), expected) + + def test_nonzero_scanner_status_rejects_clean_log(self): + # The exit code is authoritative: a clean log cannot rescue a failed scan, + # including the 124/137 statuses of the bounded scanner timeout. + for rc in ("1", "2", "124", "137"): + with self.subTest(rc): + result = guard(CLEAN_LOG, rc, CAPTURED_LOCKFILE) + self.assert_exit(result, 1) + self.assertIn("release blocked", result.stderr) + + def test_usage_errors(self): + for name, args in USAGE_ERRORS: + with self.subTest(name): + self.assert_exit(guard(*args), 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/audit-release-gate/test_workflows.py b/scripts/tests/audit-release-gate/test_workflows.py new file mode 100644 index 00000000..a7a0f88e --- /dev/null +++ b/scripts/tests/audit-release-gate/test_workflows.py @@ -0,0 +1,252 @@ +"""Offline structural tests for the release-gate workflows. + +These tests parse the real workflow YAML and check that the strict cargo-audit +gate runs unconditionally inside each protected job, before that job's first +publication effect, against the exact source revision that produced the +artifacts. One table of negative mutations proves the checks reject +representative regressions. Expected pins, refs, conditions, and commands are +literals reviewed together with the workflows, never values read from them. + +PyYAML is a test-only dependency. YAML 1.1 parses the `on:` key as boolean True, +so `load()` stores the trigger block back under "on". +""" +import shlex +import unittest +from pathlib import Path + +import yaml + +WORKFLOWS = Path(__file__).resolve().parents[3] / ".github" / "workflows" +PUBLISH = "publish.yml" +RELEASE = "release-datadog-serverless-compat.yml" +BUILD = "build-datadog-serverless-compat.yml" + +CHECKOUT = "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" +RUST_SETUP = "actions-rust-lang/setup-rust-toolchain@9399c7bb15d4c7d47b27263d024f0a4978346ba4" +GH_RELEASE = "softprops/action-gh-release@da05d552573ad5aba039eaac05058a918a7bf631" +PINS = {pin.split("@")[0]: pin for pin in (CHECKOUT, RUST_SETUP, GH_RELEASE)} +SOURCE_REF = "${{ github.sha }}" +GATE_COMMAND = './scripts/cargo-audit-release-gate.sh "$GITHUB_SHA"' +BUILD_ONLY = "${{ inputs.publish == 'Build and Publish' }}" +CROSS_RUN_SELECTORS = {"run-id", "repository", "github-token"} +SHELL_SEPARATORS = {"&&", "||", ";", "|"} + + +def load(name): + doc = yaml.safe_load((WORKFLOWS / name).read_text(encoding="utf-8")) + if True in doc: + doc["on"] = doc.pop(True) + return doc + + +def run_of(step): + return str(step.get("run", "")) + + +def action_of(step): + return str(step.get("uses", "")).split("@")[0] + + +def runs(text): + return lambda step: text in run_of(step) + + +def using(action): + return lambda step: action_of(step) == action + + +def first(steps, predicate): + return next((index for index, step in enumerate(steps) if predicate(step)), None) + + +def withs(job, action): + return [step.get("with") or {} for step in job.get("steps", []) if action_of(step) == action] + + +def shell_commands(run): + """Yield the argv of each shell command in a run block, skipping comments.""" + for line in run.replace("\\\n", " ").splitlines(): + argv = [] + for token in shlex.split(line, comments=True) + [";"]: + if token in SHELL_SEPARATORS: + yield argv + argv = [] + else: + argv.append(token) + + +class WorkflowTests(unittest.TestCase): + def check_common(self, doc, where): + for name, job in doc["jobs"].items(): + for item in (job, *job.get("steps", [])): + self.assertNotIn("continue-on-error", item, f"{where}/{name} ignores errors") + action = action_of(item) + if action in PINS: + self.assertEqual(item["uses"], PINS[action], f"{where}: {action} pin changed") + if action == "actions/download-artifact": + selectors = CROSS_RUN_SELECTORS & set(item.get("with") or {}) + self.assertFalse(selectors, f"{where}: cross-run artifact download") + + def check_gated_job(self, job, is_effect, where): + steps = job["steps"] + gate = first(steps, runs("cargo-audit-release-gate.sh")) + self.assertIsNotNone(gate, f"{where}: gate missing") + allowed = {"name", "run", "working-directory"} + self.assertLessEqual(set(steps[gate]), allowed, f"{where}: gate must be unconditional and unmasked") + self.assertEqual(run_of(steps[gate]).strip(), GATE_COMMAND, f"{where}: gate command changed") + self.assertEqual(steps[gate].get("working-directory"), "source", f"{where}: gate directory") + + checkout = first(steps, using("actions/checkout")) + rust = first(steps, using("actions-rust-lang/setup-rust-toolchain")) + self.assertIsNotNone(checkout, f"{where}: source checkout missing") + self.assertIsNotNone(rust, f"{where}: Rust setup missing") + source = steps[checkout].get("with") or {} + self.assertEqual((source.get("ref"), source.get("path")), (SOURCE_REF, "source"), f"{where}: source checkout") + self.assertIs((steps[rust].get("with") or {}).get("cache"), False, f"{where}: Rust setup cache") + self.assertLess(max(checkout, rust), gate, f"{where}: checkout and Rust setup must precede the gate") + + effect = first(steps, is_effect) + self.assertIsNotNone(effect, f"{where}: publication effect missing") + self.assertLess(gate, effect, f"{where}: gate must precede the first publication effect") + for later in steps[gate:]: + self.assertNotIn("if", later, f"{where}: step conditional after the gate") + + def check_publish(self, doc): + self.check_common(doc, "publish") + self.assertEqual(doc["permissions"], {"contents": "read", "id-token": "write"}, "publish permissions") + inputs = doc["on"]["workflow_dispatch"]["inputs"] + self.assertEqual(set(inputs), {"publish", "version"}, "publish inputs changed") + self.assertEqual(inputs["publish"]["options"], ["Build", "Build and Publish"], "publish options") + + package, publish = doc["jobs"]["package"], doc["jobs"]["publish"] + self.assertEqual(publish.get("environment"), "Production", "publish lost Production approval") + self.assertEqual(publish.get("needs"), "package", "publish must need package") + self.assertEqual(publish.get("if"), BUILD_ONLY, "publish lost the build-only condition") + package_refs = [w.get("ref") for w in withs(package, "actions/checkout")] + self.assertEqual(package_refs, [SOURCE_REF], "package checkout ref") + self.assertIn("npm-packages", [w.get("name") for w in withs(package, "actions/upload-artifact")]) + self.assertIn("npm-packages", [w.get("name") for w in withs(publish, "actions/download-artifact")]) + self.check_gated_job(publish, runs("npm publish"), "publish") + + def check_release(self, doc): + self.check_common(doc, "release") + self.assertEqual(doc["permissions"], {}, "release top-level permissions must stay empty") + release = doc["jobs"]["release"] + # Reviewed flow: the strict gate plus a draft release is the release + # check, so no extra protected environment approval is introduced. + self.assertNotIn("environment", release, "release gained a protected environment") + self.assertNotIn("if", release, "release job became conditional") + self.assertEqual(release.get("needs"), "build", "release must need build") + self.assertEqual(release.get("permissions"), {"contents": "write"}, "release job permissions") + + def is_effect(step): + return run_of(step).lstrip().startswith("zip ") or action_of(step) == "softprops/action-gh-release" + + self.check_gated_job(release, is_effect, "release") + drafts = [w.get("draft") is True for w in withs(release, "softprops/action-gh-release")] + self.assertEqual(drafts, [True], "release must stay a single draft") + + def check_build(self, doc): + self.check_common(doc, "builder") + refs = [w.get("ref") for job in doc["jobs"].values() for w in withs(job, "actions/checkout")] + self.assertEqual(refs, [SOURCE_REF, SOURCE_REF], "builder checkouts must pin the run commit") + run_blocks = [run_of(step) for job in doc["jobs"].values() for step in job.get("steps", [])] + self.assertFalse([run for run in run_blocks if "audit" in run], "builder must not audit") + builds = [argv for run in run_blocks for argv in shell_commands(run) if argv[:2] == ["cargo", "build"]] + self.assertEqual(len(builds), 5, f"expected 5 release builds, found {builds}") + for argv in builds: + self.assertLessEqual({"--release", "--locked"}, set(argv), f"unlocked build: {argv}") + + def test_real_publish_workflow(self): + self.check_publish(load(PUBLISH)) + + def test_real_release_workflow(self): + self.check_release(load(RELEASE)) + + def test_real_build_workflow(self): + self.check_build(load(BUILD)) + + def test_mutations_are_rejected(self): + checks = {PUBLISH: self.check_publish, RELEASE: self.check_release, BUILD: self.check_build} + for context, name, mutate, reason in MUTATIONS: + with self.subTest(context): + doc = load(name) + mutate(doc) # LookupError or KeyError here means the mutation target moved. + with self.assertRaisesRegex(AssertionError, reason): + checks[name](doc) + + +# Mutation helpers fail loudly when their target is absent or already holds the +# requested value, so a case cannot pass by mutating nothing. +def step(doc, job, predicate): + matches = [s for s in doc["jobs"][job]["steps"] if predicate(s)] + if len(matches) != 1: + raise LookupError(f"mutation target in job '{job}' matched {len(matches)} steps") + return matches[0] + + +def put(target, key, value): + if target.get(key) == value: + raise LookupError(f"'{key}' already equals {value!r}") + target[key] = value + + +def edit_run(target, old, new): + if old not in run_of(target): + raise LookupError(f"'{old}' not found in run block") + target["run"] = run_of(target).replace(old, new) + + +def move_to_end(doc, job, predicate): + steps = doc["jobs"][job]["steps"] + steps.append(steps.pop(steps.index(step(doc, job, predicate)))) + + +GATE = runs("cargo-audit-release-gate.sh") +BUILDER = "build-datadog-serverless-compat" +MACOS_BUILD = runs("datadog-serverless-compat --locked") + +# (context, workflow, mutation, expected rejection reason) +MUTATIONS = [ + ("publish gate removed", PUBLISH, + lambda d: d["jobs"]["publish"]["steps"].remove(step(d, "publish", GATE)), "gate missing"), + ("publish gate moved after npm publish", PUBLISH, + lambda d: move_to_end(d, "publish", GATE), "gate must precede"), + ("release gate moved after the zip and draft release", RELEASE, + lambda d: move_to_end(d, "release", GATE), "gate must precede"), + ("publish gate masked with || true", PUBLISH, + lambda d: edit_run(step(d, "publish", GATE), '"$GITHUB_SHA"', '"$GITHUB_SHA" || true'), "gate command"), + ("publish gate skipped with if: false", PUBLISH, + lambda d: put(step(d, "publish", GATE), "if", False), "unconditional and unmasked"), + ("release gate ignores its own failure", RELEASE, + lambda d: put(step(d, "release", GATE), "continue-on-error", True), "ignores errors"), + ("publish gate no longer bound to the run commit", PUBLISH, + lambda d: edit_run(step(d, "publish", GATE), ' "$GITHUB_SHA"', ""), "gate command"), + ("publish source checkout floats to main", PUBLISH, + lambda d: put(step(d, "publish", using("actions/checkout"))["with"], "ref", "main"), "source checkout"), + ("builder setup checkout floats to main", BUILD, + lambda d: put(step(d, "setup", using("actions/checkout"))["with"], "ref", "main"), "builder checkouts"), + ("publish downloads npm packages from another run", PUBLISH, + lambda d: put(step(d, "publish", using("actions/download-artifact"))["with"], "run-id", "1234"), + "cross-run artifact"), + ("publish loses Production approval", PUBLISH, + lambda d: d["jobs"]["publish"].pop("environment"), "Production approval"), + ("publish also runs on Build-only dispatches", PUBLISH, + lambda d: put(d["jobs"]["publish"], "if", "${{ always() }}"), "build-only condition"), + ("npm publish step made conditional", PUBLISH, + lambda d: put(step(d, "publish", runs("npm publish")), "if", "${{ always() }}"), "conditional after the gate"), + ("draft release becomes public", RELEASE, + lambda d: put(step(d, "release", using("softprops/action-gh-release"))["with"], "draft", False), + "single draft"), + ("i686 backslash-continued build drops --locked", BUILD, + lambda d: edit_run(step(d, BUILDER, runs("--target i686-pc-windows-msvc")), "--locked", ""), + "unlocked build"), + ("macOS build keeps --locked only inside a shell comment", BUILD, + lambda d: edit_run(step(d, BUILDER, MACOS_BUILD), "--locked", "# --locked"), "unlocked build"), + ("macOS build commented out", BUILD, + lambda d: edit_run(step(d, BUILDER, MACOS_BUILD), "cargo", "# cargo"), "expected 5 release builds"), +] + + +if __name__ == "__main__": + unittest.main()