From d2f9f6c7e33f1ab8e7d2f3a707c4ef3fa3326d37 Mon Sep 17 00:00:00 2001 From: Lucas Pimentel Date: Tue, 27 Oct 2026 22:03:28 -0400 Subject: [PATCH 1/3] Gate SCL publication on dependency scans --- .../workflows/audit-release-gate-tests.yml | 52 ++ .../build-datadog-serverless-compat.yml | 15 +- .github/workflows/publish.yml | 14 + .../release-datadog-serverless-compat.yml | 12 + scripts/cargo-audit-release-gate.sh | 202 ++++++ scripts/cargo-audit-report-guard.sh | 121 ++++ scripts/tests/audit-release-gate/README.md | 84 +++ .../fixtures/fake-scanner/cargo-audit | 155 +++++ .../fixtures/logs/README.md | 46 ++ .../fixtures/logs/clean-success.log | 4 + .../fixtures/logs/disabled-fetch-rc0.log | 2 + .../fixtures/logs/disabled-yanked-rc0.log | 3 + .../fixtures/logs/index-error-rc0.log | 5 + .../fixtures/logs/malformed-lock-rc2.log | 16 + .../fixtures/logs/missing-lock-rc2.log | 7 + .../fixtures/logs/quiet-output-rc0.log | 0 .../fixtures/logs/vulnerable-failure.log | 62 ++ .../fixtures/logs/warning-only-success.log | 13 + .../audit-release-gate/fixtures/stubs/cargo | 19 + .../audit-release-gate/fixtures/stubs/curl | 38 + .../fixtures/stubs/sha256sum | 24 + .../audit-release-gate/fixtures/stubs/timeout | 33 + .../tests/audit-release-gate/network_smoke.py | 198 ++++++ .../audit-release-gate/test_release_gate.py | 523 ++++++++++++++ .../audit-release-gate/test_report_guard.py | 257 +++++++ .../audit-release-gate/test_workflows.py | 657 ++++++++++++++++++ 26 files changed, 2557 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/audit-release-gate-tests.yml create mode 100755 scripts/cargo-audit-release-gate.sh create mode 100755 scripts/cargo-audit-report-guard.sh create mode 100644 scripts/tests/audit-release-gate/README.md create mode 100755 scripts/tests/audit-release-gate/fixtures/fake-scanner/cargo-audit create mode 100644 scripts/tests/audit-release-gate/fixtures/logs/README.md create mode 100644 scripts/tests/audit-release-gate/fixtures/logs/clean-success.log create mode 100644 scripts/tests/audit-release-gate/fixtures/logs/disabled-fetch-rc0.log create mode 100644 scripts/tests/audit-release-gate/fixtures/logs/disabled-yanked-rc0.log create mode 100644 scripts/tests/audit-release-gate/fixtures/logs/index-error-rc0.log create mode 100644 scripts/tests/audit-release-gate/fixtures/logs/malformed-lock-rc2.log create mode 100644 scripts/tests/audit-release-gate/fixtures/logs/missing-lock-rc2.log create mode 100644 scripts/tests/audit-release-gate/fixtures/logs/quiet-output-rc0.log create mode 100644 scripts/tests/audit-release-gate/fixtures/logs/vulnerable-failure.log create mode 100644 scripts/tests/audit-release-gate/fixtures/logs/warning-only-success.log create mode 100755 scripts/tests/audit-release-gate/fixtures/stubs/cargo create mode 100755 scripts/tests/audit-release-gate/fixtures/stubs/curl create mode 100755 scripts/tests/audit-release-gate/fixtures/stubs/sha256sum create mode 100755 scripts/tests/audit-release-gate/fixtures/stubs/timeout create mode 100644 scripts/tests/audit-release-gate/network_smoke.py create mode 100644 scripts/tests/audit-release-gate/test_release_gate.py create mode 100644 scripts/tests/audit-release-gate/test_report_guard.py create mode 100644 scripts/tests/audit-release-gate/test_workflows.py diff --git a/.github/workflows/audit-release-gate-tests.yml b/.github/workflows/audit-release-gate-tests.yml new file mode 100644 index 00000000..a1baaa8f --- /dev/null +++ b/.github/workflows/audit-release-gate-tests.yml @@ -0,0 +1,52 @@ +name: Audit Release Gate Tests + +on: + pull_request: + paths: + - '.github/workflows/audit-release-gate-tests.yml' + - '.github/workflows/publish.yml' + - '.github/workflows/release-datadog-serverless-compat.yml' + - '.github/workflows/build-datadog-serverless-compat.yml' + - 'Cargo.lock' + - '.cargo/audit.toml' + - 'scripts/cargo-audit-release-gate.sh' + - 'scripts/cargo-audit-report-guard.sh' + - 'scripts/tests/audit-release-gate/**' + workflow_dispatch: + inputs: + network_smoke: + description: 'Also run the opt-in real-scanner network smoke suite' + required: false + type: boolean + default: false + +# Read-only: this check never publishes, signs, or uploads anything. +permissions: + contents: read + +jobs: + offline: + name: Offline gate tests + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 + with: + python-version: '3.12' + - name: Install test-only dependencies + run: python3 -m pip install --disable-pip-version-check 'PyYAML==6.0.2' + - name: Run deterministic offline gate tests + run: | + python3 -m unittest discover -v -s scripts/tests/audit-release-gate -p 'test_*.py' + + network-smoke: + name: Opt-in real-scanner smoke + if: ${{ github.event_name == 'workflow_dispatch' && inputs.network_smoke }} + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions-rust-lang/setup-rust-toolchain@9399c7bb15d4c7d47b27263d024f0a4978346ba4 # v1.11.0 + with: + cache: false + - name: Run real scanner download, checksum, refresh, and scan + run: python3 scripts/tests/audit-release-gate/network_smoke.py diff --git a/.github/workflows/build-datadog-serverless-compat.yml b/.github/workflows/build-datadog-serverless-compat.yml index 070938fe..2b2aea22 100644 --- a/.github/workflows/build-datadog-serverless-compat.yml +++ b/.github/workflows/build-datadog-serverless-compat.yml @@ -17,6 +17,8 @@ jobs: RUSTC_WRAPPER: "sccache" steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ github.sha }} - uses: actions-rust-lang/setup-rust-toolchain@9399c7bb15d4c7d47b27263d024f0a4978346ba4 # v1.11.0 with: cache: false @@ -27,6 +29,8 @@ jobs: runs-on: ${{ inputs.runner }} steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ github.sha }} - name: Install Protoc Binary shell: bash run: chmod +x ./scripts/install-protoc.sh && ./scripts/install-protoc.sh $HOME @@ -35,7 +39,7 @@ jobs: run: | sudo apt-get update rustup target add x86_64-unknown-linux-musl && sudo apt-get install -y musl-tools - cargo build --release -p datadog-serverless-compat --target x86_64-unknown-linux-musl + cargo build --release -p datadog-serverless-compat --target x86_64-unknown-linux-musl --locked - if: ${{ inputs.runner == 'ubuntu-24.04' }} uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # 4.6.2 with: @@ -47,7 +51,7 @@ jobs: run: | sudo apt-get update rustup target add aarch64-unknown-linux-musl && sudo apt-get install -y musl-tools - cargo build --release -p datadog-serverless-compat --target aarch64-unknown-linux-musl + cargo build --release -p datadog-serverless-compat --target aarch64-unknown-linux-musl --locked - if: ${{ inputs.runner == 'ubuntu-24.04-arm' }} uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # 4.6.2 with: @@ -56,7 +60,7 @@ jobs: retention-days: 3 - if: ${{ inputs.runner == 'windows-2022' }} shell: bash - run: cargo build --release -p datadog-serverless-compat --features windows-pipes,windows-enhanced-metrics + run: cargo build --release -p datadog-serverless-compat --features windows-pipes,windows-enhanced-metrics --locked - if: ${{ inputs.runner == 'windows-2022' }} uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # 4.6.2 with: @@ -69,7 +73,8 @@ jobs: rustup target add i686-pc-windows-msvc cargo build --release -p datadog-serverless-compat \ --target i686-pc-windows-msvc \ - --features windows-pipes,windows-enhanced-metrics + --features windows-pipes,windows-enhanced-metrics \ + --locked - if: ${{ inputs.runner == 'windows-2022' }} uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # 4.6.2 with: @@ -78,7 +83,7 @@ jobs: retention-days: 3 - if: ${{ inputs.runner == 'macos-14' }} shell: bash - run: cargo build --release -p datadog-serverless-compat + run: cargo build --release -p datadog-serverless-compat --locked - if: ${{ inputs.runner == 'macos-14' }} uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # 4.6.2 with: diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 73954695..e011799f 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -17,6 +17,7 @@ on: type: string permissions: + contents: read # Read-only checkout of the audited source revision id-token: write # For OIDC npm publishing with provenance jobs: @@ -35,6 +36,7 @@ jobs: steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: + ref: ${{ github.sha }} sparse-checkout: npm sparse-checkout-cone-mode: false - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # 4.3.0 @@ -109,6 +111,11 @@ jobs: environment: Production needs: package steps: + - name: Check out the audited source revision + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ github.sha }} + path: source - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # 4.3.0 with: name: npm-packages @@ -117,6 +124,13 @@ jobs: with: node-version: "22.x" registry-url: 'https://registry.npmjs.org' + - name: Install Rust toolchain for cargo-audit + uses: actions-rust-lang/setup-rust-toolchain@9399c7bb15d4c7d47b27263d024f0a4978346ba4 # v1.11.0 + with: + cache: false + - name: Strict cargo-audit release gate + working-directory: source + run: ./scripts/cargo-audit-release-gate.sh "$GITHUB_SHA" - name: Publish npm packages run: | npm publish ./npm/datadog-serverless-compat-linux-x64 --provenance --access public diff --git a/.github/workflows/release-datadog-serverless-compat.yml b/.github/workflows/release-datadog-serverless-compat.yml index ae17f253..ae38ab04 100644 --- a/.github/workflows/release-datadog-serverless-compat.yml +++ b/.github/workflows/release-datadog-serverless-compat.yml @@ -19,12 +19,24 @@ jobs: permissions: contents: write steps: + - name: Check out the audited source revision + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ github.sha }} + path: source - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # 4.3.0 with: path: target/bin - run: | chmod +x target/bin/linux-amd64/datadog-serverless-compat chmod +x target/bin/windows-amd64/datadog-serverless-compat.exe + - name: Install Rust toolchain for cargo-audit + uses: actions-rust-lang/setup-rust-toolchain@9399c7bb15d4c7d47b27263d024f0a4978346ba4 # v1.11.0 + with: + cache: false + - name: Strict cargo-audit release gate + working-directory: source + run: ./scripts/cargo-audit-release-gate.sh "$GITHUB_SHA" - run: zip -r datadog-serverless-compat.zip ./* working-directory: target - uses: softprops/action-gh-release@da05d552573ad5aba039eaac05058a918a7bf631 # v2.2.2 diff --git a/scripts/cargo-audit-release-gate.sh b/scripts/cargo-audit-release-gate.sh new file mode 100755 index 00000000..39128793 --- /dev/null +++ b/scripts/cargo-audit-release-gate.sh @@ -0,0 +1,202 @@ +#!/usr/bin/env bash +# +# Strict cargo-audit release gate for datadog-serverless-compat publication. +# +# Usage: scripts/cargo-audit-release-gate.sh +# +# Run this gate inside a protected publication job, immediately before its first +# publication side effect. It performs downloads and a dependency scan only: +# it never publishes, signs, uploads, or pushes anything. +# +# The gate fails closed unless every one of these holds: +# * the repository root is the expected commit (usually $GITHUB_SHA); +# * the working-tree Cargo.lock and .cargo/audit.toml byte-match that commit, +# so the audited lockfile and advisory policy cannot be swapped out; +# * a working Rust toolchain is present, because the pinned prebuilt +# cargo-audit still shells out to `cargo -V`; +# * the pinned official cargo-audit 0.22.2 (Linux x86_64 musl) archive +# downloads (bounded by curl's --connect-timeout 15 and --max-time 180) and +# matches its reviewed SHA-256; +# * the installed binary reports exactly the expected version; +# * a fresh RustSec advisory database and a fresh Cargo cache are created in a +# private temporary directory, so a warm cache can never turn a failed +# advisory refresh into an offline success; +# * the scanner runs under coreutils timeout (--kill-after=10s, 5m) and exits +# zero, the terminal log shows the full Fetching/Loaded/Updating/Scanning +# sequence with nonzero advisory and dependency counts for the expected +# lockfile, and the log carries none of the crates.io index diagnostics the +# pinned binary can otherwise hide behind a zero exit status. See +# scripts/cargo-audit-report-guard.sh. +# +# The archive URL, digest, scanner version, and advisory database URL are fixed +# reviewed constants. This script deliberately honors no environment variable +# that could point it at a different archive, digest, or advisory source, and it +# passes no --no-fetch, --stale, or extra ignore flags. All exceptions come from +# the repository's reviewed .cargo/audit.toml. +set -euo pipefail + +readonly SCANNER_VERSION_LINE="cargo-audit-audit 0.22.2" +readonly ARCHIVE_URL="https://github.com/rustsec/rustsec/releases/download/cargo-audit/v0.22.2/cargo-audit-x86_64-unknown-linux-musl-v0.22.2.tgz" +readonly ARCHIVE_SHA256="7fb9497f8594b389e5fce5ef9b92db08432996895b2e0c5a0167a69ed445c428" +readonly ADVISORY_DB_URL="https://github.com/RustSec/advisory-db.git" + +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +readonly script_dir +readonly GUARD_SCRIPT="${script_dir}/cargo-audit-report-guard.sh" + +die() { + printf 'cargo-audit release gate: ERROR: %s\n' "$*" >&2 + exit 2 +} + +fail() { + printf 'cargo-audit release gate: FAILED: %s\n' "$*" >&2 + exit 1 +} + +if [ "$#" -ne 1 ]; then + die "usage: $0 " +fi + +expected_sha="$1" +if ! [[ "$expected_sha" =~ ^([0-9a-f]{40}|[0-9a-f]{64})$ ]]; then + die "expected commit must be a 40- or 64-character lowercase hex id" +fi + +[ -x "$GUARD_SCRIPT" ] || fail "report guard is missing or not executable: $GUARD_SCRIPT" + +for tool in git cargo curl tar sha256sum cut timeout; do + command -v "$tool" >/dev/null 2>&1 || die "required tool not found on PATH: $tool" +done + +cargo_version="$(cargo -V 2>&1)" || die "cargo -V failed; a working Rust toolchain must be installed before this gate" + +repo_root="$(git rev-parse --show-toplevel 2>/dev/null)" || die "not inside a git work tree" +cd "$repo_root" + +head_sha="$(git rev-parse HEAD)" || die "cannot resolve HEAD" + +lockfile="$repo_root/Cargo.lock" +policy="$repo_root/.cargo/audit.toml" +[ -f "$lockfile" ] || fail "missing Cargo.lock at repository root" +[ -f "$policy" ] || fail "missing .cargo/audit.toml at repository root" + +if [ "$head_sha" != "$expected_sha" ]; then + fail "HEAD ($head_sha) does not match expected release commit ($expected_sha)" +fi + +# Bind the audited inputs to the expected commit, not to the working tree alone. +if ! expected_lock_hash="$(git show "${expected_sha}:Cargo.lock" | sha256sum | cut -d' ' -f1)"; then + fail "cannot read Cargo.lock from expected commit $expected_sha" +fi +actual_lock_hash="$(sha256sum "$lockfile" | cut -d' ' -f1)" +if [ "$expected_lock_hash" != "$actual_lock_hash" ]; then + fail "working-tree Cargo.lock does not match the expected commit" +fi + +if ! expected_policy_hash="$(git show "${expected_sha}:.cargo/audit.toml" | sha256sum | cut -d' ' -f1)"; then + fail "cannot read .cargo/audit.toml from expected commit $expected_sha" +fi +actual_policy_hash="$(sha256sum "$policy" | cut -d' ' -f1)" +if [ "$expected_policy_hash" != "$actual_policy_hash" ]; then + fail "working-tree .cargo/audit.toml does not match the expected commit" +fi + +work="$(mktemp -d "${TMPDIR:-/tmp}/cargo-audit-release-gate.XXXXXXXX")" || die "cannot create a temporary directory" +readonly work +# Single quotes defer expansion to EXIT so the cleanup always targets this run's +# directory without interpolating it into a fragile command string. +trap 'rm -rf -- "$work"' EXIT + +archive="$work/cargo-audit.tgz" +install_dir="$work/install" +mkdir -p "$install_dir" + +if ! curl --fail --silent --show-error --location --proto '=https' \ + --connect-timeout 15 --max-time 180 --output "$archive" "$ARCHIVE_URL"; then + fail "failed to download the pinned cargo-audit archive" +fi + +archive_hash="$(sha256sum "$archive" | cut -d' ' -f1)" +if [ "$archive_hash" != "$ARCHIVE_SHA256" ]; then + fail "pinned cargo-audit archive checksum mismatch" +fi + +if ! tar -xzf "$archive" -C "$install_dir" --strip-components=1; then + fail "failed to extract the pinned cargo-audit archive" +fi + +scanner="$install_dir/cargo-audit" +[ -x "$scanner" ] || fail "extracted archive does not contain an executable cargo-audit" + +scanner_version="$("$scanner" audit --version 2>&1)" || fail "cannot execute the installed cargo-audit" +if [ "$scanner_version" != "$SCANNER_VERSION_LINE" ]; then + fail "unexpected cargo-audit version: '$scanner_version' (expected '$SCANNER_VERSION_LINE')" +fi + +# Fresh advisory database and fresh Cargo cache on every execution. Nothing from +# a previous run, job, or retry is reused, so a stale cache cannot authorize a +# release when the required refresh fails. +cargo_home="$work/cargo-home" +mkdir -p "$cargo_home" +db="$work/rustsec-db" +scan_log="$work/cargo-audit.log" + +# The scan is bounded by coreutils timeout: 5 minutes of runtime, then a 10 +# second grace before SIGKILL. A killed scan exits 124 (SIGTERM) or 137 +# (SIGKILL); both are nonzero and are rejected by the report guard below. +scan_rc=0 +CARGO_HOME="$cargo_home" timeout --kill-after=10s 5m "$scanner" audit \ + --file "$lockfile" \ + --db "$db" \ + --url "$ADVISORY_DB_URL" \ + --format terminal \ + --color never >"$scan_log" 2>&1 || scan_rc=$? + +db_origin="unavailable" +db_revision="unavailable" +if [ -d "$db/.git" ]; then + db_origin="$(git -C "$db" remote get-url origin 2>/dev/null || printf 'unavailable')" + db_revision="$(git -C "$db" rev-parse HEAD 2>/dev/null || printf 'unavailable')" +fi + +printf 'cargo-audit release gate\n' +printf 'expected-sha: %s\n' "$expected_sha" +printf 'head-sha: %s\n' "$head_sha" +printf 'cargo-lock-sha256: %s\n' "$actual_lock_hash" +printf 'audit-toml-sha256: %s\n' "$actual_policy_hash" +printf 'cargo-version: %s\n' "$cargo_version" +printf 'scanner-version: %s\n' "$scanner_version" +printf 'advisory-db-url: %s\n' "$ADVISORY_DB_URL" +printf 'advisory-db-origin: %s\n' "$db_origin" +printf 'advisory-db-revision: %s\n' "$db_revision" +printf 'scanner-exit: %s\n' "$scan_rc" + +printf '%s\n' '----- begin cargo-audit output -----' +if ! cat "$scan_log"; then + fail "cannot read the cargo-audit scan log: $scan_log" +fi +printf '%s\n' '----- end cargo-audit output -----' + +guard_rc=0 +"$GUARD_SCRIPT" "$scan_log" "$scan_rc" "$lockfile" || guard_rc=$? +case "$guard_rc" in + 0 | 1) ;; + *) fail "report guard returned an unexpected status ($guard_rc); treating as failure" ;; +esac + +if [ "$guard_rc" -ne 0 ]; then + printf 'cargo-audit release gate: result: FAIL (scanner-exit=%s)\n' "$scan_rc" >&2 + if [ "$scan_rc" -ne 0 ]; then + # Preserve the scanner's own status for rejected findings and errors. + exit "$scan_rc" + fi + exit 1 +fi + +if [ "$db_origin" != "$ADVISORY_DB_URL" ]; then + fail "unexpected advisory database origin: '$db_origin'" +fi + +printf 'cargo-audit release gate: result: PASS\n' +exit 0 diff --git a/scripts/cargo-audit-report-guard.sh b/scripts/cargo-audit-report-guard.sh new file mode 100755 index 00000000..3052f132 --- /dev/null +++ b/scripts/cargo-audit-report-guard.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash +# +# Pure verdict guard for a cargo-audit release scan. +# +# Usage: scripts/cargo-audit-report-guard.sh +# +# The guard exits 0 only when all of the following hold: +# * the scanner exit code is a non-negative integer and equals zero; +# * the scan log contains the complete Fetching/Loaded/Updating/Scanning +# progress sequence emitted by the pinned cargo-audit 0.22.2 binary, in that +# order, with a nonzero loaded-advisory count and a nonzero crate-dependency +# count for the expected lockfile; +# * the lockfile named on the Scanning line is exactly the expected lockfile +# path the gate passed to the scanner, so a scan of a different or +# substituted file cannot authorize a release; +# * the scan log contains none of the pinned binary's crates.io index +# diagnostics ("couldn't update crates.io index", "couldn't open +# crates.io index", "couldn't check if the package is yanked"). +# +# cargo-audit 0.22.2 can exit zero while hiding an internal crates.io index +# failure behind JSON output. Exit status and JSON alone are therefore not +# enough to authorize a release, which is why the human-readable terminal log +# is parsed here. +# +# The marker shapes this guard requires are pinned to the scanner version's +# terminal output. Structured lines are matched rather than bare marker words, so +# a log that merely mentions "Loaded" or "Scanning" without the substantive +# payload cannot pass. Re-review this guard and its regression tests whenever the +# pinned version changes. +# +# This script performs no network, download, installation, or publication work, +# so it can be exercised entirely offline. +set -euo pipefail + +if [ "$#" -ne 3 ]; then + printf 'cargo-audit report guard: usage: %s \n' "$0" >&2 + exit 2 +fi + +log="$1" +rc="$2" +expected_lockfile="$3" + +reject() { + printf 'cargo-audit report guard: %s\n' "$*" >&2 + exit 1 +} + +if [ -z "$expected_lockfile" ]; then + printf 'cargo-audit report guard: expected lockfile path must not be empty\n' >&2 + exit 2 +fi + +if [ ! -f "$log" ]; then + printf 'cargo-audit report guard: scan log not found: %s\n' "$log" >&2 + exit 2 +fi + +case "$rc" in + '' | *[!0-9]*) + printf 'cargo-audit report guard: scanner exit code is not a non-negative integer: %s\n' "$rc" >&2 + exit 2 + ;; +esac + +if [ "$rc" -ne 0 ]; then + printf 'cargo-audit report guard: scanner exited %s; release blocked\n' "$rc" >&2 + exit 1 +fi + +if grep -Eq "couldn't (update|open) crates\.io index|couldn't check if the package is yanked" "$log"; then + reject "scan log shows an incomplete crates.io index check; release blocked" +fi + +fetch_line=0 +loaded_line=0 +updating_line=0 +scanning_line=0 +scanned_lockfile="" +line_number=0 +while IFS= read -r line || [ -n "$line" ]; do + line_number=$((line_number + 1)) + + if [ "$fetch_line" -eq 0 ] && [[ "$line" =~ ^[[:space:]]*Fetching[[:space:]]+advisory[[:space:]]+database[[:space:]]+from[[:space:]] ]]; then + fetch_line=$line_number + continue + fi + + if [ "$loaded_line" -eq 0 ] && [[ "$line" =~ ^[[:space:]]*Loaded[[:space:]]+([1-9][0-9]*)[[:space:]]+security[[:space:]]+advisories ]]; then + loaded_line=$line_number + continue + fi + + if [ "$updating_line" -eq 0 ] && [[ "$line" =~ ^[[:space:]]*Updating[[:space:]]+crates\.io[[:space:]]+index[[:space:]]*$ ]]; then + updating_line=$line_number + continue + fi + + if [ "$scanning_line" -eq 0 ] && [[ "$line" =~ ^[[:space:]]*Scanning[[:space:]]+(.+)[[:space:]]+for[[:space:]]+vulnerabilities[[:space:]]+\(([1-9][0-9]*)[[:space:]]+crate[[:space:]]+dependencies\)[[:space:]]*$ ]]; then + scanning_line=$line_number + scanned_lockfile="${BASH_REMATCH[1]}" + continue + fi +done <"$log" + +[ "$fetch_line" -ne 0 ] || reject "scan log is missing the advisory database fetch progress line" +[ "$loaded_line" -ne 0 ] || reject "scan log does not report a nonzero count of loaded security advisories" +[ "$updating_line" -ne 0 ] || reject "scan log is missing the crates.io index update progress line" +[ "$scanning_line" -ne 0 ] || reject "scan log does not report scanning a lockfile with a nonzero crate dependency count" + +if [ "$fetch_line" -ge "$loaded_line" ] || + [ "$loaded_line" -ge "$updating_line" ] || + [ "$updating_line" -ge "$scanning_line" ]; then + reject "scan log progress lines are out of order" +fi + +if [ "$scanned_lockfile" != "$expected_lockfile" ]; then + reject "scan log scanned '$scanned_lockfile' but the gate expected '$expected_lockfile'" +fi + +exit 0 diff --git a/scripts/tests/audit-release-gate/README.md b/scripts/tests/audit-release-gate/README.md new file mode 100644 index 00000000..8789ab42 --- /dev/null +++ b/scripts/tests/audit-release-gate/README.md @@ -0,0 +1,84 @@ +# Audit release gate tests + +Tests for the strict cargo-audit release gate in +[`scripts/cargo-audit-release-gate.sh`](../../cargo-audit-release-gate.sh) and its +pure verdict guard +[`scripts/cargo-audit-report-guard.sh`](../../cargo-audit-report-guard.sh), plus +structural checks that the gate is wired into the protected publication jobs +correctly. + +## Offline suite (default) + +Deterministic, no network, no real scanner, no publishers. It runs the real +production helper against temporary git repositories, a fixture scanner archive, +and local PATH test doubles, and it checks the real workflow YAML structure. + +```sh +python3 -m pip install 'PyYAML==6.0.2' +python3 -m unittest discover -v -s scripts/tests/audit-release-gate -p 'test_*.py' +``` + +PyYAML is a test-only dependency used by `test_workflows.py`. + +| File | Coverage | +|---|---| +| `test_report_guard.py` | Guard verdicts for captured real-scanner logs plus synthetic contract fixtures: nonzero advisory/dependency counts, exact expected-lockfile matching, marker order, hidden crates.io diagnostics, exit-code validation, shell sentinel behavior | +| `test_release_gate.py` | Helper control flow: download/checksum/version/prerequisite failures, hidden index errors, missing markers, fetch failure, source and lockfile identity, bounded network and scanner invocations, an inherited warm cache, missing `timeout`, scanner timeout statuses, substituted lockfiles, fresh per-execution directories, retry, guard robustness | +| `test_workflows.py` | Gate placement, exact command, unconditional step, bash shell, and pinning in `publish.yml`, `release-datadog-serverless-compat.yml`, `build-datadog-serverless-compat.yml`, and `audit-release-gate-tests.yml`, plus negative mutations for command masking, conditions, job-level error ignoring, cross-run artifact downloads, and checkout refs | + +The test doubles shape only the helper's inputs (via `PATH` and fixture files). +The production helper reads no bypass environment variable, and the tests add +none. + +### Production time bounds + +The gate bounds every potentially hanging step. These values are production +constants with no environment override, so a bypass cannot shorten them: + +- `curl --connect-timeout 15 --max-time 180` for the pinned archive download. +- `timeout --kill-after=10s 5m audit ...` for the dependency scan. + `timeout` is a required tool; a killed scan exits 124 (SIGTERM) or 137 + (SIGKILL), both of which the guard rejects before the publication sentinel. + +## Opt-in network smoke + +Downloads the real pinned cargo-audit archive, verifies its reviewed SHA-256, +clones a fresh RustSec advisory database, updates the crates.io index, and scans +small fixture lockfiles (clean, vulnerable, informational warning, and an +existing-ignore policy). It requires network access and a working Rust +toolchain, and it never publishes, signs, or uploads anything. + +```sh +python3 scripts/tests/audit-release-gate/network_smoke.py +``` + +This suite is the runtime proof for the guard contract: it asserts that the real +scanner prints a nonzero `Loaded ... security advisories` line and an absolute +`Scanning /.../Cargo.lock for vulnerabilities ([1-9][0-9]* crate dependencies)` +line on the current machine. The captured logs under `fixtures/logs` are real +scanner output but were captured with a relative `--file Cargo.lock`; the +synthetic variants in `test_report_guard.py` are contract fixtures, not runtime +proof. See `fixtures/logs/README.md`. + +The smoke suite never expects the repository's own `Cargo.lock` to pass; it uses +isolated fixtures only. Run it from a normal checkout with `cargo` on `PATH`. + +## Shell lint + +```sh +shellcheck scripts/cargo-audit-release-gate.sh scripts/cargo-audit-report-guard.sh \ + scripts/tests/audit-release-gate/fixtures/fake-scanner/cargo-audit \ + scripts/tests/audit-release-gate/fixtures/stubs/* +``` + +## Limits + +- The guard is deliberately coupled to the pinned scanner's human-readable + terminal format. Re-review `cargo-audit-report-guard.sh` and its tests whenever + the pinned cargo-audit version changes. +- The offline workflow checks are structural. They prove the gate step exists in + the right job at the right position with the exact reviewed command; they do + not execute GitHub Actions. +- The opt-in smoke suite validates the scanner contract on the current machine. + It does not validate GitHub runner images, npm/OIDC publishing, or + action-gh-release. diff --git a/scripts/tests/audit-release-gate/fixtures/fake-scanner/cargo-audit b/scripts/tests/audit-release-gate/fixtures/fake-scanner/cargo-audit new file mode 100755 index 00000000..78b50aee --- /dev/null +++ b/scripts/tests/audit-release-gate/fixtures/fake-scanner/cargo-audit @@ -0,0 +1,155 @@ +#!/usr/bin/env bash +# +# Test double for the pinned cargo-audit CLI. +# +# This file is used only by scripts/tests/audit-release-gate. Production never +# runs it. It reads FAKE_SCANNER_MODE from the environment so the offline test +# suite can drive every controlled scenario (clean scan, findings, hidden index +# error, missing progress markers, fetch failure, wrong advisory origin) without +# a network or a real scanner. +# +# Modes: +# success clean scan, full progress sequence, exit 0 +# warning clean scan with an allowed informational warning, exit 0 +# findings report a vulnerability, exit 1 +# index-error exit 0 while hiding a crates.io index failure +# yanked-error exit 0 while hiding an incomplete yanked check +# quiet exit 0 with no output at all +# disabled-yanked exit 0 with the Updating marker absent +# disabled-fetch exit 0 with the Fetching and Updating markers absent +# fetch-error fail to fetch the advisory database, exit 1 +# config-error fail to load configuration, exit 1 +# wrong-origin exit 0 after creating a database with an untrusted origin +# warm-authorize succeed only when the requested --db already contains a warm +# git database; otherwise simulate an unavailable network +# refresh and exit 1. Used to prove a warm inherited cache is +# not reused by the gate. +# shellcheck disable=SC2016 # the backticks are literal scanner output, not expansion +set -uo pipefail + +mode="${FAKE_SCANNER_MODE:-success}" +version_line="${FAKE_SCANNER_VERSION_LINE:-cargo-audit-audit 0.22.2}" + +if [ "${1:-}" = "audit" ] && [ "${2:-}" = "--version" ]; then + printf '%s\n' "$version_line" + exit 0 +fi + +db="" +lockfile="" +previous="" +for argument in "$@"; do + case "$previous" in + --db) db="$argument" ;; + --file) lockfile="$argument" ;; + esac + previous="$argument" +done + +# The pinned scanner echoes the --file argument it was given. Tests can override +# the echoed path to prove the gate's lockfile binding is enforced end to end. +scan_path="${FAKE_SCANNER_SCAN_PATH:-$lockfile}" + +if [ -n "${FAKE_SCANNER_RECORD:-}" ]; then + { + printf 'db=%s\n' "$db" + printf 'file=%s\n' "$lockfile" + printf 'cargo_home=%s\n' "${CARGO_HOME:-}" + printf 'args=%s\n' "$*" + printf 'mode=%s\n' "$mode" + } >>"$FAKE_SCANNER_RECORD" +fi + +make_fake_db() { + [ -n "$db" ] || return 0 + mkdir -p "$db" + if [ ! -d "$db/.git" ]; then + git -C "$db" init --quiet + git -C "$db" remote add origin https://github.com/RustSec/advisory-db.git + printf 'fixture advisory database\n' >"$db/README.md" + git -C "$db" -c user.email=fixture@example.invalid -c user.name=fixture add README.md + git -C "$db" -c user.email=fixture@example.invalid -c user.name=fixture commit --quiet -m 'fixture advisory database' + fi +} + +emit_progress() { + printf ' Fetching advisory database from `https://github.com/RustSec/advisory-db.git`\n' + printf ' Loaded 1380 security advisories (from %s)\n' "$db" + printf ' Updating crates.io index\n' + printf ' Scanning %s for vulnerabilities (1 crate dependencies)\n' "$scan_path" +} + +case "$mode" in + success) + make_fake_db + emit_progress + exit 0 + ;; + warning) + make_fake_db + emit_progress + printf '\nwarning: 1 allowed warning found\n' + exit 0 + ;; + findings) + make_fake_db + emit_progress + printf '\nCrate: openssl\nVersion: 0.10.38\nTitle: fixture advisory\nID: RUSTSEC-0000-0000\n' + printf '\nerror: 1 vulnerability found!\n' + exit 1 + ;; + index-error) + make_fake_db + emit_progress + printf "warning: couldn't update crates.io index: fixture I/O failure\n" + exit 0 + ;; + yanked-error) + make_fake_db + emit_progress + printf "error: couldn't check if the package is yanked: fixture index miss\n" + exit 0 + ;; + quiet) + make_fake_db + exit 0 + ;; + disabled-yanked) + make_fake_db + printf ' Fetching advisory database from `https://github.com/RustSec/advisory-db.git`\n' + printf ' Loaded 1380 security advisories (from %s)\n' "$db" + printf ' Scanning %s for vulnerabilities (1 crate dependencies)\n' "$scan_path" + exit 0 + ;; + disabled-fetch) + printf ' Loaded 1380 security advisories (from %s)\n' "$db" + printf ' Scanning %s for vulnerabilities (1 crate dependencies)\n' "$scan_path" + exit 0 + ;; + fetch-error) + printf 'error: failed to fetch advisory database\n' >&2 + exit 1 + ;; + config-error) + printf 'error: failed to load audit configuration\n' >&2 + exit 1 + ;; + wrong-origin) + make_fake_db + git -C "$db" remote set-url origin https://example.invalid/advisory-db.git + emit_progress + exit 0 + ;; + warm-authorize) + if [ -n "$db" ] && [ -d "$db/.git" ]; then + emit_progress + exit 0 + fi + printf 'error: failed to fetch advisory database\n' >&2 + exit 1 + ;; + *) + printf 'fixture scanner: unknown FAKE_SCANNER_MODE=%s\n' "$mode" >&2 + exit 64 + ;; +esac diff --git a/scripts/tests/audit-release-gate/fixtures/logs/README.md b/scripts/tests/audit-release-gate/fixtures/logs/README.md new file mode 100644 index 00000000..d459eaf7 --- /dev/null +++ b/scripts/tests/audit-release-gate/fixtures/logs/README.md @@ -0,0 +1,46 @@ +# Captured scanner logs + +These logs are real terminal output captured from the pinned `cargo-audit 0.22.2` +official Linux x86_64 musl binary (`cargo-audit-audit 0.22.2`, archive SHA-256 +`7fb9497f8594b389e5fce5ef9b92db08432996895b2e0c5a0167a69ed445c428`). The guard +tests feed them to `scripts/cargo-audit-report-guard.sh` together with the +recorded scanner exit code and the expected lockfile path, so guard behavior is +verified against real scanner text rather than hand-written approximations. + +They are not "expected output" for a healthy network run; each one captures a +specific real scanner behavior. They were captured with a relative +`--file Cargo.lock`, so the tests pass `Cargo.lock` as the expected lockfile path +for these files. Production always passes an absolute path; the opt-in +`network_smoke.py` suite is the runtime proof for the absolute-path line. + +## Provenance categories + +- This directory holds **captured runtime logs**: real output from the pinned + binary, byte-for-byte as emitted (placeholder scratch paths aside). +- `test_report_guard.py` also derives **synthetic contract fixtures** in memory + (zero counts, missing counts, reordered markers, substituted lockfile paths). + Those are contract tests for rejection paths, not captured scanner output. +- `network_smoke.py` is the **runtime proof** that connects the two: it runs the + real gate end to end against the real binary and asserts the nonzero counts and + absolute lockfile path the guard requires. + +| Log | Scanner exit | Guard verdict | What it captures | +|---|---|---|---| +| `clean-success.log` | 0 | pass | Clean lockfile, full Fetching/Loaded/Updating/Scanning sequence | +| `warning-only-success.log` | 0 | pass | Informational warning is not fatal when the scan exits zero | +| `vulnerable-failure.log` | 1 | reject | Ordinary advisory rejects the scan | +| `malformed-lock-rc2.log` | 2 | reject | Malformed Cargo.lock | +| `missing-lock-rc2.log` | 2 | reject | Missing Cargo.lock | +| `index-error-rc0.log` | 0 | reject | crates.io index I/O failure hidden behind a zero exit status | +| `quiet-output-rc0.log` | 0 | reject | `[output] quiet = true` suppresses the progress sequence | +| `disabled-yanked-rc0.log` | 0 | reject | `[yanked] enabled = false` removes the Updating marker | +| `disabled-fetch-rc0.log` | 0 | reject | `[database] fetch = false` removes the Fetching marker | + +Capture method: a small fixture lockfile in a temporary directory was scanned with +`cargo-audit audit --file Cargo.lock --db --url +https://github.com/RustSec/advisory-db.git --format terminal --color never`, +redirecting combined stdout and stderr. The index-error and configuration logs +used the documented failure injections (a directory in place of +`CARGO_HOME/.package-cache`, and scanner configuration overrides). The temporary +paths embedded in the logs are scratch paths from the capture run and are not +part of the assertions. diff --git a/scripts/tests/audit-release-gate/fixtures/logs/clean-success.log b/scripts/tests/audit-release-gate/fixtures/logs/clean-success.log new file mode 100644 index 00000000..8b017d21 --- /dev/null +++ b/scripts/tests/audit-release-gate/fixtures/logs/clean-success.log @@ -0,0 +1,4 @@ + Fetching advisory database from `https://github.com/RustSec/advisory-db.git` + Loaded 1380 security advisories (from /tmp/cargo-release-preflight-vrpsnQ/rustsec-db) + Updating crates.io index + Scanning Cargo.lock for vulnerabilities (1 crate dependencies) diff --git a/scripts/tests/audit-release-gate/fixtures/logs/disabled-fetch-rc0.log b/scripts/tests/audit-release-gate/fixtures/logs/disabled-fetch-rc0.log new file mode 100644 index 00000000..6e5b30c7 --- /dev/null +++ b/scripts/tests/audit-release-gate/fixtures/logs/disabled-fetch-rc0.log @@ -0,0 +1,2 @@ + Loaded 1380 security advisories (from /tmp/cargo-release-preflight-vrpsnQ/rustsec-db) + Scanning Cargo.lock for vulnerabilities (1 crate dependencies) diff --git a/scripts/tests/audit-release-gate/fixtures/logs/disabled-yanked-rc0.log b/scripts/tests/audit-release-gate/fixtures/logs/disabled-yanked-rc0.log new file mode 100644 index 00000000..928d3a90 --- /dev/null +++ b/scripts/tests/audit-release-gate/fixtures/logs/disabled-yanked-rc0.log @@ -0,0 +1,3 @@ + Fetching advisory database from `https://github.com/RustSec/advisory-db.git` + Loaded 1380 security advisories (from /tmp/cargo-release-preflight-vrpsnQ/rustsec-db) + Scanning Cargo.lock for vulnerabilities (1 crate dependencies) diff --git a/scripts/tests/audit-release-gate/fixtures/logs/index-error-rc0.log b/scripts/tests/audit-release-gate/fixtures/logs/index-error-rc0.log new file mode 100644 index 00000000..5cb97983 --- /dev/null +++ b/scripts/tests/audit-release-gate/fixtures/logs/index-error-rc0.log @@ -0,0 +1,5 @@ + Fetching advisory database from `https://github.com/RustSec/advisory-db.git` + Loaded 1380 security advisories (from /tmp/cargo-release-preflight-vrpsnQ/rustsec-db) + Updating crates.io index +warning: couldn't update crates.io index: I/O operation failed: failed to obtain lock file '/tmp/gate-fixture-capture/index-home/.package-cache' + Scanning Cargo.lock for vulnerabilities (1 crate dependencies) diff --git a/scripts/tests/audit-release-gate/fixtures/logs/malformed-lock-rc2.log b/scripts/tests/audit-release-gate/fixtures/logs/malformed-lock-rc2.log new file mode 100644 index 00000000..5ef6bc40 --- /dev/null +++ b/scripts/tests/audit-release-gate/fixtures/logs/malformed-lock-rc2.log @@ -0,0 +1,16 @@ + Fetching advisory database from `https://github.com/RustSec/advisory-db.git` + Loaded 1380 security advisories (from /tmp/cargo-release-preflight-vrpsnQ/rustsec-db) + Updating crates.io index +error: not found: Couldn't load Cargo.lock +Caused by: + -> I/O operation failed: parse error: TOML parse error at line 1, column 5 + | +1 | not a valid lockfile = [ + | ^ +key with no value, expected `=` + + -> parse error: TOML parse error at line 1, column 5 + | +1 | not a valid lockfile = [ + | ^ +key with no value, expected `=` diff --git a/scripts/tests/audit-release-gate/fixtures/logs/missing-lock-rc2.log b/scripts/tests/audit-release-gate/fixtures/logs/missing-lock-rc2.log new file mode 100644 index 00000000..0ce9b776 --- /dev/null +++ b/scripts/tests/audit-release-gate/fixtures/logs/missing-lock-rc2.log @@ -0,0 +1,7 @@ + Fetching advisory database from `https://github.com/RustSec/advisory-db.git` + Loaded 1380 security advisories (from /tmp/cargo-release-preflight-vrpsnQ/rustsec-db) + Updating crates.io index +error: not found: Couldn't load missing.lock +Caused by: + -> I/O operation failed: I/O operation failed: entity not found + -> I/O operation failed: entity not found diff --git a/scripts/tests/audit-release-gate/fixtures/logs/quiet-output-rc0.log b/scripts/tests/audit-release-gate/fixtures/logs/quiet-output-rc0.log new file mode 100644 index 00000000..e69de29b diff --git a/scripts/tests/audit-release-gate/fixtures/logs/vulnerable-failure.log b/scripts/tests/audit-release-gate/fixtures/logs/vulnerable-failure.log new file mode 100644 index 00000000..3ef98b63 --- /dev/null +++ b/scripts/tests/audit-release-gate/fixtures/logs/vulnerable-failure.log @@ -0,0 +1,62 @@ + Fetching advisory database from `https://github.com/RustSec/advisory-db.git` + Loaded 1380 security advisories (from /tmp/cargo-release-preflight-vrpsnQ/rustsec-db) + Updating crates.io index + Scanning Cargo.lock for vulnerabilities (2 crate dependencies) +Crate: openssl +Version: 0.10.38 +Title: `openssl` `X509VerifyParamRef::set_host` buffer over-read +Date: 2023-06-20 +ID: RUSTSEC-2023-0044 +URL: https://rustsec.org/advisories/RUSTSEC-2023-0044 +Solution: Upgrade to >=0.10.55 + +Crate: openssl +Version: 0.10.38 +Title: `openssl` `X509Extension::new` and `X509Extension::new_nid` null pointer dereference +Date: 2023-03-24 +ID: RUSTSEC-2023-0024 +URL: https://rustsec.org/advisories/RUSTSEC-2023-0024 +Solution: Upgrade to >=0.10.48 + +Crate: openssl +Version: 0.10.38 +Title: ssl::select_next_proto use after free +Date: 2025-02-02 +ID: RUSTSEC-2025-0004 +URL: https://rustsec.org/advisories/RUSTSEC-2025-0004 +Solution: Upgrade to >=0.10.70 + +Crate: openssl +Version: 0.10.38 +Title: `openssl` `SubjectAlternativeName` and `ExtendedKeyUsage::other` allow arbitrary file read +Date: 2023-03-24 +ID: RUSTSEC-2023-0023 +URL: https://rustsec.org/advisories/RUSTSEC-2023-0023 +Solution: Upgrade to >=0.10.48 + +Crate: openssl +Version: 0.10.38 +Title: `openssl` `X509NameBuilder::build` returned object is not thread safe +Date: 2023-03-24 +ID: RUSTSEC-2023-0022 +URL: https://rustsec.org/advisories/RUSTSEC-2023-0022 +Solution: Upgrade to >=0.10.48 + +Crate: openssl +Version: 0.10.38 +Title: `MemBio::get_buf` has undefined behavior with empty buffers +Date: 2024-07-21 +ID: RUSTSEC-2024-0357 +URL: https://rustsec.org/advisories/RUSTSEC-2024-0357 +Solution: Upgrade to >=0.10.66 + +Crate: openssl +Version: 0.10.38 +Warning: unsound +Title: `openssl` `X509StoreRef::objects` is unsound +Date: 2023-11-23 +ID: RUSTSEC-2023-0072 +URL: https://rustsec.org/advisories/RUSTSEC-2023-0072 + +error: 6 vulnerabilities found! +warning: 1 allowed warning found diff --git a/scripts/tests/audit-release-gate/fixtures/logs/warning-only-success.log b/scripts/tests/audit-release-gate/fixtures/logs/warning-only-success.log new file mode 100644 index 00000000..c65cd8af --- /dev/null +++ b/scripts/tests/audit-release-gate/fixtures/logs/warning-only-success.log @@ -0,0 +1,13 @@ + Fetching advisory database from `https://github.com/RustSec/advisory-db.git` + Loaded 1380 security advisories (from /tmp/cargo-release-preflight-vrpsnQ/rustsec-db) + Updating crates.io index + Scanning Cargo.lock for vulnerabilities (2 crate dependencies) +Crate: paste +Version: 1.0.15 +Warning: unmaintained +Title: paste - no longer maintained +Date: 2024-10-07 +ID: RUSTSEC-2024-0436 +URL: https://rustsec.org/advisories/RUSTSEC-2024-0436 + +warning: 1 allowed warning found diff --git a/scripts/tests/audit-release-gate/fixtures/stubs/cargo b/scripts/tests/audit-release-gate/fixtures/stubs/cargo new file mode 100755 index 00000000..79b278e0 --- /dev/null +++ b/scripts/tests/audit-release-gate/fixtures/stubs/cargo @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# +# Test double for cargo used by the audit release gate tests. +# +# The pinned prebuilt cargo-audit shells out to `cargo -V`. FAKE_CARGO_MODE=fail +# simulates a missing or broken toolchain. +set -euo pipefail + +case "${FAKE_CARGO_MODE:-ok}" in + ok) printf 'cargo 1.99.0 (fixture)\n' ;; + fail) + printf 'error: fixture cargo is unavailable\n' >&2 + exit 1 + ;; + *) + printf 'cargo stub: unknown FAKE_CARGO_MODE\n' >&2 + exit 64 + ;; +esac diff --git a/scripts/tests/audit-release-gate/fixtures/stubs/curl b/scripts/tests/audit-release-gate/fixtures/stubs/curl new file mode 100755 index 00000000..1cb9b1b7 --- /dev/null +++ b/scripts/tests/audit-release-gate/fixtures/stubs/curl @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# +# Test double for curl used by the audit release gate tests. +# +# It copies the fixture archive to the gate's requested --output path so the +# gate's download, checksum, extraction, and installation flow is exercised +# without network access. FAKE_CURL_MODE=fail simulates a download failure. +# FAKE_CURL_RECORD captures the full argument list so tests can assert the +# reviewed network time bounds are present. +set -euo pipefail + +output="" +previous="" +for argument in "$@"; do + case "$previous" in + -o | --output) output="$argument" ;; + esac + previous="$argument" +done + +if [ -n "${FAKE_CURL_RECORD:-}" ]; then + printf '%s\n' "$*" >>"$FAKE_CURL_RECORD" +fi + +case "${FAKE_CURL_MODE:-ok}" in + ok) ;; + fail) + printf 'curl: (22) fixture download failure\n' >&2 + exit 22 + ;; + *) + printf 'curl stub: unknown FAKE_CURL_MODE\n' >&2 + exit 64 + ;; +esac + +[ -n "$output" ] || { printf 'curl stub: no --output argument\n' >&2; exit 2; } +cp "${FAKE_CURL_SOURCE:?FAKE_CURL_SOURCE must point at the fixture archive}" "$output" diff --git a/scripts/tests/audit-release-gate/fixtures/stubs/sha256sum b/scripts/tests/audit-release-gate/fixtures/stubs/sha256sum new file mode 100755 index 00000000..2581b25f --- /dev/null +++ b/scripts/tests/audit-release-gate/fixtures/stubs/sha256sum @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# +# Test double for sha256sum used by the audit release gate tests. +# +# The gate hashes real files (the repository lockfile and policy) and the +# downloaded scanner archive. Real hashing is delegated to $REAL_SHA256SUM so the +# source/lockfile identity checks are genuinely exercised. Only the fixture +# archive, identified by its real digest, is substituted with the reviewed pin +# (FAKE_ARCHIVE_SHA_MODE=ok) or with a wrong digest (FAKE_ARCHIVE_SHA_MODE=bad). +set -euo pipefail + +if [ "$#" -ge 1 ] && [ -f "$1" ]; then + actual="$("${REAL_SHA256SUM:?REAL_SHA256SUM must point at the real sha256sum}" "$1" | cut -d' ' -f1)" + if [ -n "${FAKE_ARCHIVE_REAL_SHA:-}" ] && [ "$actual" = "$FAKE_ARCHIVE_REAL_SHA" ]; then + case "${FAKE_ARCHIVE_SHA_MODE:-ok}" in + ok) printf '%s %s\n' "${FAKE_ARCHIVE_SHA:?FAKE_ARCHIVE_SHA must be set}" "$1" ;; + bad) printf '%064d %s\n' 0 "$1" ;; + *) printf 'sha256sum stub: unknown FAKE_ARCHIVE_SHA_MODE\n' >&2; exit 64 ;; + esac + exit 0 + fi +fi + +exec "${REAL_SHA256SUM:?REAL_SHA256SUM must point at the real sha256sum}" "$@" diff --git a/scripts/tests/audit-release-gate/fixtures/stubs/timeout b/scripts/tests/audit-release-gate/fixtures/stubs/timeout new file mode 100755 index 00000000..2a1999b7 --- /dev/null +++ b/scripts/tests/audit-release-gate/fixtures/stubs/timeout @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# +# Test double for coreutils timeout used by the audit release gate tests. +# +# By default (FAKE_TIMEOUT_MODE=exec) it delegates to the real timeout binary so +# the gate's bounded scanner invocation behaves exactly as in production. +# FAKE_TIMEOUT_MODE=fail124 and fail137 simulate a scan killed by the timeout +# (SIGTERM and SIGKILL respectively) without waiting for the real 5 minute bound. +# FAKE_TIMEOUT_RECORD captures the full argument list so tests can assert the +# reviewed --kill-after and duration bounds are present. +set -euo pipefail + +if [ -n "${FAKE_TIMEOUT_RECORD:-}" ]; then + printf '%s\n' "$*" >>"$FAKE_TIMEOUT_RECORD" +fi + +case "${FAKE_TIMEOUT_MODE:-exec}" in + exec) + exec "${REAL_TIMEOUT:?REAL_TIMEOUT must point at the real timeout}" "$@" + ;; + fail124) + printf 'timeout stub: simulated scanner timeout (SIGTERM)\n' >&2 + exit 124 + ;; + fail137) + printf 'timeout stub: simulated scanner kill (SIGKILL)\n' >&2 + exit 137 + ;; + *) + printf 'timeout stub: unknown FAKE_TIMEOUT_MODE\n' >&2 + exit 64 + ;; +esac diff --git a/scripts/tests/audit-release-gate/network_smoke.py b/scripts/tests/audit-release-gate/network_smoke.py new file mode 100644 index 00000000..797fb8d2 --- /dev/null +++ b/scripts/tests/audit-release-gate/network_smoke.py @@ -0,0 +1,198 @@ +"""Opt-in network smoke suite for the cargo-audit release gate. + +This suite is not part of the deterministic offline run. It downloads the real +pinned cargo-audit archive, verifies its reviewed checksum, installs it, clones a +fresh RustSec advisory database, updates the crates.io index, and scans small +fixture lockfiles. It requires a working Rust toolchain and network access. + +Run it from the repository root with: + + python3 scripts/tests/audit-release-gate/network_smoke.py + +It never publishes, signs, uploads, or pushes anything, and it never asserts that +the repository's own Cargo.lock passes: that lock is expected to be blocked while +unfixed advisories remain. + +The file is deliberately named so that `unittest discover -p 'test_*.py'` skips it. +""" +import os +import re +import subprocess +import tempfile +import unittest +from pathlib import Path + +HERE = Path(__file__).resolve().parent +REPO = HERE.parents[2] +GATE = REPO / "scripts" / "cargo-audit-release-gate.sh" + +SCANNER_VERSION_LINE = "cargo-audit-audit 0.22.2" +ADVISORY_DB_URL = "https://github.com/RustSec/advisory-db.git" +LOCK_HEADER = 'version = 3\n\n[[package]]\nname = "smoke-fixture"\nversion = "0.1.0"\n' +DEFAULT_POLICY = "[advisories]\nignore = []\n" + +GIT_ENV = {**os.environ, "GIT_CONFIG_GLOBAL": os.devnull, "GIT_CONFIG_SYSTEM": os.devnull} + + +def registry_package(name, version): + return ( + f'\n[[package]]\nname = "{name}"\nversion = "{version}"\n' + 'source = "registry+https://github.com/rust-lang/crates.io-index"\n' + ) + + +def make_repo(root, lock_text, policy_text=DEFAULT_POLICY): + repo = Path(root) / "repo" + repo.mkdir() + (repo / ".cargo").mkdir() + (repo / "Cargo.lock").write_text(lock_text) + (repo / ".cargo" / "audit.toml").write_text(policy_text) + for args in ( + ("init", "--quiet", "--initial-branch=main", str(repo)), + ("-C", str(repo), "add", "-A"), + ( + "-C", + str(repo), + "-c", + "user.email=smoke@example.invalid", + "-c", + "user.name=smoke", + "commit", + "--quiet", + "-m", + "smoke fixture", + ), + ): + subprocess.run(["git", *args], check=True, capture_output=True, text=True, env=GIT_ENV) + head = subprocess.run( + ["git", "-C", str(repo), "rev-parse", "HEAD"], + check=True, + capture_output=True, + text=True, + env=GIT_ENV, + ).stdout.strip() + return repo, head + + +def run_gate(repo, sha): + return subprocess.run( + ["bash", str(GATE), sha], + cwd=str(repo), + capture_output=True, + text=True, + check=False, + timeout=900, + ) + + +class NetworkSmoke(unittest.TestCase): + def setUp(self): + self._tmp = tempfile.TemporaryDirectory(prefix="audit-gate-smoke.") + self.addCleanup(self._tmp.cleanup) + + def assert_scanner_identity(self, output): + self.assertIn(f"scanner-version: {SCANNER_VERSION_LINE}", output) + revision = re.search(r"^advisory-db-revision: ([0-9a-f]{40,64})$", output, re.MULTILINE) + self.assertIsNotNone(revision, output) + self.assertIn(f"advisory-db-origin: {ADVISORY_DB_URL}", output) + # A per-execution fetch must be visible in the displayed log. + self.assertIn("Fetching advisory database", output) + # The real scanner must report a nonzero advisory count and scan the + # absolute lockfile path with a nonzero crate dependency count. This is + # the runtime proof for the structured report-guard contract. + self.assertIsNotNone( + re.search(r"^\s*Loaded [1-9][0-9]* security advisories", output, re.MULTILINE), + output, + ) + self.assertIsNotNone( + re.search( + r"^\s*Scanning /.*/Cargo\.lock for vulnerabilities \([1-9][0-9]* crate dependencies\)$", + output, + re.MULTILINE, + ), + output, + ) + + def test_clean_fixture_passes_and_refreshes(self): + repo, head = make_repo(self._tmp.name, LOCK_HEADER) + first = run_gate(repo, head) + self.assertEqual(first.returncode, 0, first.stdout + first.stderr) + self.assertIn("result: PASS", first.stdout) + self.assert_scanner_identity(first.stdout) + + # Retrying the gate re-fetches advisory data instead of reusing state. + second = run_gate(repo, head) + self.assertEqual(second.returncode, 0, second.stdout + second.stderr) + self.assert_scanner_identity(second.stdout) + + def test_vulnerable_fixture_blocks_publication(self): + lock = LOCK_HEADER + registry_package("openssl", "0.10.38") + repo, head = make_repo(self._tmp.name, lock) + result = run_gate(repo, head) + self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("result: FAIL", result.stdout + result.stderr) + self.assertRegex(result.stdout, r"RUSTSEC-\d{4}-\d{4}") + self.assert_scanner_identity(result.stdout) + + def test_informational_warning_fixture_passes(self): + lock = LOCK_HEADER + registry_package("paste", "1.0.15") + repo, head = make_repo(self._tmp.name, lock) + result = run_gate(repo, head) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("allowed warning found", result.stdout) + + def test_existing_ignore_policy_is_honored(self): + lock = LOCK_HEADER + registry_package("openssl", "0.10.38") + repo, head = make_repo(self._tmp.name, lock) + + blocked = run_gate(repo, head) + self.assertNotEqual(blocked.returncode, 0, blocked.stdout + blocked.stderr) + advisories = sorted(set(re.findall(r"RUSTSEC-\d{4}-\d{4}", blocked.stdout))) + self.assertTrue(advisories, blocked.stdout) + + policy = "[advisories]\nignore = [\n" + for advisory in advisories: + policy += f' "{advisory}",\n' + policy += "]\n" + (repo / ".cargo" / "audit.toml").write_text(policy) + subprocess.run( + ["git", "-C", str(repo), "add", "-A"], + check=True, + capture_output=True, + text=True, + env=GIT_ENV, + ) + subprocess.run( + [ + "git", + "-C", + str(repo), + "-c", + "user.email=smoke@example.invalid", + "-c", + "user.name=smoke", + "commit", + "--quiet", + "-m", + "ignore the fixture advisories", + ], + check=True, + capture_output=True, + text=True, + env=GIT_ENV, + ) + new_head = subprocess.run( + ["git", "-C", str(repo), "rev-parse", "HEAD"], + check=True, + capture_output=True, + text=True, + env=GIT_ENV, + ).stdout.strip() + + allowed = run_gate(repo, new_head) + self.assertEqual(allowed.returncode, 0, allowed.stdout + allowed.stderr) + self.assertIn("result: PASS", allowed.stdout) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/scripts/tests/audit-release-gate/test_release_gate.py b/scripts/tests/audit-release-gate/test_release_gate.py new file mode 100644 index 00000000..8cb0021c --- /dev/null +++ b/scripts/tests/audit-release-gate/test_release_gate.py @@ -0,0 +1,523 @@ +"""Offline control-flow tests for scripts/cargo-audit-release-gate.sh. + +These tests run the real production helper against a temporary git repository, a +fixture scanner archive, and local PATH test doubles for curl, sha256sum, and +cargo. They never touch the network, install a real scanner, run a publisher, or +read any environment variable that the production helper itself honors. The test +doubles only shape the helper's inputs and record what the helper did. + +A separate opt-in suite, network_smoke.py, exercises the real download, checksum, +database refresh, and scan path on a runner with network access. +""" +import hashlib +import os +import shutil +import subprocess +import tempfile +import unittest +from pathlib import Path + +HERE = Path(__file__).resolve().parent +REPO = HERE.parents[2] +GATE = REPO / "scripts" / "cargo-audit-release-gate.sh" +FIXTURES = HERE / "fixtures" +FAKE_SCANNER = FIXTURES / "fake-scanner" / "cargo-audit" +STUBS = FIXTURES / "stubs" + +ARCHIVE_URL = ( + "https://github.com/rustsec/rustsec/releases/download/" + "cargo-audit/v0.22.2/cargo-audit-x86_64-unknown-linux-musl-v0.22.2.tgz" +) +ARCHIVE_SHA256 = "7fb9497f8594b389e5fce5ef9b92db08432996895b2e0c5a0167a69ed445c428" +SCANNER_VERSION_LINE = "cargo-audit-audit 0.22.2" +ADVISORY_DB_URL = "https://github.com/RustSec/advisory-db.git" +LOCK_TEXT = 'version = 3\n\n[[package]]\nname = "gate-fixture"\nversion = "0.1.0"\n' +POLICY_TEXT = '[advisories]\nignore = []\n' + + +def sha256_file(path): + digest = hashlib.sha256() + digest.update(Path(path).read_bytes()) + return digest.hexdigest() + + +def find_real(tool): + for directory in os.environ["PATH"].split(os.pathsep): + candidate = Path(directory) / tool + if candidate.is_file() and os.access(candidate, os.X_OK): + return str(candidate) + raise RuntimeError(f"cannot find {tool} on PATH") + + +class GateHarness: + """Builds one isolated gate fixture: a git repo, an archive, and stubs.""" + + def __init__(self, root): + self.root = Path(root) + self.repo = self.root / "repo" + self.stub_dir = self.root / "stubs" + self.archive = self.root / "fixture.tgz" + self.record = self.root / "scanner-record.txt" + self.sentinel = self.root / "sentinel" + self.real_sha256sum = find_real("sha256sum") + self.real_timeout = find_real("timeout") + self._make_repo() + self._make_stub_dir() + self._make_archive() + + def _git(self, *args): + env = {**os.environ, "GIT_CONFIG_GLOBAL": os.devnull, "GIT_CONFIG_SYSTEM": os.devnull} + return subprocess.run(["git", *args], capture_output=True, text=True, check=True, env=env) + + def _make_repo(self): + self.repo.mkdir() + (self.repo / ".cargo").mkdir() + (self.repo / "Cargo.lock").write_text(LOCK_TEXT) + (self.repo / ".cargo" / "audit.toml").write_text(POLICY_TEXT) + self._git("init", "--quiet", "--initial-branch=main", str(self.repo)) + self._git("-C", str(self.repo), "add", "-A") + self._git( + "-C", + str(self.repo), + "-c", + "user.email=gate@example.invalid", + "-c", + "user.name=gate", + "commit", + "--quiet", + "-m", + "fixture", + ) + self.head_sha = self._git("-C", str(self.repo), "rev-parse", "HEAD").stdout.strip() + + def _make_stub_dir(self): + self.stub_dir.mkdir() + for name in ("curl", "sha256sum", "cargo", "timeout"): + destination = self.stub_dir / name + shutil.copy(STUBS / name, destination) + destination.chmod(0o755) + + def _make_archive(self): + stage = self.root / "stage" + archive_root = stage / "cargo-audit-x86_64-unknown-linux-musl-v0.22.2" + archive_root.mkdir(parents=True) + binary = archive_root / "cargo-audit" + shutil.copy(FAKE_SCANNER, binary) + binary.chmod(0o755) + subprocess.run( + ["tar", "-czf", str(self.archive), "-C", str(stage), archive_root.name], + check=True, + ) + self.archive_real_sha = sha256_file(self.archive) + + def env(self, **overrides): + env = dict(os.environ) + env["PATH"] = f"{self.stub_dir}{os.pathsep}{env['PATH']}" + env["REAL_SHA256SUM"] = self.real_sha256sum + env["REAL_TIMEOUT"] = self.real_timeout + env["FAKE_CURL_SOURCE"] = str(self.archive) + env["FAKE_CURL_MODE"] = "ok" + env["FAKE_ARCHIVE_REAL_SHA"] = self.archive_real_sha + env["FAKE_ARCHIVE_SHA"] = ARCHIVE_SHA256 + env["FAKE_ARCHIVE_SHA_MODE"] = "ok" + env["FAKE_CARGO_MODE"] = "ok" + env["FAKE_TIMEOUT_MODE"] = "exec" + env["FAKE_SCANNER_MODE"] = "success" + env["FAKE_SCANNER_RECORD"] = str(self.record) + env["GIT_CONFIG_GLOBAL"] = os.devnull + env["GIT_CONFIG_SYSTEM"] = os.devnull + env.update(overrides) + return env + + def path_without(self, *excluded): + """Build a PATH view of the real tools plus the stubs, minus exclusions. + + Used to prove the gate's required-tool check, which cannot be exercised + by removing a tool from the inherited PATH while the stubs still provide + it. + """ + shim = self.root / ("shim-no-" + "-".join(excluded)) + if shim.exists(): + return str(shim) + shim.mkdir() + seen = set() + sources = [self.stub_dir, *os.environ["PATH"].split(os.pathsep)] + for source in sources: + directory = Path(source) + if not directory.is_dir(): + continue + for entry in sorted(directory.iterdir()): + name = entry.name + if name in excluded or name in seen: + continue + try: + if entry.is_file() and os.access(entry, os.X_OK): + (shim / name).symlink_to(entry) + seen.add(name) + except OSError: + continue + return str(shim) + + def run_gate(self, script=None, expected_sha=None, sentinel=None, cwd=None, **env_overrides): + script = script or GATE + expected_sha = expected_sha if expected_sha is not None else self.head_sha + sentinel = sentinel or self.sentinel + command = f'"{script}" "{expected_sha}" && touch "{sentinel}"' + return subprocess.run( + ["bash", "-c", command], + cwd=str(cwd or self.repo), + env=self.env(**env_overrides), + capture_output=True, + text=True, + check=False, + timeout=120, + ) + + def run_raw(self, args, cwd=None, **env_overrides): + return subprocess.run( + ["bash", *[str(arg) for arg in args]], + cwd=str(cwd or self.repo), + env=self.env(**env_overrides), + capture_output=True, + text=True, + check=False, + timeout=120, + ) + + def scanner_runs(self): + if not self.record.exists(): + return [] + runs = [] + current = {} + for line in self.record.read_text().splitlines(): + if not line: + continue + key, _, value = line.partition("=") + if key == "db" and current: + runs.append(current) + current = {} + current[key] = value + if current: + runs.append(current) + return runs + + +class ReleaseGateTests(unittest.TestCase): + def setUp(self): + self._tmp = tempfile.TemporaryDirectory(prefix="audit-gate-test.") + self.addCleanup(self._tmp.cleanup) + self.harness = GateHarness(self._tmp.name) + + def assert_blocked(self, proc, message=None): + self.assertNotEqual(proc.returncode, 0, proc.stdout + proc.stderr) + self.assertFalse(self.harness.sentinel.exists(), "gate failure reached the sentinel") + if message is not None: + self.assertIn(message, proc.stdout + proc.stderr) + + # --- accepted path ----------------------------------------------------- + + def test_success_path_reaches_sentinel_and_logs_identity(self): + proc = self.harness.run_gate() + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + self.assertTrue(self.harness.sentinel.exists()) + self.assertIn("result: PASS", proc.stdout) + self.assertIn(f"expected-sha: {self.harness.head_sha}", proc.stdout) + self.assertIn(f"scanner-version: {SCANNER_VERSION_LINE}", proc.stdout) + self.assertIn(f"advisory-db-origin: {ADVISORY_DB_URL}", proc.stdout) + self.assertIn("advisory-db-revision:", proc.stdout) + self.assertNotIn("advisory-db-revision: unavailable", proc.stdout) + self.assertIn("cargo-lock-sha256:", proc.stdout) + self.assertIn("audit-toml-sha256:", proc.stdout) + + def test_scanner_invocation_uses_pinned_flags_and_no_bypasses(self): + proc = self.harness.run_gate() + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + runs = self.harness.scanner_runs() + self.assertEqual(len(runs), 1, runs) + args = runs[0]["args"] + self.assertIn(f"--url {ADVISORY_DB_URL}", args) + self.assertIn("--format terminal", args) + self.assertIn("--color never", args) + self.assertIn("--file", args) + self.assertIn("--db", args) + for bypass in ("--no-fetch", "--stale", "--deny", "--ignore"): + self.assertNotIn(bypass, args) + + def test_report_log_is_displayed(self): + proc = self.harness.run_gate() + self.assertIn("begin cargo-audit output", proc.stdout) + self.assertIn("end cargo-audit output", proc.stdout) + runs = self.harness.scanner_runs() + self.assertEqual(len(runs), 1, runs) + scanned = runs[0]["file"] + self.assertTrue(scanned.startswith("/"), scanned) + self.assertIn(f"Scanning {scanned} for vulnerabilities (1 crate dependencies)", proc.stdout) + + def test_scan_of_a_substituted_lockfile_blocks_sentinel(self): + # The scanner reports a different lockfile than the one the gate passed + # to it; the pure guard must reject the mismatch before the sentinel. + proc = self.harness.run_gate(FAKE_SCANNER_SCAN_PATH="Cargo.lock") + self.assert_blocked(proc, "but the gate expected") + + def test_download_is_bounded(self): + record = self.harness.root / "curl-args.txt" + proc = self.harness.run_gate(FAKE_CURL_RECORD=str(record)) + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + argv = record.read_text() + self.assertIn("--connect-timeout 15", argv) + self.assertIn("--max-time 180", argv) + self.assertIn("--proto =https", argv) + + def test_scanner_timeout_is_bounded_and_blocks_sentinel(self): + for mode, expected in (("fail124", 124), ("fail137", 137)): + with self.subTest(mode=mode): + record = self.harness.root / f"timeout-{mode}.txt" + proc = self.harness.run_gate( + FAKE_TIMEOUT_MODE=mode, + FAKE_TIMEOUT_RECORD=str(record), + ) + self.assert_blocked(proc) + self.assertEqual(proc.returncode, expected, proc.stdout + proc.stderr) + argv = record.read_text() + self.assertIn("--kill-after=10s", argv) + self.assertIn("5m", argv) + self.assertIn("audit", argv) + # The bounded timeout fired before the scanner could run. + self.assertEqual(self.harness.scanner_runs(), []) + + def test_missing_timeout_tool_blocks_sentinel(self): + proc = self.harness.run_gate(PATH=self.harness.path_without("timeout")) + self.assert_blocked(proc, "required tool not found on PATH: timeout") + self.assertEqual(proc.returncode, 2, proc.stdout + proc.stderr) + + # --- scanner verdicts -------------------------------------------------- + + def test_findings_block_sentinel(self): + proc = self.harness.run_gate(FAKE_SCANNER_MODE="findings") + self.assert_blocked(proc) + self.assertEqual(proc.returncode, 1, proc.stdout + proc.stderr) + + def test_hidden_index_error_blocks_sentinel(self): + proc = self.harness.run_gate(FAKE_SCANNER_MODE="index-error") + self.assert_blocked(proc) + + def test_yanked_diagnostic_blocks_sentinel(self): + proc = self.harness.run_gate(FAKE_SCANNER_MODE="yanked-error") + self.assert_blocked(proc) + + def test_missing_progress_markers_block_sentinel(self): + for mode in ("quiet", "disabled-yanked", "disabled-fetch"): + with self.subTest(mode=mode): + self.harness.sentinel.unlink(missing_ok=True) + proc = self.harness.run_gate(FAKE_SCANNER_MODE=mode) + self.assert_blocked(proc) + + def test_fetch_failure_blocks_sentinel(self): + proc = self.harness.run_gate(FAKE_SCANNER_MODE="fetch-error") + self.assert_blocked(proc) + self.assertEqual(proc.returncode, 1, proc.stdout + proc.stderr) + + def test_config_error_blocks_sentinel(self): + proc = self.harness.run_gate(FAKE_SCANNER_MODE="config-error") + self.assert_blocked(proc) + + def test_untrusted_advisory_origin_blocks_sentinel(self): + proc = self.harness.run_gate(FAKE_SCANNER_MODE="wrong-origin") + self.assert_blocked(proc, "unexpected advisory database origin") + + # --- installation and prerequisite failures ---------------------------- + + def test_download_failure_blocks_sentinel(self): + proc = self.harness.run_gate(FAKE_CURL_MODE="fail") + self.assert_blocked(proc, "failed to download") + + def test_checksum_mismatch_blocks_sentinel(self): + proc = self.harness.run_gate(FAKE_ARCHIVE_SHA_MODE="bad") + self.assert_blocked(proc, "checksum mismatch") + + def test_extraction_failure_blocks_sentinel(self): + corrupt = Path(self._tmp.name) / "corrupt.tgz" + corrupt.write_text("this is not a gzip archive\n") + proc = self.harness.run_gate( + FAKE_CURL_SOURCE=str(corrupt), + FAKE_ARCHIVE_REAL_SHA=sha256_file(corrupt), + FAKE_ARCHIVE_SHA_MODE="ok", + ) + self.assert_blocked(proc, "failed to extract") + + def test_version_mismatch_blocks_sentinel(self): + proc = self.harness.run_gate(FAKE_SCANNER_VERSION_LINE="cargo-audit-audit 0.22.1") + self.assert_blocked(proc, "unexpected cargo-audit version") + + def test_non_exact_version_is_rejected(self): + # The check must match the whole version line, not a substring. + proc = self.harness.run_gate( + FAKE_SCANNER_VERSION_LINE="cargo-audit-audit 0.22.20 extra" + ) + self.assert_blocked(proc, "unexpected cargo-audit version") + + def test_broken_cargo_toolchain_blocks_sentinel(self): + proc = self.harness.run_gate(FAKE_CARGO_MODE="fail") + self.assert_blocked(proc, "cargo -V failed") + + # --- source and policy identity --------------------------------------- + + def test_sha_mismatch_blocks_sentinel(self): + proc = self.harness.run_gate(expected_sha="a" * 40) + self.assert_blocked(proc, "does not match expected release commit") + + def test_working_tree_lockfile_mismatch_blocks_sentinel(self): + (self.harness.repo / "Cargo.lock").write_text(LOCK_TEXT + "\n# tampered\n") + proc = self.harness.run_gate() + self.assert_blocked(proc, "Cargo.lock does not match") + + def test_working_tree_policy_mismatch_blocks_sentinel(self): + (self.harness.repo / ".cargo" / "audit.toml").write_text( + '[advisories]\nignore = ["RUSTSEC-0000-0000"]\n' + ) + proc = self.harness.run_gate() + self.assert_blocked(proc, "audit.toml does not match") + + def test_missing_lockfile_blocks_sentinel(self): + (self.harness.repo / "Cargo.lock").unlink() + proc = self.harness.run_gate() + self.assert_blocked(proc, "missing Cargo.lock") + + def test_missing_policy_blocks_sentinel(self): + (self.harness.repo / ".cargo" / "audit.toml").unlink() + proc = self.harness.run_gate() + self.assert_blocked(proc, "missing .cargo/audit.toml") + + # --- usage and input validation --------------------------------------- + + def test_usage_and_input_validation(self): + cases = [ + ([GATE], 2), + ([GATE, "a", "b"], 2), + ([GATE, "not-a-commit"], 2), + ([GATE, "A" * 40], 2), + ] + for args, expected in cases: + with self.subTest(args=args): + proc = self.harness.run_raw(args) + self.assertEqual(proc.returncode, expected, proc.stdout + proc.stderr) + + # --- freshness and retry ---------------------------------------------- + + def test_fresh_private_directories_per_execution(self): + first = self.harness.run_gate() + self.assertEqual(first.returncode, 0, first.stdout + first.stderr) + second = self.harness.run_gate() + self.assertEqual(second.returncode, 0, second.stdout + second.stderr) + + runs = self.harness.scanner_runs() + self.assertEqual(len(runs), 2, runs) + for run in runs: + self.assertTrue(run["db"].startswith(os.path.join(tempfile.gettempdir(), "cargo-audit-release-gate"))) + self.assertTrue(run["cargo_home"].startswith(os.path.join(tempfile.gettempdir(), "cargo-audit-release-gate"))) + self.assertNotEqual(run["db"], run["cargo_home"]) + self.assertNotEqual(runs[0]["db"], runs[1]["db"]) + self.assertNotEqual(runs[0]["cargo_home"], runs[1]["cargo_home"]) + + def test_inherited_warm_cache_cannot_authorize_refresh_failure(self): + # The inherited environment carries a warm RustSec clone with an + # untrusted origin plus a warm Cargo home, both under the inherited + # TMPDIR. The stub below would authorize the release if it were handed + # the warm database, and fails when the database path is fresh. The gate + # must therefore fail here rather than reuse any inherited state. + inherited = self.harness.root / "inherited" + inherited.mkdir() + warm_db = inherited / "rustsec-db" + warm_db.mkdir() + self.harness._git("init", "--quiet", str(warm_db)) + self.harness._git( + "-C", str(warm_db), "remote", "add", "origin", "https://example.invalid/advisory-db.git" + ) + (warm_db / "README.md").write_text("warm fixture advisory database\n") + self.harness._git("-C", str(warm_db), "add", "-A") + self.harness._git( + "-C", + str(warm_db), + "-c", + "user.email=fixture@example.invalid", + "-c", + "user.name=fixture", + "commit", + "--quiet", + "-m", + "warm fixture database", + ) + warm_cargo_home = inherited / "cargo-home" + (warm_cargo_home / "registry" / "index").mkdir(parents=True) + (warm_cargo_home / ".package-cache").write_text("warm cargo cache\n") + + # Prove the stub discriminates: it succeeds on the warm path and fails + # on a fresh path that simulates an unavailable network refresh. + warm = self.harness.run_raw( + [FAKE_SCANNER, "audit", "--db", str(warm_db), "--file", "Cargo.lock"], + FAKE_SCANNER_MODE="warm-authorize", + FAKE_SCANNER_RECORD="", + ) + self.assertEqual(warm.returncode, 0, warm.stdout + warm.stderr) + self.assertIn("Fetching advisory database", warm.stdout) + fresh = self.harness.run_raw( + [FAKE_SCANNER, "audit", "--db", str(inherited / "no-such-db"), "--file", "Cargo.lock"], + FAKE_SCANNER_MODE="warm-authorize", + FAKE_SCANNER_RECORD="", + ) + self.assertNotEqual(fresh.returncode, 0, fresh.stdout + fresh.stderr) + + proc = self.harness.run_gate( + FAKE_SCANNER_MODE="warm-authorize", + TMPDIR=str(inherited), + CARGO_HOME=str(warm_cargo_home), + ) + self.assert_blocked(proc) + runs = self.harness.scanner_runs() + self.assertEqual(len(runs), 1, runs) + fresh_work = str(inherited / "cargo-audit-release-gate") + self.assertTrue(runs[0]["db"].startswith(fresh_work), runs) + self.assertTrue(runs[0]["cargo_home"].startswith(fresh_work), runs) + self.assertNotEqual(runs[0]["db"], str(warm_db)) + self.assertNotEqual(runs[0]["cargo_home"], str(warm_cargo_home)) + # The inherited untrusted-origin database is never inspected or reused. + self.assertNotIn("example.invalid", proc.stdout + proc.stderr) + + def test_retry_after_refresh_failure_rescans_and_can_pass(self): + failed = self.harness.run_gate(FAKE_SCANNER_MODE="fetch-error") + self.assert_blocked(failed) + passed = self.harness.run_gate() + self.assertEqual(passed.returncode, 0, passed.stdout + passed.stderr) + runs = self.harness.scanner_runs() + self.assertEqual(len(runs), 2, runs) + self.assertNotEqual(runs[0]["db"], runs[1]["db"]) + self.assertNotEqual(runs[0]["cargo_home"], runs[1]["cargo_home"]) + + # --- helper robustness ------------------------------------------------ + + def test_unexpected_guard_status_is_rejected(self): + copydir = self.harness.root / "copyscripts" + copydir.mkdir() + gate_copy = copydir / "cargo-audit-release-gate.sh" + shutil.copy(GATE, gate_copy) + gate_copy.chmod(0o755) + guard_copy = copydir / "cargo-audit-report-guard.sh" + guard_copy.write_text("#!/usr/bin/env bash\nexit 127\n") + guard_copy.chmod(0o755) + proc = self.harness.run_gate(script=gate_copy) + self.assert_blocked(proc, "unexpected status") + self.assertEqual(proc.returncode, 1, proc.stdout + proc.stderr) + + def test_pinned_constants_are_literal_and_reviewed(self): + text = GATE.read_text() + self.assertIn(ARCHIVE_URL, text) + self.assertIn(ARCHIVE_SHA256, text) + self.assertIn(SCANNER_VERSION_LINE, text) + self.assertIn(ADVISORY_DB_URL, text) + # No dynamic digest lookup and no alternate-archive environment hook. + self.assertNotIn("sha256sum -c", text) + self.assertNotIn("curl --head", text) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/audit-release-gate/test_report_guard.py b/scripts/tests/audit-release-gate/test_report_guard.py new file mode 100644 index 00000000..ca39a2f7 --- /dev/null +++ b/scripts/tests/audit-release-gate/test_report_guard.py @@ -0,0 +1,257 @@ +"""Offline tests for scripts/cargo-audit-report-guard.sh. + +The guard is pure: it takes a captured cargo-audit terminal log, the numeric +scanner exit code, and the lockfile path the gate expected the scan to cover, and +returns a verdict. None of these tests touch the network, install anything, or +run a publisher. + +The logs under fixtures/logs were captured from the real pinned cargo-audit +0.22.2 binary; see fixtures/logs/README.md for provenance. They were captured +with a relative `--file Cargo.lock`, so the tests pass `Cargo.lock` as the +expected path for them. Synthetic variants are derived in memory to cover +individual rejection reasons (zero counts, out-of-order markers, substituted +paths); those are contract fixtures, not captured runtime output. +""" +import os +import subprocess +import tempfile +import unittest +from pathlib import Path + +HERE = Path(__file__).resolve().parent +REPO = HERE.parents[2] +GUARD = REPO / "scripts" / "cargo-audit-report-guard.sh" +LOGS = HERE / "fixtures" / "logs" + +# (captured log name, captured scanner exit code, expected guard exit code) +CAPTURED = [ + ("clean-success", 0, 0), + ("warning-only-success", 0, 0), + ("vulnerable-failure", 1, 1), + ("malformed-lock-rc2", 2, 1), + ("missing-lock-rc2", 2, 1), + ("index-error-rc0", 0, 1), + ("quiet-output-rc0", 0, 1), + ("disabled-yanked-rc0", 0, 1), + ("disabled-fetch-rc0", 0, 1), +] + +# The capture run scanned a relative `Cargo.lock`. +CAPTURED_LOCKFILE = "Cargo.lock" +MARKERS = ("Fetching", "Loaded", "Updating", "Scanning") + + +def run_guard(log, rc, expected_lockfile=CAPTURED_LOCKFILE, extra=()): + return subprocess.run( + ["bash", str(GUARD), str(log), str(rc), str(expected_lockfile), *extra], + capture_output=True, + text=True, + check=False, + ) + + +def clean_text(): + return (LOGS / "clean-success.log").read_text() + + +def write_tmp_log(tmp, text, name="log.txt"): + path = Path(tmp) / name + path.write_text(text) + return path + + +class ReportGuardTests(unittest.TestCase): + def test_captured_logs_guard_verdicts(self): + for name, rc, expected in CAPTURED: + with self.subTest(log=name, rc=rc): + log = LOGS / f"{name}.log" + self.assertTrue(log.is_file(), f"missing captured log: {log}") + result = run_guard(log, rc) + self.assertEqual(result.returncode, expected, result.stdout + result.stderr) + + def test_informational_warning_is_not_a_rejection(self): + # A zero-exit scan whose only output is informational warnings must pass, + # preserving the existing scanner warning policy. + log = LOGS / "warning-only-success.log" + self.assertIn("allowed warning found", log.read_text()) + self.assertEqual(run_guard(log, 0).returncode, 0) + + def test_clean_zero_exit_requires_every_marker(self): + text = clean_text() + for marker in MARKERS: + self.assertIn(marker, text) + for marker in MARKERS: + with self.subTest(removed=marker): + mutated = "\n".join( + line for line in text.splitlines() if marker not in line + ) + with tempfile.TemporaryDirectory() as tmp: + self.assertEqual(run_guard(write_tmp_log(tmp, mutated + "\n"), 0).returncode, 1) + + def test_zero_exit_with_hidden_index_diagnostics_is_rejected(self): + for diagnostic in ( + "warning: couldn't update crates.io index: fixture failure\n", + "warning: couldn't open crates.io index: fixture failure\n", + "error: couldn't check if the package is yanked: fixture miss\n", + ): + with self.subTest(diagnostic=diagnostic): + text = clean_text() + diagnostic + with tempfile.TemporaryDirectory() as tmp: + self.assertEqual(run_guard(write_tmp_log(tmp, text), 0).returncode, 1) + + def test_empty_log_with_zero_exit_is_rejected(self): + with tempfile.TemporaryDirectory() as tmp: + self.assertEqual(run_guard(write_tmp_log(tmp, ""), 0).returncode, 1) + + def test_bare_marker_words_are_not_enough(self): + # The guard must require the substantive marker lines, not the words + # "Fetching", "Loaded", "Updating", and "Scanning" in unrelated text. + text = ( + "Fetching something else\n" + "Loaded a note\n" + "Updating something else\n" + "Scanning something else\n" + ) + with tempfile.TemporaryDirectory() as tmp: + self.assertEqual(run_guard(write_tmp_log(tmp, text), 0).returncode, 1) + + def test_zero_advisory_count_is_rejected(self): + text = clean_text().replace("Loaded 1380 security advisories", "Loaded 0 security advisories") + with tempfile.TemporaryDirectory() as tmp: + self.assertEqual(run_guard(write_tmp_log(tmp, text), 0).returncode, 1) + + def test_missing_advisory_count_is_rejected(self): + text = clean_text().replace("Loaded 1380 security advisories", "Loaded some security advisories") + with tempfile.TemporaryDirectory() as tmp: + self.assertEqual(run_guard(write_tmp_log(tmp, text), 0).returncode, 1) + + def test_zero_dependency_count_is_rejected(self): + text = clean_text().replace("(1 crate dependencies)", "(0 crate dependencies)") + with tempfile.TemporaryDirectory() as tmp: + self.assertEqual(run_guard(write_tmp_log(tmp, text), 0).returncode, 1) + + def test_missing_dependency_count_is_rejected(self): + text = clean_text().replace("(1 crate dependencies)", "(several crate dependencies)") + with tempfile.TemporaryDirectory() as tmp: + self.assertEqual(run_guard(write_tmp_log(tmp, text), 0).returncode, 1) + + def test_out_of_order_markers_are_rejected(self): + text = clean_text() + lines = text.splitlines() + fetch = next(line for line in lines if "Fetching advisory database" in line) + loaded = next(line for line in lines if "Loaded" in line) + reordered = [line for line in lines if line is not fetch] + reordered.insert(reordered.index(loaded) + 1, fetch) + with tempfile.TemporaryDirectory() as tmp: + self.assertEqual(run_guard(write_tmp_log(tmp, "\n".join(reordered) + "\n"), 0).returncode, 1) + + def test_wrong_expected_lockfile_is_rejected(self): + with tempfile.TemporaryDirectory() as tmp: + log = write_tmp_log(tmp, clean_text()) + result = run_guard(log, 0, "/expected/elsewhere/Cargo.lock") + self.assertEqual(result.returncode, 1, result.stdout + result.stderr) + self.assertIn("but the gate expected", result.stderr) + + def test_lockfile_path_suffix_is_not_accepted(self): + # A scan of /a/b/Cargo.lock must not satisfy an expectation of + # /b/Cargo.lock: the comparison is exact, not a suffix match. + text = clean_text().replace( + "Scanning Cargo.lock for vulnerabilities", + "Scanning /a/b/Cargo.lock for vulnerabilities", + ) + with tempfile.TemporaryDirectory() as tmp: + self.assertEqual(run_guard(write_tmp_log(tmp, text), 0, "/b/Cargo.lock").returncode, 1) + + def test_absolute_expected_lockfile_is_accepted(self): + text = clean_text().replace( + "Scanning Cargo.lock for vulnerabilities", + "Scanning /tmp/absolute/path/Cargo.lock for vulnerabilities", + ) + with tempfile.TemporaryDirectory() as tmp: + self.assertEqual( + run_guard(write_tmp_log(tmp, text), 0, "/tmp/absolute/path/Cargo.lock").returncode, + 0, + ) + + def test_nonzero_scanner_exit_is_rejected_even_with_clean_log(self): + # The scanner exit code is authoritative and independent of the log: + # a complete, clean-looking log must not rescue a nonzero scan, including + # the 124/137 statuses produced by the bounded scanner timeout. + log = LOGS / "clean-success.log" + for rc in ("1", "2", "124", "137"): + with self.subTest(rc=rc): + result = run_guard(log, rc) + self.assertEqual(result.returncode, 1, result.stdout + result.stderr) + self.assertIn("release blocked", result.stderr) + + def test_non_numeric_scanner_exit_code_is_rejected(self): + log = LOGS / "clean-success.log" + for bogus in ("abc", "", "-1", "0x0", "1.0"): + with self.subTest(rc=bogus): + self.assertEqual(run_guard(log, bogus).returncode, 2) + + def test_missing_log_is_a_usage_error(self): + self.assertEqual(run_guard(LOGS / "does-not-exist.log", 0).returncode, 2) + + def test_empty_expected_lockfile_is_a_usage_error(self): + self.assertEqual(run_guard(LOGS / "clean-success.log", 0, "").returncode, 2) + + def test_wrong_argument_count_is_a_usage_error(self): + for args in ( + [], + [str(LOGS / "clean-success.log")], + [str(LOGS / "clean-success.log"), "0"], + ["a", "b", "c", "d"], + ): + with self.subTest(args=args): + result = subprocess.run( + ["bash", str(GUARD), *args], + capture_output=True, + text=True, + check=False, + ) + self.assertEqual(result.returncode, 2) + + def test_rejection_prevents_following_command(self): + # A shell sequence must not reach its next command after a rejected scan + # and must reach it after an accepted scan. + with tempfile.TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + sentinel = tmp_path / "sentinel" + failing = LOGS / "index-error-rc0.log" + passing = LOGS / "clean-success.log" + script = 'set -e; bash "$1" "$2" "$3" "$4"; touch "$5"' + + rejected = subprocess.run( + [ + "bash", "-c", script, "guard", + str(GUARD), str(failing), "0", CAPTURED_LOCKFILE, str(sentinel), + ], + capture_output=True, + text=True, + check=False, + ) + self.assertNotEqual(rejected.returncode, 0) + self.assertFalse(sentinel.exists()) + + accepted = subprocess.run( + [ + "bash", "-c", script, "guard", + str(GUARD), str(passing), "0", CAPTURED_LOCKFILE, str(sentinel), + ], + capture_output=True, + text=True, + check=False, + ) + self.assertEqual(accepted.returncode, 0, accepted.stdout + accepted.stderr) + self.assertTrue(sentinel.exists()) + + def test_guard_is_executable_and_offline_only(self): + self.assertTrue(os.access(GUARD, os.X_OK)) + text = GUARD.read_text() + for forbidden in ("curl", "mktemp", "npm publish", "action-gh-release", "aws "): + self.assertNotIn(forbidden, text) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/audit-release-gate/test_workflows.py b/scripts/tests/audit-release-gate/test_workflows.py new file mode 100644 index 00000000..474425a3 --- /dev/null +++ b/scripts/tests/audit-release-gate/test_workflows.py @@ -0,0 +1,657 @@ +"""Offline structural tests for the release-gate workflows. + +These tests parse the real workflow YAML and assert that the strict cargo-audit +gate runs inside each protected job, before that job's first publication side +effect, against the exact source revision that produced the artifacts. They also +run a set of negative mutations to prove the checks reject an absent, reordered, +masked, or unlocked gate. + +PyYAML is a test-only dependency. `on:` is a YAML 1.1 boolean key, so the trigger +block is read through the `trigger()` helper rather than by key name. +""" +import copy +import unittest +from pathlib import Path + +import yaml + +HERE = Path(__file__).resolve().parent +REPO = HERE.parents[2] +WORKFLOWS = REPO / ".github" / "workflows" +PUBLISH = WORKFLOWS / "publish.yml" +RELEASE = WORKFLOWS / "release-datadog-serverless-compat.yml" +BUILD = WORKFLOWS / "build-datadog-serverless-compat.yml" +TESTS = WORKFLOWS / "audit-release-gate-tests.yml" + +CHECKOUT_PIN = "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" +RUST_SETUP_PIN = "actions-rust-lang/setup-rust-toolchain@9399c7bb15d4c7d47b27263d024f0a4978346ba4" +GH_RELEASE_PIN = "softprops/action-gh-release@da05d552573ad5aba039eaac05058a918a7bf631" +GATE_CALL = "cargo-audit-release-gate.sh" +GATE_COMMAND = './scripts/cargo-audit-release-gate.sh "$GITHUB_SHA"' +SOURCE_REF = "${{ github.sha }}" +BUILD_ONLY_CONDITION = "${{ inputs.publish == 'Build and Publish' }}" +DOWNLOAD_ARTIFACT_PREFIX = "actions/download-artifact@" +EXTERNAL_ARTIFACT_SELECTORS = ("run-id", "repository", "github-token") + + +def load(path): + with path.open(encoding="utf-8") as handle: + return yaml.safe_load(handle) + + +def trigger(doc): + for key in ("on", True): + if key in doc: + return doc[key] + raise AssertionError("workflow has no trigger block") + + +def step_run(step): + return str(step.get("run", "")) + + +def find_step(steps, predicate): + for index, step in enumerate(steps): + if predicate(step): + return index, step + return None, None + + +def normalized_run(step): + return " ".join(step_run(step).split()) + + +def assert_no_failure_masks(doc): + for job_name, job in doc["jobs"].items(): + if job.get("continue-on-error"): + raise AssertionError(f"job '{job_name}' sets continue-on-error") + job_condition = str(job.get("if", "")) + if "always(" in job_condition or "cancelled(" in job_condition: + raise AssertionError(f"job '{job_name}' has an error-masking condition") + for step in job.get("steps", []): + if step.get("continue-on-error"): + raise AssertionError(f"job '{job_name}' has a step with continue-on-error") + condition = str(step.get("if", "")) + if "always(" in condition or "cancelled(" in condition: + raise AssertionError(f"job '{job_name}' has an error-masking step condition") + + +def assert_same_run_artifact_downloads(steps, where): + # Downloading artifacts with a run id, repository, or explicit token could + # pull artifacts produced by a different run than the gated source revision. + for step in steps: + if not str(step.get("uses", "")).startswith(DOWNLOAD_ARTIFACT_PREFIX): + continue + with_block = step.get("with", {}) or {} + for selector in EXTERNAL_ARTIFACT_SELECTORS: + if selector in with_block: + raise AssertionError( + f"{where}: artifact download must come from the current run, not '{selector}'" + ) + + +def assert_gate_step(step, where): + if step is None: + raise AssertionError(f"{where}: strict cargo-audit gate is missing") + if "if" in step: + raise AssertionError(f"{where}: gate step must be unconditional") + if "continue-on-error" in step: + raise AssertionError(f"{where}: gate step must not ignore failures") + run = normalized_run(step) + if run != GATE_COMMAND: + raise AssertionError( + f"{where}: gate step must run exactly '{GATE_COMMAND}', found '{run}'" + ) + shell = step.get("shell") + if shell is not None and str(shell) not in ("bash", "bash {0}"): + raise AssertionError(f"{where}: gate step must use the reviewed bash shell, found '{shell}'") + if step.get("working-directory") != "source": + raise AssertionError(f"{where}: gate step must run from the source checkout") + + +def assert_checkout_ref(step, where): + if step is None: + raise AssertionError(f"{where}: checkout is missing") + if step.get("uses") != CHECKOUT_PIN: + raise AssertionError(f"{where}: checkout is not pinned to the reviewed commit") + if step.get("with", {}).get("ref") != SOURCE_REF: + raise AssertionError(f"{where}: checkout does not pin ref {SOURCE_REF}") + + +def assert_source_checkout(step, where): + assert_checkout_ref(step, where) + if step.get("with", {}).get("path") != "source": + raise AssertionError(f"{where}: checkout does not use path 'source'") + + +def assert_rust_setup(step, where): + if step is None: + raise AssertionError(f"{where}: Rust toolchain setup is missing") + if step.get("uses") != RUST_SETUP_PIN: + raise AssertionError(f"{where}: Rust setup is not pinned to the reviewed commit") + if step.get("with", {}).get("cache") is not False: + raise AssertionError(f"{where}: Rust setup must disable the toolchain cache") + + +def assert_package_job(doc): + package = doc["jobs"]["package"] + steps = package["steps"] + _, checkout = find_step(steps, lambda s: str(s.get("uses", "")).startswith("actions/checkout@")) + assert_checkout_ref(checkout, "package") + index, upload = find_step( + steps, lambda s: str(s.get("uses", "")).startswith("actions/upload-artifact@") + ) + if index is None: + raise AssertionError("package job must upload the npm packages it produces") + if upload.get("with", {}).get("name") != "npm-packages": + raise AssertionError("package job must upload the 'npm-packages' artifact") + + +def assert_publish_workflow(doc): + assert_no_failure_masks(doc) + publish = doc["jobs"]["publish"] + if publish.get("environment") != "Production": + raise AssertionError("publish job must keep the Production environment approval") + if publish.get("needs") != "package": + raise AssertionError("publish job must depend on the package job") + condition = " ".join(str(publish.get("if", "")).split()) + if condition != BUILD_ONLY_CONDITION: + raise AssertionError("publish job must keep the build-only condition") + + assert_package_job(doc) + + steps = publish["steps"] + assert_same_run_artifact_downloads(steps, "publish") + artifact_names = [ + step.get("with", {}).get("name") + for step in steps + if str(step.get("uses", "")).startswith(DOWNLOAD_ARTIFACT_PREFIX) + ] + if "npm-packages" not in artifact_names: + raise AssertionError("publish job must download the artifact produced in the same run") + _, checkout = find_step(steps, lambda s: str(s.get("uses", "")).startswith("actions/checkout@")) + assert_source_checkout(checkout, "publish") + + _, rust = find_step( + steps, + lambda s: str(s.get("uses", "")).startswith("actions-rust-lang/setup-rust-toolchain@"), + ) + assert_rust_setup(rust, "publish") + + gate_index, gate_step = find_step(steps, lambda s: GATE_CALL in step_run(s)) + assert_gate_step(gate_step, "publish") + + publish_index, publisher = find_step(steps, lambda s: "npm publish" in step_run(s)) + if publish_index is None: + raise AssertionError("publish job has no npm publish step") + if "if" in publisher: + raise AssertionError("npm publish step must not be conditional") + if gate_index > publish_index: + raise AssertionError("strict cargo-audit gate must run before the first npm publish") + + permissions = doc.get("permissions", {}) + if permissions.get("contents") != "read": + raise AssertionError("publish workflow must grant only contents: read") + if permissions.get("id-token") != "write": + raise AssertionError("publish workflow must keep id-token: write for provenance") + + inputs = trigger(doc).get("workflow_dispatch", {}).get("inputs", {}) or {} + for name in inputs: + if "skip" in name.lower() or "bypass" in name.lower(): + raise AssertionError(f"publish workflow must not expose a bypass input: {name}") + + +def assert_release_workflow(doc): + assert_no_failure_masks(doc) + release = doc["jobs"]["release"] + if release.get("permissions", {}).get("contents") != "write": + raise AssertionError("release job must keep contents: write") + if release.get("needs") != "build": + raise AssertionError("release job must depend on the build job") + # This freezes the reviewed release flow: the strict cargo-audit gate plus a + # draft release is the release check, so no additional protected environment + # (and therefore no extra manual approval) is introduced. This is an + # invariant of the reviewed change, not a claim that environments are unsafe. + if "environment" in release: + raise AssertionError("release job must not add an extra protected environment approval") + + steps = release["steps"] + assert_same_run_artifact_downloads(steps, "release") + _, checkout = find_step(steps, lambda s: str(s.get("uses", "")).startswith("actions/checkout@")) + assert_source_checkout(checkout, "release") + + _, rust = find_step( + steps, + lambda s: str(s.get("uses", "")).startswith("actions-rust-lang/setup-rust-toolchain@"), + ) + assert_rust_setup(rust, "release") + + gate_index, gate_step = find_step(steps, lambda s: GATE_CALL in step_run(s)) + assert_gate_step(gate_step, "release") + + zip_index, _ = find_step(steps, lambda s: step_run(s).strip().startswith("zip ")) + if zip_index is None: + raise AssertionError("release job has no zip step") + release_index, gh_release = find_step( + steps, lambda s: str(s.get("uses", "")).startswith("softprops/action-gh-release@") + ) + if release_index is None: + raise AssertionError("release job has no action-gh-release step") + if gate_index > zip_index or gate_index > release_index: + raise AssertionError("strict cargo-audit gate must run before the zip and the draft release") + + if gh_release.get("uses") != GH_RELEASE_PIN: + raise AssertionError("action-gh-release pin changed") + if gh_release.get("with", {}).get("draft") is not True: + raise AssertionError("draft release behavior changed") + if "if" in gh_release: + raise AssertionError("the draft release step must not be conditional") + + if doc.get("permissions") != {}: + raise AssertionError("release workflow top-level permissions must stay empty") + + +def assert_build_workflow(doc): + builds = [] + checkouts = [] + for job_name, job in doc["jobs"].items(): + for step in job.get("steps", []): + run = step_run(step) + if "cargo build --release -p datadog-serverless-compat" in run: + builds.append((job_name, run)) + if "cargo-audit" in run: + raise AssertionError("the shared build workflow must not run audits") + if str(step.get("uses", "")).startswith("actions/checkout@"): + checkouts.append(step) + + if len(builds) != 5: + raise AssertionError(f"expected 5 release builds, found {len(builds)}") + for job_name, run in builds: + if "--locked" not in run: + raise AssertionError(f"release build in job '{job_name}' is not --locked") + + if len(checkouts) != 2: + raise AssertionError(f"expected 2 producer checkouts, found {len(checkouts)}") + for checkout in checkouts: + assert_checkout_ref(checkout, "build producer") + + +def assert_tests_workflow(doc, raw): + assert_no_failure_masks(doc) + if "if" in doc["jobs"]["offline"]: + raise AssertionError("offline gate tests must run unconditionally") + if doc.get("permissions") != {"contents": "read"}: + raise AssertionError("tests workflow must grant only contents: read") + if "id-token" in raw: + raise AssertionError("tests workflow must not request OIDC") + + for job_name, job in doc["jobs"].items(): + if "environment" in job: + raise AssertionError(f"tests job '{job_name}' must not use a protected environment") + + for forbidden in ("npm publish", "action-gh-release", "gh workflow run", "git push"): + if forbidden in raw: + raise AssertionError(f"tests workflow must not publish or dispatch: {forbidden}") + + triggers = trigger(doc) + if "workflow_dispatch" not in triggers: + raise AssertionError("tests workflow needs workflow_dispatch for the opt-in network smoke") + inputs = triggers["workflow_dispatch"].get("inputs", {}) or {} + if "network_smoke" not in inputs: + raise AssertionError("workflow_dispatch must expose the opt-in network_smoke input") + + pull_request = triggers.get("pull_request") + if not pull_request or "paths" not in pull_request: + raise AssertionError("pull_request trigger must be limited to relevant paths") + for required in ("scripts/cargo-audit-release-gate.sh", "scripts/tests/audit-release-gate/**"): + if required not in pull_request["paths"]: + raise AssertionError(f"pull_request paths must include {required}") + + if "unittest discover" not in raw: + raise AssertionError("tests workflow must run the deterministic offline suite") + + smoke = doc["jobs"].get("network-smoke") + if smoke is None: + raise AssertionError("tests workflow is missing the opt-in network smoke job") + condition = str(smoke.get("if", "")) + if "workflow_dispatch" not in condition or "inputs.network_smoke" not in condition: + raise AssertionError("network smoke job must be gated behind workflow_dispatch and inputs.network_smoke") + + +class WorkflowStructureTests(unittest.TestCase): + def test_publish_workflow(self): + assert_publish_workflow(load(PUBLISH)) + + def test_release_workflow(self): + assert_release_workflow(load(RELEASE)) + + def test_build_workflow(self): + assert_build_workflow(load(BUILD)) + + def test_tests_workflow(self): + assert_tests_workflow(load(TESTS), TESTS.read_text()) + + +class WorkflowMutationTests(unittest.TestCase): + def mutated(self, path, mutate): + doc = load(path) + mutate(doc) + return doc + + def assert_rejected(self, checker, doc, message): + with self.assertRaises(AssertionError, msg=message): + checker(doc) + + def test_disabled_or_error_masked_offline_tests_are_rejected(self): + for mutation in ("job-error-mask", "step-error-mask", "conditional-skip"): + with self.subTest(mutation=mutation): + doc = load(TESTS) + offline = doc["jobs"]["offline"] + if mutation == "job-error-mask": + offline["continue-on-error"] = True + elif mutation == "step-error-mask": + offline["steps"][-1]["continue-on-error"] = True + else: + offline["if"] = False + with self.assertRaises(AssertionError): + assert_tests_workflow(doc, TESTS.read_text()) + + def test_removing_gate_is_rejected(self): + def mutate(doc): + steps = doc["jobs"]["publish"]["steps"] + steps[:] = [step for step in steps if GATE_CALL not in step_run(step)] + + self.assert_rejected( + assert_publish_workflow, self.mutated(PUBLISH, mutate), "gate removal must be rejected" + ) + + def test_reordering_gate_after_publish_is_rejected(self): + def mutate(doc): + steps = doc["jobs"]["publish"]["steps"] + gate = next(step for step in steps if GATE_CALL in step_run(step)) + steps.remove(gate) + steps.append(gate) + + self.assert_rejected( + assert_publish_workflow, self.mutated(PUBLISH, mutate), "gate reorder must be rejected" + ) + + def test_continue_on_error_is_rejected(self): + def mutate(doc): + for step in doc["jobs"]["publish"]["steps"]: + if GATE_CALL in step_run(step): + step["continue-on-error"] = True + + self.assert_rejected( + assert_publish_workflow, self.mutated(PUBLISH, mutate), "continue-on-error must be rejected" + ) + + def test_always_condition_is_rejected(self): + def mutate(doc): + for step in doc["jobs"]["publish"]["steps"]: + if "npm publish" in step_run(step): + step["if"] = "always()" + + self.assert_rejected( + assert_publish_workflow, self.mutated(PUBLISH, mutate), "always() must be rejected" + ) + + def test_unpinned_source_ref_is_rejected(self): + def mutate(doc): + for step in doc["jobs"]["publish"]["steps"]: + if str(step.get("uses", "")).startswith("actions/checkout@"): + step["with"]["ref"] = "main" + + self.assert_rejected( + assert_publish_workflow, self.mutated(PUBLISH, mutate), "floating source ref must be rejected" + ) + + def test_missing_locked_flag_is_rejected(self): + def mutate(doc): + for job in doc["jobs"].values(): + for step in job.get("steps", []): + if "cargo build --release -p datadog-serverless-compat" in step_run(step): + step["run"] = step_run(step).replace(" --locked", "") + + self.assert_rejected( + assert_build_workflow, self.mutated(BUILD, mutate), "unlocked build must be rejected" + ) + + def test_missing_producer_source_ref_is_rejected(self): + def mutate(doc): + removed = False + for job in doc["jobs"].values(): + for step in job.get("steps", []): + if str(step.get("uses", "")).startswith("actions/checkout@"): + step.get("with", {}).pop("ref", None) + removed = True + self.assertTrue(removed) + + self.assert_rejected( + assert_build_workflow, self.mutated(BUILD, mutate), "unpinned producer checkout must be rejected" + ) + + def test_release_gate_after_draft_is_rejected(self): + def mutate(doc): + steps = doc["jobs"]["release"]["steps"] + gate = next(step for step in steps if GATE_CALL in step_run(step)) + steps.remove(gate) + steps.append(gate) + + self.assert_rejected( + assert_release_workflow, self.mutated(RELEASE, mutate), "late release gate must be rejected" + ) + + def test_non_draft_release_is_rejected(self): + def mutate(doc): + for step in doc["jobs"]["release"]["steps"]: + if str(step.get("uses", "")).startswith("softprops/action-gh-release@"): + step["with"]["draft"] = False + + self.assert_rejected( + assert_release_workflow, self.mutated(RELEASE, mutate), "draft change must be rejected" + ) + + def test_tests_workflow_oidc_is_rejected(self): + doc = load(TESTS) + doc["permissions"] = {"contents": "read", "id-token": "write"} + raw = TESTS.read_text() + "\n# id-token: write\n" + self.assert_rejected( + lambda candidate: assert_tests_workflow(candidate, raw), doc, "OIDC must be rejected" + ) + + # --- gate command masking --------------------------------------------- + + def mutate_gate_run(self, doc, replacement): + for step in doc["jobs"]["publish"]["steps"]: + if GATE_CALL in step_run(step): + step["run"] = replacement(step_run(step)) + return + raise AssertionError("gate step not found") + + def test_gate_or_true_masking_is_rejected(self): + self.assert_rejected( + assert_publish_workflow, + self.mutated(PUBLISH, lambda doc: self.mutate_gate_run(doc, lambda run: run + " || true")), + "|| true masking must be rejected", + ) + + def test_gate_exit_zero_masking_is_rejected(self): + self.assert_rejected( + assert_publish_workflow, + self.mutated(PUBLISH, lambda doc: self.mutate_gate_run(doc, lambda run: run + "; exit 0")), + "; exit 0 masking must be rejected", + ) + + def test_gate_set_plus_e_masking_is_rejected(self): + self.assert_rejected( + assert_publish_workflow, + self.mutated(PUBLISH, lambda doc: self.mutate_gate_run(doc, lambda run: "set +e\n" + run)), + "set +e masking must be rejected", + ) + + def test_gate_missing_expected_sha_is_rejected(self): + self.assert_rejected( + assert_publish_workflow, + self.mutated( + PUBLISH, + lambda doc: self.mutate_gate_run(doc, lambda run: "./scripts/cargo-audit-release-gate.sh"), + ), + "unbound gate command must be rejected", + ) + + def test_gate_conditional_false_is_rejected(self): + def mutate(doc): + for step in doc["jobs"]["publish"]["steps"]: + if GATE_CALL in step_run(step): + step["if"] = False + + self.assert_rejected( + assert_publish_workflow, self.mutated(PUBLISH, mutate), "conditional gate must be rejected" + ) + + def test_gate_condition_expression_is_rejected(self): + def mutate(doc): + for step in doc["jobs"]["publish"]["steps"]: + if GATE_CALL in step_run(step): + step["if"] = "${{ false }}" + + self.assert_rejected( + assert_publish_workflow, self.mutated(PUBLISH, mutate), "gate expression must be rejected" + ) + + def test_gate_shell_change_is_rejected(self): + def mutate(doc): + for step in doc["jobs"]["publish"]["steps"]: + if GATE_CALL in step_run(step): + step["shell"] = "sh" + + self.assert_rejected( + assert_publish_workflow, self.mutated(PUBLISH, mutate), "non-bash gate shell must be rejected" + ) + + def test_gate_working_directory_change_is_rejected(self): + def mutate(doc): + for step in doc["jobs"]["publish"]["steps"]: + if GATE_CALL in step_run(step): + step["working-directory"] = "." + + self.assert_rejected( + assert_publish_workflow, self.mutated(PUBLISH, mutate), "wrong gate directory must be rejected" + ) + + # --- job-level and artifact boundaries -------------------- + + def test_publish_job_continue_on_error_is_rejected(self): + def mutate(doc): + doc["jobs"]["publish"]["continue-on-error"] = True + + self.assert_rejected( + assert_publish_workflow, + self.mutated(PUBLISH, mutate), + "job-level continue-on-error must be rejected", + ) + + def test_publish_job_build_only_condition_removed_is_rejected(self): + def mutate(doc): + doc["jobs"]["publish"].pop("if", None) + + self.assert_rejected( + assert_publish_workflow, self.mutated(PUBLISH, mutate), "missing build-only condition must be rejected" + ) + + def test_publish_job_build_only_condition_replaced_is_rejected(self): + def mutate(doc): + doc["jobs"]["publish"]["if"] = "always()" + + self.assert_rejected( + assert_publish_workflow, self.mutated(PUBLISH, mutate), "always() build condition must be rejected" + ) + + def test_publisher_cancelled_condition_is_rejected(self): + def mutate(doc): + for step in doc["jobs"]["publish"]["steps"]: + if "npm publish" in step_run(step): + step["if"] = "cancelled()" + + self.assert_rejected( + assert_publish_workflow, self.mutated(PUBLISH, mutate), "cancelled() masking must be rejected" + ) + + def test_npm_publish_condition_is_rejected(self): + def mutate(doc): + for step in doc["jobs"]["publish"]["steps"]: + if "npm publish" in step_run(step): + step["if"] = "${{ always() }}" + + self.assert_rejected( + assert_publish_workflow, self.mutated(PUBLISH, mutate), "conditional publish must be rejected" + ) + + def test_external_artifact_selectors_are_rejected(self): + for selector in EXTERNAL_ARTIFACT_SELECTORS: + with self.subTest(selector=selector): + def mutate(doc, selector=selector): + for step in doc["jobs"]["publish"]["steps"]: + if str(step.get("uses", "")).startswith(DOWNLOAD_ARTIFACT_PREFIX): + step.setdefault("with", {})[selector] = "from-elsewhere" + return + raise AssertionError("no download-artifact step") + + self.assert_rejected( + assert_publish_workflow, + self.mutated(PUBLISH, mutate), + f"artifact selector {selector} must be rejected", + ) + + def test_release_external_artifact_selector_is_rejected(self): + def mutate(doc): + for step in doc["jobs"]["release"]["steps"]: + if str(step.get("uses", "")).startswith(DOWNLOAD_ARTIFACT_PREFIX): + step.setdefault("with", {})["run-id"] = "1234" + return + raise AssertionError("no download-artifact step") + + self.assert_rejected( + assert_release_workflow, self.mutated(RELEASE, mutate), "release artifact selector must be rejected" + ) + + def test_package_checkout_ref_change_is_rejected(self): + def mutate(doc): + for step in doc["jobs"]["package"]["steps"]: + if str(step.get("uses", "")).startswith("actions/checkout@"): + step["with"]["ref"] = "main" + + self.assert_rejected( + assert_publish_workflow, self.mutated(PUBLISH, mutate), "floating package checkout must be rejected" + ) + + def test_package_artifact_name_change_is_rejected(self): + def mutate(doc): + for step in doc["jobs"]["package"]["steps"]: + if str(step.get("uses", "")).startswith("actions/upload-artifact@"): + step["with"]["name"] = "other-packages" + + self.assert_rejected( + assert_publish_workflow, self.mutated(PUBLISH, mutate), "renamed package artifact must be rejected" + ) + + def test_release_needs_change_is_rejected(self): + def mutate(doc): + doc["jobs"]["release"]["needs"] = None + + self.assert_rejected( + assert_release_workflow, self.mutated(RELEASE, mutate), "release dependency change must be rejected" + ) + + def test_release_extra_environment_is_rejected(self): + def mutate(doc): + doc["jobs"]["release"]["environment"] = "Production" + + self.assert_rejected( + assert_release_workflow, self.mutated(RELEASE, mutate), "extra release approval must be rejected" + ) + + +if __name__ == "__main__": + unittest.main() From 6ad271598b9185bc90cc126e860fc70cf257218f Mon Sep 17 00:00:00 2001 From: Lucas Pimentel Date: Wed, 28 Oct 2026 11:04:41 -0400 Subject: [PATCH 2/3] Simplify dependency audit regression tests --- .../tests/audit-release-gate/network_smoke.py | 80 ++++++------------- .../audit-release-gate/test_release_gate.py | 8 +- .../audit-release-gate/test_report_guard.py | 44 +++++----- .../audit-release-gate/test_workflows.py | 4 +- 4 files changed, 51 insertions(+), 85 deletions(-) diff --git a/scripts/tests/audit-release-gate/network_smoke.py b/scripts/tests/audit-release-gate/network_smoke.py index 797fb8d2..52642743 100644 --- a/scripts/tests/audit-release-gate/network_smoke.py +++ b/scripts/tests/audit-release-gate/network_smoke.py @@ -41,37 +41,41 @@ def registry_package(name, version): ) +def git(repo, *args): + return subprocess.run( + ["git", "-C", str(repo), *args], + check=True, + capture_output=True, + text=True, + env=GIT_ENV, + ) + + +def commit_all(repo, message): + git(repo, "add", "-A") + git( + repo, + "-c", "user.email=smoke@example.invalid", + "-c", "user.name=smoke", + "commit", "--quiet", "-m", message, + ) + return git(repo, "rev-parse", "HEAD").stdout.strip() + + def make_repo(root, lock_text, policy_text=DEFAULT_POLICY): repo = Path(root) / "repo" repo.mkdir() (repo / ".cargo").mkdir() (repo / "Cargo.lock").write_text(lock_text) (repo / ".cargo" / "audit.toml").write_text(policy_text) - for args in ( - ("init", "--quiet", "--initial-branch=main", str(repo)), - ("-C", str(repo), "add", "-A"), - ( - "-C", - str(repo), - "-c", - "user.email=smoke@example.invalid", - "-c", - "user.name=smoke", - "commit", - "--quiet", - "-m", - "smoke fixture", - ), - ): - subprocess.run(["git", *args], check=True, capture_output=True, text=True, env=GIT_ENV) - head = subprocess.run( - ["git", "-C", str(repo), "rev-parse", "HEAD"], + subprocess.run( + ["git", "init", "--quiet", "--initial-branch=main", str(repo)], check=True, capture_output=True, text=True, env=GIT_ENV, - ).stdout.strip() - return repo, head + ) + return repo, commit_all(repo, "smoke fixture") def run_gate(repo, sha): @@ -155,39 +159,7 @@ def test_existing_ignore_policy_is_honored(self): policy += f' "{advisory}",\n' policy += "]\n" (repo / ".cargo" / "audit.toml").write_text(policy) - subprocess.run( - ["git", "-C", str(repo), "add", "-A"], - check=True, - capture_output=True, - text=True, - env=GIT_ENV, - ) - subprocess.run( - [ - "git", - "-C", - str(repo), - "-c", - "user.email=smoke@example.invalid", - "-c", - "user.name=smoke", - "commit", - "--quiet", - "-m", - "ignore the fixture advisories", - ], - check=True, - capture_output=True, - text=True, - env=GIT_ENV, - ) - new_head = subprocess.run( - ["git", "-C", str(repo), "rev-parse", "HEAD"], - check=True, - capture_output=True, - text=True, - env=GIT_ENV, - ).stdout.strip() + new_head = commit_all(repo, "ignore the fixture advisories") allowed = run_gate(repo, new_head) self.assertEqual(allowed.returncode, 0, allowed.stdout + allowed.stderr) diff --git a/scripts/tests/audit-release-gate/test_release_gate.py b/scripts/tests/audit-release-gate/test_release_gate.py index 8cb0021c..5cf09a66 100644 --- a/scripts/tests/audit-release-gate/test_release_gate.py +++ b/scripts/tests/audit-release-gate/test_release_gate.py @@ -158,14 +158,14 @@ def path_without(self, *excluded): continue return str(shim) - def run_gate(self, script=None, expected_sha=None, sentinel=None, cwd=None, **env_overrides): + def run_gate(self, script=None, expected_sha=None, sentinel=None, **env_overrides): script = script or GATE expected_sha = expected_sha if expected_sha is not None else self.head_sha sentinel = sentinel or self.sentinel command = f'"{script}" "{expected_sha}" && touch "{sentinel}"' return subprocess.run( ["bash", "-c", command], - cwd=str(cwd or self.repo), + cwd=str(self.repo), env=self.env(**env_overrides), capture_output=True, text=True, @@ -173,10 +173,10 @@ def run_gate(self, script=None, expected_sha=None, sentinel=None, cwd=None, **en timeout=120, ) - def run_raw(self, args, cwd=None, **env_overrides): + def run_raw(self, args, **env_overrides): return subprocess.run( ["bash", *[str(arg) for arg in args]], - cwd=str(cwd or self.repo), + cwd=str(self.repo), env=self.env(**env_overrides), capture_output=True, text=True, diff --git a/scripts/tests/audit-release-gate/test_report_guard.py b/scripts/tests/audit-release-gate/test_report_guard.py index ca39a2f7..09db317b 100644 --- a/scripts/tests/audit-release-gate/test_report_guard.py +++ b/scripts/tests/audit-release-gate/test_report_guard.py @@ -41,9 +41,9 @@ MARKERS = ("Fetching", "Loaded", "Updating", "Scanning") -def run_guard(log, rc, expected_lockfile=CAPTURED_LOCKFILE, extra=()): +def run_guard(log, rc, expected_lockfile=CAPTURED_LOCKFILE): return subprocess.run( - ["bash", str(GUARD), str(log), str(rc), str(expected_lockfile), *extra], + ["bash", str(GUARD), str(log), str(rc), str(expected_lockfile)], capture_output=True, text=True, check=False, @@ -54,13 +54,18 @@ def clean_text(): return (LOGS / "clean-success.log").read_text() -def write_tmp_log(tmp, text, name="log.txt"): - path = Path(tmp) / name +def write_tmp_log(tmp, text): + path = Path(tmp) / "log.txt" path.write_text(text) return path class ReportGuardTests(unittest.TestCase): + def assert_rejected(self, text, expected_lockfile=CAPTURED_LOCKFILE): + with tempfile.TemporaryDirectory() as tmp: + result = run_guard(write_tmp_log(tmp, text), 0, expected_lockfile) + self.assertEqual(result.returncode, 1, result.stdout + result.stderr) + def test_captured_logs_guard_verdicts(self): for name, rc, expected in CAPTURED: with self.subTest(log=name, rc=rc): @@ -85,8 +90,7 @@ def test_clean_zero_exit_requires_every_marker(self): mutated = "\n".join( line for line in text.splitlines() if marker not in line ) - with tempfile.TemporaryDirectory() as tmp: - self.assertEqual(run_guard(write_tmp_log(tmp, mutated + "\n"), 0).returncode, 1) + self.assert_rejected(mutated + "\n") def test_zero_exit_with_hidden_index_diagnostics_is_rejected(self): for diagnostic in ( @@ -95,13 +99,10 @@ def test_zero_exit_with_hidden_index_diagnostics_is_rejected(self): "error: couldn't check if the package is yanked: fixture miss\n", ): with self.subTest(diagnostic=diagnostic): - text = clean_text() + diagnostic - with tempfile.TemporaryDirectory() as tmp: - self.assertEqual(run_guard(write_tmp_log(tmp, text), 0).returncode, 1) + self.assert_rejected(clean_text() + diagnostic) def test_empty_log_with_zero_exit_is_rejected(self): - with tempfile.TemporaryDirectory() as tmp: - self.assertEqual(run_guard(write_tmp_log(tmp, ""), 0).returncode, 1) + self.assert_rejected("") def test_bare_marker_words_are_not_enough(self): # The guard must require the substantive marker lines, not the words @@ -112,28 +113,23 @@ def test_bare_marker_words_are_not_enough(self): "Updating something else\n" "Scanning something else\n" ) - with tempfile.TemporaryDirectory() as tmp: - self.assertEqual(run_guard(write_tmp_log(tmp, text), 0).returncode, 1) + self.assert_rejected(text) def test_zero_advisory_count_is_rejected(self): text = clean_text().replace("Loaded 1380 security advisories", "Loaded 0 security advisories") - with tempfile.TemporaryDirectory() as tmp: - self.assertEqual(run_guard(write_tmp_log(tmp, text), 0).returncode, 1) + self.assert_rejected(text) def test_missing_advisory_count_is_rejected(self): text = clean_text().replace("Loaded 1380 security advisories", "Loaded some security advisories") - with tempfile.TemporaryDirectory() as tmp: - self.assertEqual(run_guard(write_tmp_log(tmp, text), 0).returncode, 1) + self.assert_rejected(text) def test_zero_dependency_count_is_rejected(self): text = clean_text().replace("(1 crate dependencies)", "(0 crate dependencies)") - with tempfile.TemporaryDirectory() as tmp: - self.assertEqual(run_guard(write_tmp_log(tmp, text), 0).returncode, 1) + self.assert_rejected(text) def test_missing_dependency_count_is_rejected(self): text = clean_text().replace("(1 crate dependencies)", "(several crate dependencies)") - with tempfile.TemporaryDirectory() as tmp: - self.assertEqual(run_guard(write_tmp_log(tmp, text), 0).returncode, 1) + self.assert_rejected(text) def test_out_of_order_markers_are_rejected(self): text = clean_text() @@ -142,8 +138,7 @@ def test_out_of_order_markers_are_rejected(self): loaded = next(line for line in lines if "Loaded" in line) reordered = [line for line in lines if line is not fetch] reordered.insert(reordered.index(loaded) + 1, fetch) - with tempfile.TemporaryDirectory() as tmp: - self.assertEqual(run_guard(write_tmp_log(tmp, "\n".join(reordered) + "\n"), 0).returncode, 1) + self.assert_rejected("\n".join(reordered) + "\n") def test_wrong_expected_lockfile_is_rejected(self): with tempfile.TemporaryDirectory() as tmp: @@ -159,8 +154,7 @@ def test_lockfile_path_suffix_is_not_accepted(self): "Scanning Cargo.lock for vulnerabilities", "Scanning /a/b/Cargo.lock for vulnerabilities", ) - with tempfile.TemporaryDirectory() as tmp: - self.assertEqual(run_guard(write_tmp_log(tmp, text), 0, "/b/Cargo.lock").returncode, 1) + self.assert_rejected(text, "/b/Cargo.lock") def test_absolute_expected_lockfile_is_accepted(self): text = clean_text().replace( diff --git a/scripts/tests/audit-release-gate/test_workflows.py b/scripts/tests/audit-release-gate/test_workflows.py index 474425a3..41610144 100644 --- a/scripts/tests/audit-release-gate/test_workflows.py +++ b/scripts/tests/audit-release-gate/test_workflows.py @@ -9,7 +9,6 @@ PyYAML is a test-only dependency. `on:` is a YAML 1.1 boolean key, so the trigger block is read through the `trigger()` helper rather than by key name. """ -import copy import unittest from pathlib import Path @@ -343,6 +342,7 @@ def assert_rejected(self, checker, doc, message): checker(doc) def test_disabled_or_error_masked_offline_tests_are_rejected(self): + raw = TESTS.read_text() for mutation in ("job-error-mask", "step-error-mask", "conditional-skip"): with self.subTest(mutation=mutation): doc = load(TESTS) @@ -354,7 +354,7 @@ def test_disabled_or_error_masked_offline_tests_are_rejected(self): else: offline["if"] = False with self.assertRaises(AssertionError): - assert_tests_workflow(doc, TESTS.read_text()) + assert_tests_workflow(doc, raw) def test_removing_gate_is_rejected(self): def mutate(doc): From 0fee406e44a8a2869bd6a2560332dd494c99743e Mon Sep 17 00:00:00 2001 From: Lucas Pimentel Date: Wed, 28 Oct 2026 12:13:02 -0400 Subject: [PATCH 3/3] Streamline release-gate regression tests --- scripts/tests/audit-release-gate/README.md | 115 +-- .../fixtures/fake-scanner/cargo-audit | 182 +--- .../fixtures/logs/README.md | 46 - .../fixtures/logs/malformed-lock-rc2.log | 16 - .../fixtures/logs/missing-lock-rc2.log | 7 - .../audit-release-gate/fixtures/stubs/cargo | 25 +- .../audit-release-gate/fixtures/stubs/curl | 45 +- .../fixtures/stubs/sha256sum | 28 +- .../audit-release-gate/fixtures/stubs/timeout | 33 - .../tests/audit-release-gate/network_smoke.py | 193 ++-- .../audit-release-gate/test_release_gate.py | 750 ++++++--------- .../audit-release-gate/test_report_guard.py | 302 ++---- .../audit-release-gate/test_workflows.py | 869 +++++------------- 13 files changed, 752 insertions(+), 1859 deletions(-) delete mode 100644 scripts/tests/audit-release-gate/fixtures/logs/README.md delete mode 100644 scripts/tests/audit-release-gate/fixtures/logs/malformed-lock-rc2.log delete mode 100644 scripts/tests/audit-release-gate/fixtures/logs/missing-lock-rc2.log delete mode 100755 scripts/tests/audit-release-gate/fixtures/stubs/timeout diff --git a/scripts/tests/audit-release-gate/README.md b/scripts/tests/audit-release-gate/README.md index 8789ab42..4c175f2e 100644 --- a/scripts/tests/audit-release-gate/README.md +++ b/scripts/tests/audit-release-gate/README.md @@ -1,84 +1,61 @@ -# Audit release gate tests +# Audit release-gate tests -Tests for the strict cargo-audit release gate in -[`scripts/cargo-audit-release-gate.sh`](../../cargo-audit-release-gate.sh) and its -pure verdict guard -[`scripts/cargo-audit-report-guard.sh`](../../cargo-audit-report-guard.sh), plus -structural checks that the gate is wired into the protected publication jobs -correctly. +Exercise the real release gate offline with disposable git repositories and +fixture-only scanner/download doubles. No test publishes, signs or uploads. -## Offline suite (default) +## Run -Deterministic, no network, no real scanner, no publishers. It runs the real -production helper against temporary git repositories, a fixture scanner archive, -and local PATH test doubles, and it checks the real workflow YAML structure. +Activate a disposable virtual environment outside the checkout and install +`PyYAML==6.0.2`. Use that environment's Python for the offline suite: ```sh -python3 -m pip install 'PyYAML==6.0.2' -python3 -m unittest discover -v -s scripts/tests/audit-release-gate -p 'test_*.py' +PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -v -s scripts/tests/audit-release-gate -p 'test_*.py' +# Opt-in: requires cargo and network access, uses isolated fixture lockfiles. +PYTHONDONTWRITEBYTECODE=1 python3 scripts/tests/audit-release-gate/network_smoke.py ``` -PyYAML is a test-only dependency used by `test_workflows.py`. - | File | Coverage | |---|---| -| `test_report_guard.py` | Guard verdicts for captured real-scanner logs plus synthetic contract fixtures: nonzero advisory/dependency counts, exact expected-lockfile matching, marker order, hidden crates.io diagnostics, exit-code validation, shell sentinel behavior | -| `test_release_gate.py` | Helper control flow: download/checksum/version/prerequisite failures, hidden index errors, missing markers, fetch failure, source and lockfile identity, bounded network and scanner invocations, an inherited warm cache, missing `timeout`, scanner timeout statuses, substituted lockfiles, fresh per-execution directories, retry, guard robustness | -| `test_workflows.py` | Gate placement, exact command, unconditional step, bash shell, and pinning in `publish.yml`, `release-datadog-serverless-compat.yml`, `build-datadog-serverless-compat.yml`, and `audit-release-gate-tests.yml`, plus negative mutations for command masking, conditions, job-level error ignoring, cross-run artifact downloads, and checkout refs | - -The test doubles shape only the helper's inputs (via `PATH` and fixture files). -The production helper reads no bypass environment variable, and the tests add -none. - -### Production time bounds - -The gate bounds every potentially hanging step. These values are production -constants with no environment override, so a bypass cannot shorten them: - -- `curl --connect-timeout 15 --max-time 180` for the pinned archive download. -- `timeout --kill-after=10s 5m audit ...` for the dependency scan. - `timeout` is a required tool; a killed scan exits 124 (SIGTERM) or 137 - (SIGKILL), both of which the guard rejects before the publication sentinel. - -## Opt-in network smoke - -Downloads the real pinned cargo-audit archive, verifies its reviewed SHA-256, -clones a fresh RustSec advisory database, updates the crates.io index, and scans -small fixture lockfiles (clean, vulnerable, informational warning, and an -existing-ignore policy). It requires network access and a working Rust -toolchain, and it never publishes, signs, or uploads anything. - -```sh -python3 scripts/tests/audit-release-gate/network_smoke.py -``` - -This suite is the runtime proof for the guard contract: it asserts that the real -scanner prints a nonzero `Loaded ... security advisories` line and an absolute -`Scanning /.../Cargo.lock for vulnerabilities ([1-9][0-9]* crate dependencies)` -line on the current machine. The captured logs under `fixtures/logs` are real -scanner output but were captured with a relative `--file Cargo.lock`; the -synthetic variants in `test_report_guard.py` are contract fixtures, not runtime -proof. See `fixtures/logs/README.md`. - -The smoke suite never expects the repository's own `Cargo.lock` to pass; it uses -isolated fixtures only. Run it from a normal checkout with `cargo` on `PATH`. - -## Shell lint +| `test_release_gate.py` | Identity, installation, verdicts, sentinel blocking, fresh state/retry, cleanup, pins and command bounds | +| `test_report_guard.py` | Captured verdicts, synthetic incomplete scans, ordered markers, counts, exact paths and status/usage errors | +| `test_workflows.py` | Inline ordering, approvals, permissions, source/artifact binding and locked builds, with representative mutations | +| `network_smoke.py` | Real clean/retry, findings/committed ignore, informational warning, and scanner/database identity | + +## Captured log provenance + +`fixtures/logs/` contains terminal output from the official cargo-audit 0.22.2 +Linux x86_64 musl archive, SHA-256 +`7fb9497f8594b389e5fce5ef9b92db08432996895b2e0c5a0167a69ed445c428`. +Capture combined stdout/stderr from a temporary fixture using `cargo-audit audit +--file Cargo.lock --db --url https://github.com/RustSec/advisory-db.git +--format terminal --color never`. Index failure was induced with a directory at +`CARGO_HOME/.package-cache`; configuration cases disabled fetching or yanked checks. +The relative captured path is expected as `Cargo.lock`; embedded scratch paths +are not assertions. Retained logs are unchanged from the original captures. + +| Log | Scanner status | Guard verdict | +|---|---|---| +| `clean-success.log` | 0 | pass | +| `warning-only-success.log` | 0 | pass | +| `vulnerable-failure.log` | 1 | reject | +| `index-error-rc0.log` | 0 | reject | +| `quiet-output-rc0.log` | 0 | reject | +| `disabled-yanked-rc0.log` | 0 | reject | +| `disabled-fetch-rc0.log` | 0 | reject | + +## Lint and limits ```sh shellcheck scripts/cargo-audit-release-gate.sh scripts/cargo-audit-report-guard.sh \ scripts/tests/audit-release-gate/fixtures/fake-scanner/cargo-audit \ - scripts/tests/audit-release-gate/fixtures/stubs/* + scripts/tests/audit-release-gate/fixtures/stubs/{cargo,curl,sha256sum} ``` -## Limits - -- The guard is deliberately coupled to the pinned scanner's human-readable - terminal format. Re-review `cargo-audit-report-guard.sh` and its tests whenever - the pinned cargo-audit version changes. -- The offline workflow checks are structural. They prove the gate step exists in - the right job at the right position with the exact reviewed command; they do - not execute GitHub Actions. -- The opt-in smoke suite validates the scanner contract on the current machine. - It does not validate GitHub runner images, npm/OIDC publishing, or - action-gh-release. +Captured logs are real output; in-memory variants are synthetic contract fixtures. +Only the opt-in smoke executes the real scanner and proves current output shapes, +nonzero counts and absolute scan paths. The guard remains coupled to 0.22.2. +Timeout statuses are replayed, not actual timer/signal tests; bounds are checked +on command tokens with comments excluded. Workflow checks cover publication and +build wiring, not the tests workflow itself or GitHub-runner integration. +The repository lockfile need not +pass: the smoke suite uses fixtures, not the current unresolved advisories. diff --git a/scripts/tests/audit-release-gate/fixtures/fake-scanner/cargo-audit b/scripts/tests/audit-release-gate/fixtures/fake-scanner/cargo-audit index 78b50aee..8afd3686 100755 --- a/scripts/tests/audit-release-gate/fixtures/fake-scanner/cargo-audit +++ b/scripts/tests/audit-release-gate/fixtures/fake-scanner/cargo-audit @@ -1,155 +1,49 @@ #!/usr/bin/env bash # -# Test double for the pinned cargo-audit CLI. -# -# This file is used only by scripts/tests/audit-release-gate. Production never -# runs it. It reads FAKE_SCANNER_MODE from the environment so the offline test -# suite can drive every controlled scenario (clean scan, findings, hidden index -# error, missing progress markers, fetch failure, wrong advisory origin) without -# a network or a real scanner. -# -# Modes: -# success clean scan, full progress sequence, exit 0 -# warning clean scan with an allowed informational warning, exit 0 -# findings report a vulnerability, exit 1 -# index-error exit 0 while hiding a crates.io index failure -# yanked-error exit 0 while hiding an incomplete yanked check -# quiet exit 0 with no output at all -# disabled-yanked exit 0 with the Updating marker absent -# disabled-fetch exit 0 with the Fetching and Updating markers absent -# fetch-error fail to fetch the advisory database, exit 1 -# config-error fail to load configuration, exit 1 -# wrong-origin exit 0 after creating a database with an untrusted origin -# warm-authorize succeed only when the requested --db already contains a warm -# git database; otherwise simulate an unavailable network -# refresh and exit 1. Used to prove a warm inherited cache is -# not reused by the gate. -# shellcheck disable=SC2016 # the backticks are literal scanner output, not expansion -set -uo pipefail - -mode="${FAKE_SCANNER_MODE:-success}" -version_line="${FAKE_SCANNER_VERSION_LINE:-cargo-audit-audit 0.22.2}" - -if [ "${1:-}" = "audit" ] && [ "${2:-}" = "--version" ]; then - printf '%s\n' "$version_line" +# Replay double for the pinned cargo-audit binary, used only by +# test_release_gate.py and never by production. Fixture-only variables: +# FAKE_VERSION line printed for `audit --version` +# FAKE_RECORD file that receives one record block per scan +# FAKE_LOG captured scanner log to replay +# FAKE_RC exit status to return after replaying (default 0) +# FAKE_NO_DB when set, create no advisory database (fetch failure) +# FAKE_ORIGIN origin URL of the created advisory database +# FAKE_SCAN_PATH lockfile path shown on the Scanning line instead of --file +set -euo pipefail +export GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_SYSTEM=/dev/null + +if [ "$*" = "audit --version" ]; then + printf '%s\n' "${FAKE_VERSION:-cargo-audit-audit 0.22.2}" exit 0 fi -db="" -lockfile="" -previous="" -for argument in "$@"; do +db="" lockfile="" previous="" +for arg in "$@"; do case "$previous" in - --db) db="$argument" ;; - --file) lockfile="$argument" ;; + --db) db="$arg" ;; + --file) lockfile="$arg" ;; esac - previous="$argument" + previous="$arg" done -# The pinned scanner echoes the --file argument it was given. Tests can override -# the echoed path to prove the gate's lockfile binding is enforced end to end. -scan_path="${FAKE_SCANNER_SCAN_PATH:-$lockfile}" - -if [ -n "${FAKE_SCANNER_RECORD:-}" ]; then - { - printf 'db=%s\n' "$db" - printf 'file=%s\n' "$lockfile" - printf 'cargo_home=%s\n' "${CARGO_HOME:-}" - printf 'args=%s\n' "$*" - printf 'mode=%s\n' "$mode" - } >>"$FAKE_SCANNER_RECORD" +# Record the state this scan inherited, before the database is created. +home="${CARGO_HOME:-}" +{ + printf -- '---\ndb=%s\nhome=%s\n' "$db" "$home" + if [ -e "$db" ]; then echo db_existed=yes; else echo db_existed=no; fi + if [ -d "$home" ] && [ -z "$(ls -A -- "$home")" ]; then echo home_empty=yes; else echo home_empty=no; fi + printf 'arg=%s\n' "$@" +} >>"$FAKE_RECORD" + +if [ -z "${FAKE_NO_DB:-}" ]; then + git init --quiet "$db" + git -C "$db" remote add origin "${FAKE_ORIGIN:-https://github.com/RustSec/advisory-db.git}" + git -C "$db" -c user.name=fixture -c user.email=fixture@example.invalid \ + commit --quiet --allow-empty -m 'fixture advisory database' fi -make_fake_db() { - [ -n "$db" ] || return 0 - mkdir -p "$db" - if [ ! -d "$db/.git" ]; then - git -C "$db" init --quiet - git -C "$db" remote add origin https://github.com/RustSec/advisory-db.git - printf 'fixture advisory database\n' >"$db/README.md" - git -C "$db" -c user.email=fixture@example.invalid -c user.name=fixture add README.md - git -C "$db" -c user.email=fixture@example.invalid -c user.name=fixture commit --quiet -m 'fixture advisory database' - fi -} - -emit_progress() { - printf ' Fetching advisory database from `https://github.com/RustSec/advisory-db.git`\n' - printf ' Loaded 1380 security advisories (from %s)\n' "$db" - printf ' Updating crates.io index\n' - printf ' Scanning %s for vulnerabilities (1 crate dependencies)\n' "$scan_path" -} - -case "$mode" in - success) - make_fake_db - emit_progress - exit 0 - ;; - warning) - make_fake_db - emit_progress - printf '\nwarning: 1 allowed warning found\n' - exit 0 - ;; - findings) - make_fake_db - emit_progress - printf '\nCrate: openssl\nVersion: 0.10.38\nTitle: fixture advisory\nID: RUSTSEC-0000-0000\n' - printf '\nerror: 1 vulnerability found!\n' - exit 1 - ;; - index-error) - make_fake_db - emit_progress - printf "warning: couldn't update crates.io index: fixture I/O failure\n" - exit 0 - ;; - yanked-error) - make_fake_db - emit_progress - printf "error: couldn't check if the package is yanked: fixture index miss\n" - exit 0 - ;; - quiet) - make_fake_db - exit 0 - ;; - disabled-yanked) - make_fake_db - printf ' Fetching advisory database from `https://github.com/RustSec/advisory-db.git`\n' - printf ' Loaded 1380 security advisories (from %s)\n' "$db" - printf ' Scanning %s for vulnerabilities (1 crate dependencies)\n' "$scan_path" - exit 0 - ;; - disabled-fetch) - printf ' Loaded 1380 security advisories (from %s)\n' "$db" - printf ' Scanning %s for vulnerabilities (1 crate dependencies)\n' "$scan_path" - exit 0 - ;; - fetch-error) - printf 'error: failed to fetch advisory database\n' >&2 - exit 1 - ;; - config-error) - printf 'error: failed to load audit configuration\n' >&2 - exit 1 - ;; - wrong-origin) - make_fake_db - git -C "$db" remote set-url origin https://example.invalid/advisory-db.git - emit_progress - exit 0 - ;; - warm-authorize) - if [ -n "$db" ] && [ -d "$db/.git" ]; then - emit_progress - exit 0 - fi - printf 'error: failed to fetch advisory database\n' >&2 - exit 1 - ;; - *) - printf 'fixture scanner: unknown FAKE_SCANNER_MODE=%s\n' "$mode" >&2 - exit 64 - ;; -esac +scan_path="${FAKE_SCAN_PATH:-$lockfile}" +while IFS= read -r line || [ -n "$line" ]; do + printf '%s\n' "${line/Scanning Cargo.lock for/"Scanning $scan_path for"}" +done <"$FAKE_LOG" +exit "${FAKE_RC:-0}" diff --git a/scripts/tests/audit-release-gate/fixtures/logs/README.md b/scripts/tests/audit-release-gate/fixtures/logs/README.md deleted file mode 100644 index d459eaf7..00000000 --- a/scripts/tests/audit-release-gate/fixtures/logs/README.md +++ /dev/null @@ -1,46 +0,0 @@ -# Captured scanner logs - -These logs are real terminal output captured from the pinned `cargo-audit 0.22.2` -official Linux x86_64 musl binary (`cargo-audit-audit 0.22.2`, archive SHA-256 -`7fb9497f8594b389e5fce5ef9b92db08432996895b2e0c5a0167a69ed445c428`). The guard -tests feed them to `scripts/cargo-audit-report-guard.sh` together with the -recorded scanner exit code and the expected lockfile path, so guard behavior is -verified against real scanner text rather than hand-written approximations. - -They are not "expected output" for a healthy network run; each one captures a -specific real scanner behavior. They were captured with a relative -`--file Cargo.lock`, so the tests pass `Cargo.lock` as the expected lockfile path -for these files. Production always passes an absolute path; the opt-in -`network_smoke.py` suite is the runtime proof for the absolute-path line. - -## Provenance categories - -- This directory holds **captured runtime logs**: real output from the pinned - binary, byte-for-byte as emitted (placeholder scratch paths aside). -- `test_report_guard.py` also derives **synthetic contract fixtures** in memory - (zero counts, missing counts, reordered markers, substituted lockfile paths). - Those are contract tests for rejection paths, not captured scanner output. -- `network_smoke.py` is the **runtime proof** that connects the two: it runs the - real gate end to end against the real binary and asserts the nonzero counts and - absolute lockfile path the guard requires. - -| Log | Scanner exit | Guard verdict | What it captures | -|---|---|---|---| -| `clean-success.log` | 0 | pass | Clean lockfile, full Fetching/Loaded/Updating/Scanning sequence | -| `warning-only-success.log` | 0 | pass | Informational warning is not fatal when the scan exits zero | -| `vulnerable-failure.log` | 1 | reject | Ordinary advisory rejects the scan | -| `malformed-lock-rc2.log` | 2 | reject | Malformed Cargo.lock | -| `missing-lock-rc2.log` | 2 | reject | Missing Cargo.lock | -| `index-error-rc0.log` | 0 | reject | crates.io index I/O failure hidden behind a zero exit status | -| `quiet-output-rc0.log` | 0 | reject | `[output] quiet = true` suppresses the progress sequence | -| `disabled-yanked-rc0.log` | 0 | reject | `[yanked] enabled = false` removes the Updating marker | -| `disabled-fetch-rc0.log` | 0 | reject | `[database] fetch = false` removes the Fetching marker | - -Capture method: a small fixture lockfile in a temporary directory was scanned with -`cargo-audit audit --file Cargo.lock --db --url -https://github.com/RustSec/advisory-db.git --format terminal --color never`, -redirecting combined stdout and stderr. The index-error and configuration logs -used the documented failure injections (a directory in place of -`CARGO_HOME/.package-cache`, and scanner configuration overrides). The temporary -paths embedded in the logs are scratch paths from the capture run and are not -part of the assertions. diff --git a/scripts/tests/audit-release-gate/fixtures/logs/malformed-lock-rc2.log b/scripts/tests/audit-release-gate/fixtures/logs/malformed-lock-rc2.log deleted file mode 100644 index 5ef6bc40..00000000 --- a/scripts/tests/audit-release-gate/fixtures/logs/malformed-lock-rc2.log +++ /dev/null @@ -1,16 +0,0 @@ - Fetching advisory database from `https://github.com/RustSec/advisory-db.git` - Loaded 1380 security advisories (from /tmp/cargo-release-preflight-vrpsnQ/rustsec-db) - Updating crates.io index -error: not found: Couldn't load Cargo.lock -Caused by: - -> I/O operation failed: parse error: TOML parse error at line 1, column 5 - | -1 | not a valid lockfile = [ - | ^ -key with no value, expected `=` - - -> parse error: TOML parse error at line 1, column 5 - | -1 | not a valid lockfile = [ - | ^ -key with no value, expected `=` diff --git a/scripts/tests/audit-release-gate/fixtures/logs/missing-lock-rc2.log b/scripts/tests/audit-release-gate/fixtures/logs/missing-lock-rc2.log deleted file mode 100644 index 0ce9b776..00000000 --- a/scripts/tests/audit-release-gate/fixtures/logs/missing-lock-rc2.log +++ /dev/null @@ -1,7 +0,0 @@ - Fetching advisory database from `https://github.com/RustSec/advisory-db.git` - Loaded 1380 security advisories (from /tmp/cargo-release-preflight-vrpsnQ/rustsec-db) - Updating crates.io index -error: not found: Couldn't load missing.lock -Caused by: - -> I/O operation failed: I/O operation failed: entity not found - -> I/O operation failed: entity not found diff --git a/scripts/tests/audit-release-gate/fixtures/stubs/cargo b/scripts/tests/audit-release-gate/fixtures/stubs/cargo index 79b278e0..4ae3e558 100755 --- a/scripts/tests/audit-release-gate/fixtures/stubs/cargo +++ b/scripts/tests/audit-release-gate/fixtures/stubs/cargo @@ -1,19 +1,8 @@ #!/usr/bin/env bash -# -# Test double for cargo used by the audit release gate tests. -# -# The pinned prebuilt cargo-audit shells out to `cargo -V`. FAKE_CARGO_MODE=fail -# simulates a missing or broken toolchain. -set -euo pipefail - -case "${FAKE_CARGO_MODE:-ok}" in - ok) printf 'cargo 1.99.0 (fixture)\n' ;; - fail) - printf 'error: fixture cargo is unavailable\n' >&2 - exit 1 - ;; - *) - printf 'cargo stub: unknown FAKE_CARGO_MODE\n' >&2 - exit 64 - ;; -esac +# cargo double for test_release_gate.py: answers `cargo -V`, or simulates a +# missing toolchain when FAKE_CARGO_FAIL is set. +if [ -n "${FAKE_CARGO_FAIL:-}" ]; then + echo 'error: fixture cargo is unavailable' >&2 + exit 1 +fi +echo 'cargo 1.99.0 (fixture)' diff --git a/scripts/tests/audit-release-gate/fixtures/stubs/curl b/scripts/tests/audit-release-gate/fixtures/stubs/curl index 1cb9b1b7..a6a90119 100755 --- a/scripts/tests/audit-release-gate/fixtures/stubs/curl +++ b/scripts/tests/audit-release-gate/fixtures/stubs/curl @@ -1,38 +1,13 @@ #!/usr/bin/env bash -# -# Test double for curl used by the audit release gate tests. -# -# It copies the fixture archive to the gate's requested --output path so the -# gate's download, checksum, extraction, and installation flow is exercised -# without network access. FAKE_CURL_MODE=fail simulates a download failure. -# FAKE_CURL_RECORD captures the full argument list so tests can assert the -# reviewed network time bounds are present. +# curl double for test_release_gate.py: records argv (one arg per line), then +# copies FAKE_ARCHIVE to the --output path. FAKE_CURL_FAIL simulates a failure. set -euo pipefail - -output="" -previous="" -for argument in "$@"; do - case "$previous" in - -o | --output) output="$argument" ;; - esac - previous="$argument" +{ printf -- '---\n'; printf 'arg=%s\n' "$@"; } >>"$FAKE_CURL_RECORD" +[ -z "${FAKE_CURL_FAIL:-}" ] || { echo 'curl: (22) fixture download failure' >&2; exit 22; } +output="" previous="" +for arg in "$@"; do + [ "$previous" != --output ] || output="$arg" + previous="$arg" done - -if [ -n "${FAKE_CURL_RECORD:-}" ]; then - printf '%s\n' "$*" >>"$FAKE_CURL_RECORD" -fi - -case "${FAKE_CURL_MODE:-ok}" in - ok) ;; - fail) - printf 'curl: (22) fixture download failure\n' >&2 - exit 22 - ;; - *) - printf 'curl stub: unknown FAKE_CURL_MODE\n' >&2 - exit 64 - ;; -esac - -[ -n "$output" ] || { printf 'curl stub: no --output argument\n' >&2; exit 2; } -cp "${FAKE_CURL_SOURCE:?FAKE_CURL_SOURCE must point at the fixture archive}" "$output" +[ -n "$output" ] || { echo 'curl stub: missing --output' >&2; exit 2; } +cp -- "$FAKE_ARCHIVE" "$output" diff --git a/scripts/tests/audit-release-gate/fixtures/stubs/sha256sum b/scripts/tests/audit-release-gate/fixtures/stubs/sha256sum index 2581b25f..efd5e539 100755 --- a/scripts/tests/audit-release-gate/fixtures/stubs/sha256sum +++ b/scripts/tests/audit-release-gate/fixtures/stubs/sha256sum @@ -1,24 +1,10 @@ #!/usr/bin/env bash -# -# Test double for sha256sum used by the audit release gate tests. -# -# The gate hashes real files (the repository lockfile and policy) and the -# downloaded scanner archive. Real hashing is delegated to $REAL_SHA256SUM so the -# source/lockfile identity checks are genuinely exercised. Only the fixture -# archive, identified by its real digest, is substituted with the reviewed pin -# (FAKE_ARCHIVE_SHA_MODE=ok) or with a wrong digest (FAKE_ARCHIVE_SHA_MODE=bad). +# sha256sum double for test_release_gate.py: reports FAKE_ARCHIVE_SHA256 for the +# downloaded */cargo-audit.tgz only and delegates every other call (lockfile, +# policy, and stdin hashing) to the real binary at REAL_SHA256SUM. set -euo pipefail - -if [ "$#" -ge 1 ] && [ -f "$1" ]; then - actual="$("${REAL_SHA256SUM:?REAL_SHA256SUM must point at the real sha256sum}" "$1" | cut -d' ' -f1)" - if [ -n "${FAKE_ARCHIVE_REAL_SHA:-}" ] && [ "$actual" = "$FAKE_ARCHIVE_REAL_SHA" ]; then - case "${FAKE_ARCHIVE_SHA_MODE:-ok}" in - ok) printf '%s %s\n' "${FAKE_ARCHIVE_SHA:?FAKE_ARCHIVE_SHA must be set}" "$1" ;; - bad) printf '%064d %s\n' 0 "$1" ;; - *) printf 'sha256sum stub: unknown FAKE_ARCHIVE_SHA_MODE\n' >&2; exit 64 ;; - esac - exit 0 - fi +if [ "$#" -eq 1 ] && [[ "$1" == */cargo-audit.tgz ]]; then + printf '%s %s\n' "$FAKE_ARCHIVE_SHA256" "$1" + exit 0 fi - -exec "${REAL_SHA256SUM:?REAL_SHA256SUM must point at the real sha256sum}" "$@" +exec "$REAL_SHA256SUM" "$@" diff --git a/scripts/tests/audit-release-gate/fixtures/stubs/timeout b/scripts/tests/audit-release-gate/fixtures/stubs/timeout deleted file mode 100755 index 2a1999b7..00000000 --- a/scripts/tests/audit-release-gate/fixtures/stubs/timeout +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash -# -# Test double for coreutils timeout used by the audit release gate tests. -# -# By default (FAKE_TIMEOUT_MODE=exec) it delegates to the real timeout binary so -# the gate's bounded scanner invocation behaves exactly as in production. -# FAKE_TIMEOUT_MODE=fail124 and fail137 simulate a scan killed by the timeout -# (SIGTERM and SIGKILL respectively) without waiting for the real 5 minute bound. -# FAKE_TIMEOUT_RECORD captures the full argument list so tests can assert the -# reviewed --kill-after and duration bounds are present. -set -euo pipefail - -if [ -n "${FAKE_TIMEOUT_RECORD:-}" ]; then - printf '%s\n' "$*" >>"$FAKE_TIMEOUT_RECORD" -fi - -case "${FAKE_TIMEOUT_MODE:-exec}" in - exec) - exec "${REAL_TIMEOUT:?REAL_TIMEOUT must point at the real timeout}" "$@" - ;; - fail124) - printf 'timeout stub: simulated scanner timeout (SIGTERM)\n' >&2 - exit 124 - ;; - fail137) - printf 'timeout stub: simulated scanner kill (SIGKILL)\n' >&2 - exit 137 - ;; - *) - printf 'timeout stub: unknown FAKE_TIMEOUT_MODE\n' >&2 - exit 64 - ;; -esac diff --git a/scripts/tests/audit-release-gate/network_smoke.py b/scripts/tests/audit-release-gate/network_smoke.py index 52642743..822870c6 100644 --- a/scripts/tests/audit-release-gate/network_smoke.py +++ b/scripts/tests/audit-release-gate/network_smoke.py @@ -1,19 +1,8 @@ -"""Opt-in network smoke suite for the cargo-audit release gate. +"""Opt-in real-scanner tests; excluded from offline test_*.py discovery. -This suite is not part of the deterministic offline run. It downloads the real -pinned cargo-audit archive, verifies its reviewed checksum, installs it, clones a -fresh RustSec advisory database, updates the crates.io index, and scans small -fixture lockfiles. It requires a working Rust toolchain and network access. - -Run it from the repository root with: - - python3 scripts/tests/audit-release-gate/network_smoke.py - -It never publishes, signs, uploads, or pushes anything, and it never asserts that -the repository's own Cargo.lock passes: that lock is expected to be blocked while -unfixed advisories remain. - -The file is deliberately named so that `unittest discover -p 'test_*.py'` skips it. +Run from the repository root with cargo and network access. Each gate invocation +uses the real pinned download, checksum, fresh advisory DB and crates.io index. +Only temporary fixture lockfiles are scanned; nothing is published or uploaded. """ import os import re @@ -22,148 +11,82 @@ import unittest from pathlib import Path -HERE = Path(__file__).resolve().parent -REPO = HERE.parents[2] -GATE = REPO / "scripts" / "cargo-audit-release-gate.sh" - -SCANNER_VERSION_LINE = "cargo-audit-audit 0.22.2" -ADVISORY_DB_URL = "https://github.com/RustSec/advisory-db.git" -LOCK_HEADER = 'version = 3\n\n[[package]]\nname = "smoke-fixture"\nversion = "0.1.0"\n' -DEFAULT_POLICY = "[advisories]\nignore = []\n" - +GATE = Path(__file__).resolve().parents[2] / "cargo-audit-release-gate.sh" +VERSION = "cargo-audit-audit 0.22.2" +TRUSTED_DB = "https://github.com/RustSec/advisory-db.git" +LOCK = 'version = 3\n\n[[package]]\nname = "smoke-fixture"\nversion = "0.1.0"\n' GIT_ENV = {**os.environ, "GIT_CONFIG_GLOBAL": os.devnull, "GIT_CONFIG_SYSTEM": os.devnull} -def registry_package(name, version): - return ( - f'\n[[package]]\nname = "{name}"\nversion = "{version}"\n' - 'source = "registry+https://github.com/rust-lang/crates.io-index"\n' - ) - - def git(repo, *args): - return subprocess.run( - ["git", "-C", str(repo), *args], - check=True, - capture_output=True, - text=True, - env=GIT_ENV, - ) + return subprocess.run(["git", "-C", str(repo), *args], check=True, + capture_output=True, text=True, env=GIT_ENV).stdout.strip() def commit_all(repo, message): git(repo, "add", "-A") - git( - repo, - "-c", "user.email=smoke@example.invalid", - "-c", "user.name=smoke", - "commit", "--quiet", "-m", message, - ) - return git(repo, "rev-parse", "HEAD").stdout.strip() + git(repo, "-c", "user.email=smoke@example.invalid", "-c", "user.name=smoke", + "commit", "--quiet", "-m", message) + return git(repo, "rev-parse", "HEAD") -def make_repo(root, lock_text, policy_text=DEFAULT_POLICY): - repo = Path(root) / "repo" - repo.mkdir() - (repo / ".cargo").mkdir() - (repo / "Cargo.lock").write_text(lock_text) - (repo / ".cargo" / "audit.toml").write_text(policy_text) - subprocess.run( - ["git", "init", "--quiet", "--initial-branch=main", str(repo)], - check=True, - capture_output=True, - text=True, - env=GIT_ENV, - ) - return repo, commit_all(repo, "smoke fixture") +def registry_package(name, version): + return (f'\n[[package]]\nname = "{name}"\nversion = "{version}"\n' + 'source = "registry+https://github.com/rust-lang/crates.io-index"\n') -def run_gate(repo, sha): - return subprocess.run( - ["bash", str(GATE), sha], - cwd=str(repo), - capture_output=True, - text=True, - check=False, - timeout=900, - ) +def make_repo(root, lock): + repo = Path(root) / "repo" + (repo / ".cargo").mkdir(parents=True) + (repo / "Cargo.lock").write_text(lock) + (repo / ".cargo" / "audit.toml").write_text("[advisories]\nignore = []\n") + git(repo, "init", "--quiet", "--initial-branch=main") + return repo, commit_all(repo, "smoke fixture") class NetworkSmoke(unittest.TestCase): def setUp(self): - self._tmp = tempfile.TemporaryDirectory(prefix="audit-gate-smoke.") - self.addCleanup(self._tmp.cleanup) + tmp = tempfile.TemporaryDirectory(prefix="audit-gate-smoke.") + self.addCleanup(tmp.cleanup) + self.root = tmp.name def assert_scanner_identity(self, output): - self.assertIn(f"scanner-version: {SCANNER_VERSION_LINE}", output) - revision = re.search(r"^advisory-db-revision: ([0-9a-f]{40,64})$", output, re.MULTILINE) - self.assertIsNotNone(revision, output) - self.assertIn(f"advisory-db-origin: {ADVISORY_DB_URL}", output) - # A per-execution fetch must be visible in the displayed log. + self.assertIn(f"scanner-version: {VERSION}", output) + self.assertIn(f"advisory-db-origin: {TRUSTED_DB}", output) self.assertIn("Fetching advisory database", output) - # The real scanner must report a nonzero advisory count and scan the - # absolute lockfile path with a nonzero crate dependency count. This is - # the runtime proof for the structured report-guard contract. - self.assertIsNotNone( - re.search(r"^\s*Loaded [1-9][0-9]* security advisories", output, re.MULTILINE), - output, - ) - self.assertIsNotNone( - re.search( - r"^\s*Scanning /.*/Cargo\.lock for vulnerabilities \([1-9][0-9]* crate dependencies\)$", - output, - re.MULTILINE, - ), - output, + self.assertRegex(output, r"(?m)^advisory-db-revision: (?:[0-9a-f]{40}|[0-9a-f]{64})$") + self.assertRegex(output, r"(?m)^\s*Loaded [1-9][0-9]* security advisories") + self.assertRegex( + output, r"(?m)^\s*Scanning /.*/Cargo\.lock for vulnerabilities " + r"\([1-9][0-9]* crate dependencies\)$", ) - def test_clean_fixture_passes_and_refreshes(self): - repo, head = make_repo(self._tmp.name, LOCK_HEADER) - first = run_gate(repo, head) - self.assertEqual(first.returncode, 0, first.stdout + first.stderr) - self.assertIn("result: PASS", first.stdout) - self.assert_scanner_identity(first.stdout) - - # Retrying the gate re-fetches advisory data instead of reusing state. - second = run_gate(repo, head) - self.assertEqual(second.returncode, 0, second.stdout + second.stderr) - self.assert_scanner_identity(second.stdout) - - def test_vulnerable_fixture_blocks_publication(self): - lock = LOCK_HEADER + registry_package("openssl", "0.10.38") - repo, head = make_repo(self._tmp.name, lock) - result = run_gate(repo, head) - self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr) - self.assertIn("result: FAIL", result.stdout + result.stderr) - self.assertRegex(result.stdout, r"RUSTSEC-\d{4}-\d{4}") + def scan(self, repo, sha, accepted=True): + result = subprocess.run(["bash", str(GATE), sha], cwd=repo, capture_output=True, + text=True, timeout=900) + self.assertEqual(result.returncode == 0, accepted, result.stdout + result.stderr) + self.assertIn("result: PASS" if accepted else "result: FAIL", result.stdout + result.stderr) self.assert_scanner_identity(result.stdout) - - def test_informational_warning_fixture_passes(self): - lock = LOCK_HEADER + registry_package("paste", "1.0.15") - repo, head = make_repo(self._tmp.name, lock) - result = run_gate(repo, head) - self.assertEqual(result.returncode, 0, result.stdout + result.stderr) - self.assertIn("allowed warning found", result.stdout) - - def test_existing_ignore_policy_is_honored(self): - lock = LOCK_HEADER + registry_package("openssl", "0.10.38") - repo, head = make_repo(self._tmp.name, lock) - - blocked = run_gate(repo, head) - self.assertNotEqual(blocked.returncode, 0, blocked.stdout + blocked.stderr) - advisories = sorted(set(re.findall(r"RUSTSEC-\d{4}-\d{4}", blocked.stdout))) - self.assertTrue(advisories, blocked.stdout) - - policy = "[advisories]\nignore = [\n" - for advisory in advisories: - policy += f' "{advisory}",\n' - policy += "]\n" - (repo / ".cargo" / "audit.toml").write_text(policy) - new_head = commit_all(repo, "ignore the fixture advisories") - - allowed = run_gate(repo, new_head) - self.assertEqual(allowed.returncode, 0, allowed.stdout + allowed.stderr) - self.assertIn("result: PASS", allowed.stdout) + return result.stdout + + def test_clean_fixture_and_retry_refresh(self): + repo, sha = make_repo(self.root, LOCK) + self.scan(repo, sha) + # A retry must fetch again, not authorize from inherited state. + self.scan(repo, sha) + + def test_findings_block_until_ignore_policy_is_committed(self): + repo, sha = make_repo(self.root, LOCK + registry_package("openssl", "0.10.38")) + blocked = self.scan(repo, sha, accepted=False) + advisories = sorted(set(re.findall(r"RUSTSEC-\d{4}-\d{4}", blocked))) + self.assertTrue(advisories, blocked) + ignores = ",\n".join(f' "{advisory}"' for advisory in advisories) + (repo / ".cargo" / "audit.toml").write_text(f"[advisories]\nignore = [\n{ignores}\n]\n") + self.scan(repo, commit_all(repo, "ignore the fixture advisories")) + + def test_informational_warning_passes(self): + repo, sha = make_repo(self.root, LOCK + registry_package("paste", "1.0.15")) + self.assertIn("allowed warning found", self.scan(repo, sha)) if __name__ == "__main__": diff --git a/scripts/tests/audit-release-gate/test_release_gate.py b/scripts/tests/audit-release-gate/test_release_gate.py index 5cf09a66..3c5649d8 100644 --- a/scripts/tests/audit-release-gate/test_release_gate.py +++ b/scripts/tests/audit-release-gate/test_release_gate.py @@ -1,522 +1,298 @@ """Offline control-flow tests for scripts/cargo-audit-release-gate.sh. -These tests run the real production helper against a temporary git repository, a -fixture scanner archive, and local PATH test doubles for curl, sha256sum, and -cargo. They never touch the network, install a real scanner, run a publisher, or -read any environment variable that the production helper itself honors. The test -doubles only shape the helper's inputs and record what the helper did. - -A separate opt-in suite, network_smoke.py, exercises the real download, checksum, -database refresh, and scan path on a runner with network access. +Each test runs the real production gate inside a temporary git repository with a +tiny committed Cargo.lock and .cargo/audit.toml. The "downloaded" archive is a tar +of the replay scanner in fixtures/fake-scanner, and PATH doubles stand in only for +curl, sha256sum, and cargo; git, tar, cut, and coreutils timeout are real. The +doubles read fixture-only FAKE_* variables; the gate itself honors none of them. + +Scanner exit statuses 124 and 137 are returned by the replay scanner through the +real timeout binary. That proves the gate's exit-status handling for a killed scan, +not actual timer expiry or signal delivery; the bounds themselves are covered by +the static command test below. + +network_smoke.py is the opt-in runtime proof against the real scanner. """ import hashlib import os +import re +import shlex import shutil import subprocess +import tarfile import tempfile import unittest from pathlib import Path HERE = Path(__file__).resolve().parent -REPO = HERE.parents[2] -GATE = REPO / "scripts" / "cargo-audit-release-gate.sh" -FIXTURES = HERE / "fixtures" -FAKE_SCANNER = FIXTURES / "fake-scanner" / "cargo-audit" -STUBS = FIXTURES / "stubs" +GATE = HERE.parents[1] / "cargo-audit-release-gate.sh" +FAKE_SCANNER = HERE / "fixtures" / "fake-scanner" / "cargo-audit" +STUBS = HERE / "fixtures" / "stubs" +LOGS = HERE / "fixtures" / "logs" +# Reviewed pins, written out independently of the production script. ARCHIVE_URL = ( "https://github.com/rustsec/rustsec/releases/download/" "cargo-audit/v0.22.2/cargo-audit-x86_64-unknown-linux-musl-v0.22.2.tgz" ) ARCHIVE_SHA256 = "7fb9497f8594b389e5fce5ef9b92db08432996895b2e0c5a0167a69ed445c428" -SCANNER_VERSION_LINE = "cargo-audit-audit 0.22.2" -ADVISORY_DB_URL = "https://github.com/RustSec/advisory-db.git" -LOCK_TEXT = 'version = 3\n\n[[package]]\nname = "gate-fixture"\nversion = "0.1.0"\n' -POLICY_TEXT = '[advisories]\nignore = []\n' +SCANNER_VERSION = "cargo-audit-audit 0.22.2" +TRUSTED_DB = "https://github.com/RustSec/advisory-db.git" + +LOCK = 'version = 3\n\n[[package]]\nname = "gate-fixture"\nversion = "0.1.0"\n' +POLICY = "[advisories]\nignore = []\n" +GIT_ENV = {"GIT_CONFIG_GLOBAL": os.devnull, "GIT_CONFIG_SYSTEM": os.devnull} +REAL_SHA256SUM = shutil.which("sha256sum") +if REAL_SHA256SUM is None: + raise RuntimeError("sha256sum is required to run the offline release-gate tests") + + +def read_records(path): + """Parse '---'-separated blocks of key=value lines; 'arg' keys accumulate.""" + runs = [] + if path.exists(): + for line in path.read_text().splitlines(): + if line == "---": + runs.append({"arg": []}) + else: + key, _, value = line.partition("=") + if key == "arg": + runs[-1]["arg"].append(value) + else: + runs[-1][key] = value + return runs + + +def logical_commands(script_text): + """Join backslash continuations and shell-split each line, dropping comments.""" + joined = script_text.replace("\\\n", " ") + return [shlex.split(line, comments=True) for line in joined.splitlines()] -def sha256_file(path): - digest = hashlib.sha256() - digest.update(Path(path).read_bytes()) - return digest.hexdigest() - - -def find_real(tool): - for directory in os.environ["PATH"].split(os.pathsep): - candidate = Path(directory) / tool - if candidate.is_file() and os.access(candidate, os.X_OK): - return str(candidate) - raise RuntimeError(f"cannot find {tool} on PATH") - +class ReleaseGateTests(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory(prefix="audit-gate-test.") + self.addCleanup(tmp.cleanup) + self.root = Path(tmp.name).resolve() + self.repo = self.root / "repo" + self.lock = self.repo / "Cargo.lock" + self.policy = self.repo / ".cargo" / "audit.toml" + self.tmpdir = self.root / "tmp" + self.tmpdir.mkdir() + self.sentinel = self.root / "sentinel" + self.scan_record = self.root / "scans.txt" + self.curl_record = self.root / "curl.txt" -class GateHarness: - """Builds one isolated gate fixture: a git repo, an archive, and stubs.""" + self.policy.parent.mkdir(parents=True) + self.write_inputs() + self.git("init", "--quiet") + self.git("add", "-A") + self.git("-c", "user.name=gate", "-c", "user.email=gate@example.invalid", + "commit", "--quiet", "-m", "fixture") + self.head = self.git("rev-parse", "HEAD") - def __init__(self, root): - self.root = Path(root) - self.repo = self.root / "repo" - self.stub_dir = self.root / "stubs" self.archive = self.root / "fixture.tgz" - self.record = self.root / "scanner-record.txt" - self.sentinel = self.root / "sentinel" - self.real_sha256sum = find_real("sha256sum") - self.real_timeout = find_real("timeout") - self._make_repo() - self._make_stub_dir() - self._make_archive() - - def _git(self, *args): - env = {**os.environ, "GIT_CONFIG_GLOBAL": os.devnull, "GIT_CONFIG_SYSTEM": os.devnull} - return subprocess.run(["git", *args], capture_output=True, text=True, check=True, env=env) - - def _make_repo(self): - self.repo.mkdir() - (self.repo / ".cargo").mkdir() - (self.repo / "Cargo.lock").write_text(LOCK_TEXT) - (self.repo / ".cargo" / "audit.toml").write_text(POLICY_TEXT) - self._git("init", "--quiet", "--initial-branch=main", str(self.repo)) - self._git("-C", str(self.repo), "add", "-A") - self._git( - "-C", - str(self.repo), - "-c", - "user.email=gate@example.invalid", - "-c", - "user.name=gate", - "commit", - "--quiet", - "-m", - "fixture", - ) - self.head_sha = self._git("-C", str(self.repo), "rev-parse", "HEAD").stdout.strip() - - def _make_stub_dir(self): - self.stub_dir.mkdir() - for name in ("curl", "sha256sum", "cargo", "timeout"): - destination = self.stub_dir / name - shutil.copy(STUBS / name, destination) - destination.chmod(0o755) - - def _make_archive(self): - stage = self.root / "stage" - archive_root = stage / "cargo-audit-x86_64-unknown-linux-musl-v0.22.2" - archive_root.mkdir(parents=True) - binary = archive_root / "cargo-audit" - shutil.copy(FAKE_SCANNER, binary) - binary.chmod(0o755) - subprocess.run( - ["tar", "-czf", str(self.archive), "-C", str(stage), archive_root.name], - check=True, - ) - self.archive_real_sha = sha256_file(self.archive) - - def env(self, **overrides): - env = dict(os.environ) - env["PATH"] = f"{self.stub_dir}{os.pathsep}{env['PATH']}" - env["REAL_SHA256SUM"] = self.real_sha256sum - env["REAL_TIMEOUT"] = self.real_timeout - env["FAKE_CURL_SOURCE"] = str(self.archive) - env["FAKE_CURL_MODE"] = "ok" - env["FAKE_ARCHIVE_REAL_SHA"] = self.archive_real_sha - env["FAKE_ARCHIVE_SHA"] = ARCHIVE_SHA256 - env["FAKE_ARCHIVE_SHA_MODE"] = "ok" - env["FAKE_CARGO_MODE"] = "ok" - env["FAKE_TIMEOUT_MODE"] = "exec" - env["FAKE_SCANNER_MODE"] = "success" - env["FAKE_SCANNER_RECORD"] = str(self.record) - env["GIT_CONFIG_GLOBAL"] = os.devnull - env["GIT_CONFIG_SYSTEM"] = os.devnull - env.update(overrides) - return env - - def path_without(self, *excluded): - """Build a PATH view of the real tools plus the stubs, minus exclusions. - - Used to prove the gate's required-tool check, which cannot be exercised - by removing a tool from the inherited PATH while the stubs still provide - it. - """ - shim = self.root / ("shim-no-" + "-".join(excluded)) - if shim.exists(): - return str(shim) - shim.mkdir() - seen = set() - sources = [self.stub_dir, *os.environ["PATH"].split(os.pathsep)] - for source in sources: - directory = Path(source) - if not directory.is_dir(): - continue - for entry in sorted(directory.iterdir()): - name = entry.name - if name in excluded or name in seen: - continue + with tarfile.open(self.archive, "w:gz") as tar: + tar.add(FAKE_SCANNER, arcname="cargo-audit-x86_64-unknown-linux-musl-v0.22.2/cargo-audit") + + def write_inputs(self): + self.lock.write_text(LOCK) + self.policy.write_text(POLICY) + + def git(self, *args): + return subprocess.run(["git", "-C", str(self.repo), *args], env={**os.environ, **GIT_ENV}, + capture_output=True, text=True, check=True).stdout.strip() + + def run_gate(self, args=None, script=GATE, log="clean-success.log", **fake_env): + """Run ` && touch sentinel` from fresh record and sentinel state.""" + for path in (self.sentinel, self.scan_record, self.curl_record): + path.unlink(missing_ok=True) + env = { + **os.environ, **GIT_ENV, + "PATH": f"{STUBS}{os.pathsep}{os.environ['PATH']}", + "TMPDIR": str(self.tmpdir), + "REAL_SHA256SUM": REAL_SHA256SUM, + "FAKE_ARCHIVE": str(self.archive), + "FAKE_ARCHIVE_SHA256": ARCHIVE_SHA256, + "FAKE_CURL_RECORD": str(self.curl_record), + "FAKE_RECORD": str(self.scan_record), + "FAKE_LOG": str(LOGS / log), + **fake_env, + } + args = [self.head] if args is None else args + command = 'sentinel="$1" gate="$2"; shift 2; "$gate" "$@" && touch "$sentinel"' + return subprocess.run(["bash", "-c", command, "run-gate", str(self.sentinel), str(script), *args], + cwd=self.repo, env=env, capture_output=True, text=True, timeout=120) + + def assert_blocked(self, proc, message, rc=1): + output = proc.stdout + proc.stderr + self.assertEqual(proc.returncode, rc, output) + self.assertFalse(self.sentinel.exists(), "gate failure reached the sentinel") + self.assertIn(message, output) + + def test_clean_scan_passes_with_pinned_identity_and_invocation(self): + proc = self.run_gate() + out = proc.stdout + self.assertEqual(proc.returncode, 0, out + proc.stderr) + self.assertTrue(self.sentinel.exists()) + self.assertIn("cargo-audit release gate: result: PASS", out) + for line in ( + f"expected-sha: {self.head}", + f"head-sha: {self.head}", + f"cargo-lock-sha256: {hashlib.sha256(LOCK.encode()).hexdigest()}", + f"audit-toml-sha256: {hashlib.sha256(POLICY.encode()).hexdigest()}", + f"scanner-version: {SCANNER_VERSION}", + f"advisory-db-origin: {TRUSTED_DB}", + "----- begin cargo-audit output -----", + f"Scanning {self.lock} for vulnerabilities (1 crate dependencies)", + "----- end cargo-audit output -----", + ): + self.assertIn(line, out) + self.assertRegex(out, r"(?m)^advisory-db-revision: [0-9a-f]{40}$") + + [scan] = read_records(self.scan_record) + self.assertEqual(scan["arg"], [ + "audit", "--file", str(self.lock), "--db", scan["db"], "--url", TRUSTED_DB, + "--format", "terminal", "--color", "never", + ]) + [curl] = read_records(self.curl_record) + pairs = list(zip(curl["arg"], curl["arg"][1:])) + for pair in (("--proto", "=https"), ("--connect-timeout", "15"), ("--max-time", "180")): + self.assertIn(pair, pairs) + self.assertEqual(curl["arg"][-1], ARCHIVE_URL) + + def test_installation_failures_block_before_scan(self): + corrupt = self.root / "corrupt.tgz" + corrupt.write_text("not a gzip archive\n") + cases = [ + ({"FAKE_CURL_FAIL": "1"}, "failed to download the pinned cargo-audit archive"), + ({"FAKE_ARCHIVE_SHA256": "0" * 64}, "pinned cargo-audit archive checksum mismatch"), + ({"FAKE_ARCHIVE": str(corrupt)}, "failed to extract the pinned cargo-audit archive"), + ({"FAKE_VERSION": "cargo-audit-audit 0.22.1"}, "unexpected cargo-audit version"), + ({"FAKE_VERSION": "cargo-audit-audit 0.22.2 extra"}, "unexpected cargo-audit version"), + ({"FAKE_CARGO_FAIL": "1"}, "cargo -V failed"), + ] + for fake_env, message in cases: + with self.subTest(fake_env=fake_env): + self.assert_blocked(self.run_gate(**fake_env), message, rc=2 if "FAKE_CARGO_FAIL" in fake_env else 1) + self.assertEqual(read_records(self.scan_record), []) + + def test_source_identity_failures_block_before_scan(self): + cases = [ + ("a" * 40, {}, "does not match expected release commit"), + (None, {self.lock: LOCK + "# tampered\n"}, "working-tree Cargo.lock does not match"), + (None, {self.policy: '[advisories]\nignore = ["RUSTSEC-0000-0000"]\n'}, + "working-tree .cargo/audit.toml does not match"), + (None, {self.lock: None}, "missing Cargo.lock at repository root"), + (None, {self.policy: None}, "missing .cargo/audit.toml at repository root"), + ] + for sha, edits, message in cases: + with self.subTest(message=message): + for path, text in edits.items(): + path.unlink() if text is None else path.write_text(text) try: - if entry.is_file() and os.access(entry, os.X_OK): - (shim / name).symlink_to(entry) - seen.add(name) - except OSError: - continue - return str(shim) - - def run_gate(self, script=None, expected_sha=None, sentinel=None, **env_overrides): - script = script or GATE - expected_sha = expected_sha if expected_sha is not None else self.head_sha - sentinel = sentinel or self.sentinel - command = f'"{script}" "{expected_sha}" && touch "{sentinel}"' - return subprocess.run( - ["bash", "-c", command], - cwd=str(self.repo), - env=self.env(**env_overrides), - capture_output=True, - text=True, - check=False, - timeout=120, - ) - - def run_raw(self, args, **env_overrides): - return subprocess.run( - ["bash", *[str(arg) for arg in args]], - cwd=str(self.repo), - env=self.env(**env_overrides), - capture_output=True, - text=True, - check=False, - timeout=120, - ) - - def scanner_runs(self): - if not self.record.exists(): - return [] - runs = [] - current = {} - for line in self.record.read_text().splitlines(): - if not line: - continue - key, _, value = line.partition("=") - if key == "db" and current: - runs.append(current) - current = {} - current[key] = value - if current: - runs.append(current) - return runs + self.assert_blocked(self.run_gate(args=[sha] if sha else None), message) + self.assertEqual(read_records(self.scan_record), []) + finally: + self.write_inputs() + def test_usage_errors_exit_2_before_scan(self): + for args in ([], [self.head, self.head], ["not-a-commit"], [self.head.upper()]): + with self.subTest(args=args): + proc = self.run_gate(args=args) + self.assert_blocked(proc, "cargo-audit release gate: ERROR:", rc=2) + self.assertFalse(self.curl_record.exists() or self.scan_record.exists()) -class ReleaseGateTests(unittest.TestCase): - def setUp(self): - self._tmp = tempfile.TemporaryDirectory(prefix="audit-gate-test.") - self.addCleanup(self._tmp.cleanup) - self.harness = GateHarness(self._tmp.name) - - def assert_blocked(self, proc, message=None): - self.assertNotEqual(proc.returncode, 0, proc.stdout + proc.stderr) - self.assertFalse(self.harness.sentinel.exists(), "gate failure reached the sentinel") - if message is not None: - self.assertIn(message, proc.stdout + proc.stderr) - - # --- accepted path ----------------------------------------------------- - - def test_success_path_reaches_sentinel_and_logs_identity(self): - proc = self.harness.run_gate() - self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) - self.assertTrue(self.harness.sentinel.exists()) - self.assertIn("result: PASS", proc.stdout) - self.assertIn(f"expected-sha: {self.harness.head_sha}", proc.stdout) - self.assertIn(f"scanner-version: {SCANNER_VERSION_LINE}", proc.stdout) - self.assertIn(f"advisory-db-origin: {ADVISORY_DB_URL}", proc.stdout) - self.assertIn("advisory-db-revision:", proc.stdout) - self.assertNotIn("advisory-db-revision: unavailable", proc.stdout) - self.assertIn("cargo-lock-sha256:", proc.stdout) - self.assertIn("audit-toml-sha256:", proc.stdout) - - def test_scanner_invocation_uses_pinned_flags_and_no_bypasses(self): - proc = self.harness.run_gate() - self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) - runs = self.harness.scanner_runs() - self.assertEqual(len(runs), 1, runs) - args = runs[0]["args"] - self.assertIn(f"--url {ADVISORY_DB_URL}", args) - self.assertIn("--format terminal", args) - self.assertIn("--color never", args) - self.assertIn("--file", args) - self.assertIn("--db", args) - for bypass in ("--no-fetch", "--stale", "--deny", "--ignore"): - self.assertNotIn(bypass, args) - - def test_report_log_is_displayed(self): - proc = self.harness.run_gate() - self.assertIn("begin cargo-audit output", proc.stdout) - self.assertIn("end cargo-audit output", proc.stdout) - runs = self.harness.scanner_runs() - self.assertEqual(len(runs), 1, runs) - scanned = runs[0]["file"] - self.assertTrue(scanned.startswith("/"), scanned) - self.assertIn(f"Scanning {scanned} for vulnerabilities (1 crate dependencies)", proc.stdout) - - def test_scan_of_a_substituted_lockfile_blocks_sentinel(self): - # The scanner reports a different lockfile than the one the gate passed - # to it; the pure guard must reject the mismatch before the sentinel. - proc = self.harness.run_gate(FAKE_SCANNER_SCAN_PATH="Cargo.lock") - self.assert_blocked(proc, "but the gate expected") - - def test_download_is_bounded(self): - record = self.harness.root / "curl-args.txt" - proc = self.harness.run_gate(FAKE_CURL_RECORD=str(record)) - self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) - argv = record.read_text() - self.assertIn("--connect-timeout 15", argv) - self.assertIn("--max-time 180", argv) - self.assertIn("--proto =https", argv) - - def test_scanner_timeout_is_bounded_and_blocks_sentinel(self): - for mode, expected in (("fail124", 124), ("fail137", 137)): - with self.subTest(mode=mode): - record = self.harness.root / f"timeout-{mode}.txt" - proc = self.harness.run_gate( - FAKE_TIMEOUT_MODE=mode, - FAKE_TIMEOUT_RECORD=str(record), - ) - self.assert_blocked(proc) - self.assertEqual(proc.returncode, expected, proc.stdout + proc.stderr) - argv = record.read_text() - self.assertIn("--kill-after=10s", argv) - self.assertIn("5m", argv) - self.assertIn("audit", argv) - # The bounded timeout fired before the scanner could run. - self.assertEqual(self.harness.scanner_runs(), []) - - def test_missing_timeout_tool_blocks_sentinel(self): - proc = self.harness.run_gate(PATH=self.harness.path_without("timeout")) - self.assert_blocked(proc, "required tool not found on PATH: timeout") - self.assertEqual(proc.returncode, 2, proc.stdout + proc.stderr) - - # --- scanner verdicts -------------------------------------------------- - - def test_findings_block_sentinel(self): - proc = self.harness.run_gate(FAKE_SCANNER_MODE="findings") - self.assert_blocked(proc) - self.assertEqual(proc.returncode, 1, proc.stdout + proc.stderr) - - def test_hidden_index_error_blocks_sentinel(self): - proc = self.harness.run_gate(FAKE_SCANNER_MODE="index-error") - self.assert_blocked(proc) - - def test_yanked_diagnostic_blocks_sentinel(self): - proc = self.harness.run_gate(FAKE_SCANNER_MODE="yanked-error") - self.assert_blocked(proc) - - def test_missing_progress_markers_block_sentinel(self): - for mode in ("quiet", "disabled-yanked", "disabled-fetch"): - with self.subTest(mode=mode): - self.harness.sentinel.unlink(missing_ok=True) - proc = self.harness.run_gate(FAKE_SCANNER_MODE=mode) - self.assert_blocked(proc) - - def test_fetch_failure_blocks_sentinel(self): - proc = self.harness.run_gate(FAKE_SCANNER_MODE="fetch-error") - self.assert_blocked(proc) - self.assertEqual(proc.returncode, 1, proc.stdout + proc.stderr) - - def test_config_error_blocks_sentinel(self): - proc = self.harness.run_gate(FAKE_SCANNER_MODE="config-error") - self.assert_blocked(proc) - - def test_untrusted_advisory_origin_blocks_sentinel(self): - proc = self.harness.run_gate(FAKE_SCANNER_MODE="wrong-origin") - self.assert_blocked(proc, "unexpected advisory database origin") - - # --- installation and prerequisite failures ---------------------------- - - def test_download_failure_blocks_sentinel(self): - proc = self.harness.run_gate(FAKE_CURL_MODE="fail") - self.assert_blocked(proc, "failed to download") - - def test_checksum_mismatch_blocks_sentinel(self): - proc = self.harness.run_gate(FAKE_ARCHIVE_SHA_MODE="bad") - self.assert_blocked(proc, "checksum mismatch") - - def test_extraction_failure_blocks_sentinel(self): - corrupt = Path(self._tmp.name) / "corrupt.tgz" - corrupt.write_text("this is not a gzip archive\n") - proc = self.harness.run_gate( - FAKE_CURL_SOURCE=str(corrupt), - FAKE_ARCHIVE_REAL_SHA=sha256_file(corrupt), - FAKE_ARCHIVE_SHA_MODE="ok", - ) - self.assert_blocked(proc, "failed to extract") - - def test_version_mismatch_blocks_sentinel(self): - proc = self.harness.run_gate(FAKE_SCANNER_VERSION_LINE="cargo-audit-audit 0.22.1") - self.assert_blocked(proc, "unexpected cargo-audit version") - - def test_non_exact_version_is_rejected(self): - # The check must match the whole version line, not a substring. - proc = self.harness.run_gate( - FAKE_SCANNER_VERSION_LINE="cargo-audit-audit 0.22.20 extra" - ) - self.assert_blocked(proc, "unexpected cargo-audit version") - - def test_broken_cargo_toolchain_blocks_sentinel(self): - proc = self.harness.run_gate(FAKE_CARGO_MODE="fail") - self.assert_blocked(proc, "cargo -V failed") - - # --- source and policy identity --------------------------------------- - - def test_sha_mismatch_blocks_sentinel(self): - proc = self.harness.run_gate(expected_sha="a" * 40) - self.assert_blocked(proc, "does not match expected release commit") - - def test_working_tree_lockfile_mismatch_blocks_sentinel(self): - (self.harness.repo / "Cargo.lock").write_text(LOCK_TEXT + "\n# tampered\n") - proc = self.harness.run_gate() - self.assert_blocked(proc, "Cargo.lock does not match") - - def test_working_tree_policy_mismatch_blocks_sentinel(self): - (self.harness.repo / ".cargo" / "audit.toml").write_text( - '[advisories]\nignore = ["RUSTSEC-0000-0000"]\n' - ) - proc = self.harness.run_gate() - self.assert_blocked(proc, "audit.toml does not match") - - def test_missing_lockfile_blocks_sentinel(self): - (self.harness.repo / "Cargo.lock").unlink() - proc = self.harness.run_gate() - self.assert_blocked(proc, "missing Cargo.lock") - - def test_missing_policy_blocks_sentinel(self): - (self.harness.repo / ".cargo" / "audit.toml").unlink() - proc = self.harness.run_gate() - self.assert_blocked(proc, "missing .cargo/audit.toml") - - # --- usage and input validation --------------------------------------- - - def test_usage_and_input_validation(self): + def test_scanner_verdicts(self): + fetch_failure = {"log": os.devnull, "FAKE_RC": "1", "FAKE_NO_DB": "1"} cases = [ - ([GATE], 2), - ([GATE, "a", "b"], 2), - ([GATE, "not-a-commit"], 2), - ([GATE, "A" * 40], 2), + ({"log": "warning-only-success.log"}, 0, "result: PASS"), + ({"log": "vulnerable-failure.log", "FAKE_RC": "1"}, 1, "scanner exited 1"), + ({"log": "index-error-rc0.log"}, 1, "incomplete crates.io index check"), + ({"log": "quiet-output-rc0.log"}, 1, "missing the advisory database fetch progress line"), + ({"log": "disabled-yanked-rc0.log"}, 1, "missing the crates.io index update progress line"), + ({"log": "disabled-fetch-rc0.log"}, 1, "missing the advisory database fetch progress line"), + (fetch_failure, 1, "advisory-db-origin: unavailable"), + ({"FAKE_ORIGIN": "https://example.invalid/advisory-db.git"}, 1, "unexpected advisory database origin"), + ({"FAKE_SCAN_PATH": "Cargo.lock"}, 1, f"scanned 'Cargo.lock' but the gate expected '{self.lock}'"), + ({"FAKE_RC": "124"}, 124, "scanner exited 124; release blocked"), + ({"FAKE_RC": "137"}, 137, "scanner exited 137; release blocked"), ] - for args, expected in cases: - with self.subTest(args=args): - proc = self.harness.run_raw(args) - self.assertEqual(proc.returncode, expected, proc.stdout + proc.stderr) - - # --- freshness and retry ---------------------------------------------- - - def test_fresh_private_directories_per_execution(self): - first = self.harness.run_gate() - self.assertEqual(first.returncode, 0, first.stdout + first.stderr) - second = self.harness.run_gate() - self.assertEqual(second.returncode, 0, second.stdout + second.stderr) - - runs = self.harness.scanner_runs() - self.assertEqual(len(runs), 2, runs) - for run in runs: - self.assertTrue(run["db"].startswith(os.path.join(tempfile.gettempdir(), "cargo-audit-release-gate"))) - self.assertTrue(run["cargo_home"].startswith(os.path.join(tempfile.gettempdir(), "cargo-audit-release-gate"))) - self.assertNotEqual(run["db"], run["cargo_home"]) - self.assertNotEqual(runs[0]["db"], runs[1]["db"]) - self.assertNotEqual(runs[0]["cargo_home"], runs[1]["cargo_home"]) - - def test_inherited_warm_cache_cannot_authorize_refresh_failure(self): - # The inherited environment carries a warm RustSec clone with an - # untrusted origin plus a warm Cargo home, both under the inherited - # TMPDIR. The stub below would authorize the release if it were handed - # the warm database, and fails when the database path is fresh. The gate - # must therefore fail here rather than reuse any inherited state. - inherited = self.harness.root / "inherited" - inherited.mkdir() - warm_db = inherited / "rustsec-db" - warm_db.mkdir() - self.harness._git("init", "--quiet", str(warm_db)) - self.harness._git( - "-C", str(warm_db), "remote", "add", "origin", "https://example.invalid/advisory-db.git" - ) - (warm_db / "README.md").write_text("warm fixture advisory database\n") - self.harness._git("-C", str(warm_db), "add", "-A") - self.harness._git( - "-C", - str(warm_db), - "-c", - "user.email=fixture@example.invalid", - "-c", - "user.name=fixture", - "commit", - "--quiet", - "-m", - "warm fixture database", - ) - warm_cargo_home = inherited / "cargo-home" - (warm_cargo_home / "registry" / "index").mkdir(parents=True) - (warm_cargo_home / ".package-cache").write_text("warm cargo cache\n") - - # Prove the stub discriminates: it succeeds on the warm path and fails - # on a fresh path that simulates an unavailable network refresh. - warm = self.harness.run_raw( - [FAKE_SCANNER, "audit", "--db", str(warm_db), "--file", "Cargo.lock"], - FAKE_SCANNER_MODE="warm-authorize", - FAKE_SCANNER_RECORD="", - ) - self.assertEqual(warm.returncode, 0, warm.stdout + warm.stderr) - self.assertIn("Fetching advisory database", warm.stdout) - fresh = self.harness.run_raw( - [FAKE_SCANNER, "audit", "--db", str(inherited / "no-such-db"), "--file", "Cargo.lock"], - FAKE_SCANNER_MODE="warm-authorize", - FAKE_SCANNER_RECORD="", - ) - self.assertNotEqual(fresh.returncode, 0, fresh.stdout + fresh.stderr) - - proc = self.harness.run_gate( - FAKE_SCANNER_MODE="warm-authorize", - TMPDIR=str(inherited), - CARGO_HOME=str(warm_cargo_home), - ) - self.assert_blocked(proc) - runs = self.harness.scanner_runs() - self.assertEqual(len(runs), 1, runs) - fresh_work = str(inherited / "cargo-audit-release-gate") - self.assertTrue(runs[0]["db"].startswith(fresh_work), runs) - self.assertTrue(runs[0]["cargo_home"].startswith(fresh_work), runs) - self.assertNotEqual(runs[0]["db"], str(warm_db)) - self.assertNotEqual(runs[0]["cargo_home"], str(warm_cargo_home)) - # The inherited untrusted-origin database is never inspected or reused. - self.assertNotIn("example.invalid", proc.stdout + proc.stderr) - - def test_retry_after_refresh_failure_rescans_and_can_pass(self): - failed = self.harness.run_gate(FAKE_SCANNER_MODE="fetch-error") - self.assert_blocked(failed) - passed = self.harness.run_gate() + for fake_env, rc, message in cases: + with self.subTest(fake_env=fake_env): + proc = self.run_gate(**fake_env) + if rc == 0: + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + self.assertTrue(self.sentinel.exists()) + self.assertIn(message, proc.stdout) + else: + self.assert_blocked(proc, message, rc=rc) + self.assertEqual(len(read_records(self.scan_record)), 1) + + def test_retry_after_refresh_failure_uses_fresh_private_state(self): + warm_db = self.tmpdir / "rustsec-db" + warm_home = self.root / "warm-cargo-home" + for warm in (warm_db, warm_home): + (warm / "registry").mkdir(parents=True) + (warm / "registry" / "cached").write_text("warm\n") + warm_env = {"CARGO_HOME": str(warm_home)} + + failed = self.run_gate(log=os.devnull, FAKE_RC="1", FAKE_NO_DB="1", **warm_env) + self.assert_blocked(failed, "scanner exited 1") + scans = read_records(self.scan_record) + passed = self.run_gate(**warm_env) self.assertEqual(passed.returncode, 0, passed.stdout + passed.stderr) - runs = self.harness.scanner_runs() - self.assertEqual(len(runs), 2, runs) - self.assertNotEqual(runs[0]["db"], runs[1]["db"]) - self.assertNotEqual(runs[0]["cargo_home"], runs[1]["cargo_home"]) - - # --- helper robustness ------------------------------------------------ - - def test_unexpected_guard_status_is_rejected(self): - copydir = self.harness.root / "copyscripts" - copydir.mkdir() - gate_copy = copydir / "cargo-audit-release-gate.sh" + scans += read_records(self.scan_record) + + self.assertEqual(len(scans), 2, scans) + private = f"{self.tmpdir}/cargo-audit-release-gate." + for scan in scans: + self.assertTrue(scan["db"].startswith(private), scan) + self.assertTrue(scan["home"].startswith(private), scan) + self.assertEqual((scan["db_existed"], scan["home_empty"]), ("no", "yes"), scan) + self.assertNotIn(scan["db"], (str(warm_db), str(warm_home))) + self.assertNotIn(scan["home"], (str(warm_db), str(warm_home))) + self.assertNotEqual(scans[0]["db"], scans[1]["db"]) + self.assertNotEqual(scans[0]["home"], scans[1]["home"]) + # Each run removed its private directory; the warm state is untouched. + self.assertEqual([p.name for p in self.tmpdir.iterdir()], ["rustsec-db"]) + self.assertEqual((warm_home / "registry" / "cached").read_text(), "warm\n") + + def test_unexpected_guard_status_fails_closed(self): + copies = self.root / "scripts" + copies.mkdir() + gate_copy = copies / GATE.name shutil.copy(GATE, gate_copy) - gate_copy.chmod(0o755) - guard_copy = copydir / "cargo-audit-report-guard.sh" + guard_copy = copies / "cargo-audit-report-guard.sh" guard_copy.write_text("#!/usr/bin/env bash\nexit 127\n") guard_copy.chmod(0o755) - proc = self.harness.run_gate(script=gate_copy) - self.assert_blocked(proc, "unexpected status") - self.assertEqual(proc.returncode, 1, proc.stdout + proc.stderr) - - def test_pinned_constants_are_literal_and_reviewed(self): - text = GATE.read_text() - self.assertIn(ARCHIVE_URL, text) - self.assertIn(ARCHIVE_SHA256, text) - self.assertIn(SCANNER_VERSION_LINE, text) - self.assertIn(ADVISORY_DB_URL, text) - # No dynamic digest lookup and no alternate-archive environment hook. - self.assertNotIn("sha256sum -c", text) - self.assertNotIn("curl --head", text) + self.assert_blocked(self.run_gate(script=gate_copy), "report guard returned an unexpected status (127)") + + def test_static_commands_keep_pins_and_bounds(self): + commands = logical_commands(GATE.read_text()) + tokens = [token for command in commands for token in command] + + assignments = {c[1] for c in commands if len(c) == 2 and c[0] == "readonly"} + for pin in (f"ARCHIVE_URL={ARCHIVE_URL}", f"ARCHIVE_SHA256={ARCHIVE_SHA256}", + f"SCANNER_VERSION_LINE={SCANNER_VERSION}", f"ADVISORY_DB_URL={TRUSTED_DB}"): + self.assertIn(pin, assignments) + + [curl] = [c for c in commands if "curl" in c and any("$ARCHIVE_URL" in t for t in c)] + curl_pairs = list(zip(curl, curl[1:])) + for flag, value in (("--proto", "=https"), ("--connect-timeout", "15"), ("--max-time", "180")): + self.assertEqual(curl.count(flag), 1, curl) + self.assertIn((flag, value), curl_pairs) + + [scan] = [c for c in commands if "timeout" in c and "audit" in c] + start = scan.index("timeout") + self.assertEqual(scan[start:start + 5], ["timeout", "--kill-after=10s", "5m", "$scanner", "audit"]) + scan_pairs = list(zip(scan, scan[1:])) + for pair in (("--file", "$lockfile"), ("--db", "$db"), ("--url", "$ADVISORY_DB_URL"), + ("--format", "terminal"), ("--color", "never")): + self.assertIn(pair, scan_pairs) + + forbidden = re.compile(r"^--(no-fetch|stale|ignore|deny)") + self.assertEqual([t for t in tokens if forbidden.match(t)], []) if __name__ == "__main__": diff --git a/scripts/tests/audit-release-gate/test_report_guard.py b/scripts/tests/audit-release-gate/test_report_guard.py index 09db317b..b378a65c 100644 --- a/scripts/tests/audit-release-gate/test_report_guard.py +++ b/scripts/tests/audit-release-gate/test_report_guard.py @@ -2,249 +2,129 @@ The guard is pure: it takes a captured cargo-audit terminal log, the numeric scanner exit code, and the lockfile path the gate expected the scan to cover, and -returns a verdict. None of these tests touch the network, install anything, or -run a publisher. - -The logs under fixtures/logs were captured from the real pinned cargo-audit -0.22.2 binary; see fixtures/logs/README.md for provenance. They were captured -with a relative `--file Cargo.lock`, so the tests pass `Cargo.lock` as the -expected path for them. Synthetic variants are derived in memory to cover -individual rejection reasons (zero counts, out-of-order markers, substituted -paths); those are contract fixtures, not captured runtime output. +returns a verdict. None of these tests touch the network or install anything. + +Provenance: the logs under fixtures/logs are captured runtime output from the +real pinned cargo-audit 0.22.2 binary (see README.md), recorded +with a relative `--file Cargo.lock`, so `Cargo.lock` is their expected path. The +SYNTHETIC variants are contract fixtures derived in memory from the clean log to +isolate single rejection reasons; they are not scanner output. The runtime proof +for the absolute-path Scanning line is the opt-in network_smoke.py suite. """ -import os import subprocess import tempfile import unittest from pathlib import Path HERE = Path(__file__).resolve().parent -REPO = HERE.parents[2] -GUARD = REPO / "scripts" / "cargo-audit-report-guard.sh" +GUARD = HERE.parents[1] / "cargo-audit-report-guard.sh" LOGS = HERE / "fixtures" / "logs" +CLEAN_LOG = LOGS / "clean-success.log" +CLEAN = CLEAN_LOG.read_text() +CAPTURED_LOCKFILE = "Cargo.lock" -# (captured log name, captured scanner exit code, expected guard exit code) +# (captured log, scanner exit code it was captured with, expected guard exit code) CAPTURED = [ ("clean-success", 0, 0), ("warning-only-success", 0, 0), ("vulnerable-failure", 1, 1), - ("malformed-lock-rc2", 2, 1), - ("missing-lock-rc2", 2, 1), ("index-error-rc0", 0, 1), ("quiet-output-rc0", 0, 1), ("disabled-yanked-rc0", 0, 1), ("disabled-fetch-rc0", 0, 1), ] -# The capture run scanned a relative `Cargo.lock`. -CAPTURED_LOCKFILE = "Cargo.lock" -MARKERS = ("Fetching", "Loaded", "Updating", "Scanning") - - -def run_guard(log, rc, expected_lockfile=CAPTURED_LOCKFILE): - return subprocess.run( - ["bash", str(GUARD), str(log), str(rc), str(expected_lockfile)], - capture_output=True, - text=True, - check=False, - ) +def without(marker): + kept = [line for line in CLEAN.splitlines() if marker not in line] + if len(kept) == len(CLEAN.splitlines()): + raise LookupError(f"clean log has no '{marker}' line") + return "\n".join(kept) + "\n" + + +def replaced(old, new): + if old not in CLEAN: + raise LookupError(f"clean log does not contain '{old}'") + return CLEAN.replace(old, new) + + +def reordered(): + fetch, loaded, *rest = CLEAN.splitlines() + if "Fetching" not in fetch or "Loaded" not in loaded: + raise LookupError("clean log no longer starts with Fetching then Loaded") + return "\n".join([loaded, fetch, *rest]) + "\n" + + +ABSOLUTE_SCAN = "Scanning /tmp/absolute/path/Cargo.lock for vulnerabilities" + +# (name, log text, expected lockfile, expected guard exit code); scanner exit is 0. +SYNTHETIC = [ + *[(f"missing-{m}", without(m), CAPTURED_LOCKFILE, 1) for m in ("Fetching", "Loaded", "Updating", "Scanning")], + ("update-index-diagnostic", CLEAN + "warning: couldn't update crates.io index: fixture\n", CAPTURED_LOCKFILE, 1), + ("open-index-diagnostic", CLEAN + "warning: couldn't open crates.io index: fixture\n", CAPTURED_LOCKFILE, 1), + ("yanked-diagnostic", CLEAN + "error: couldn't check if the package is yanked: fixture\n", CAPTURED_LOCKFILE, 1), + ("zero-advisories", replaced("Loaded 1380 security", "Loaded 0 security"), CAPTURED_LOCKFILE, 1), + ("missing-advisory-count", replaced("Loaded 1380 security", "Loaded some security"), CAPTURED_LOCKFILE, 1), + ("zero-dependencies", replaced("(1 crate dependencies)", "(0 crate dependencies)"), CAPTURED_LOCKFILE, 1), + ("missing-dependency-count", replaced("(1 crate", "(several crate"), CAPTURED_LOCKFILE, 1), + ("empty-log", "", CAPTURED_LOCKFILE, 1), + ("bare-marker-words", "Fetching x\nLoaded a note\nUpdating x\nScanning x\n", CAPTURED_LOCKFILE, 1), + ("fetch-after-loaded", reordered(), CAPTURED_LOCKFILE, 1), + ("exact-wrong-path", CLEAN, "/expected/elsewhere/Cargo.lock", 1), + ("suffix-wrong-path", replaced("Scanning Cargo.lock", "Scanning /a/b/Cargo.lock"), "/b/Cargo.lock", 1), + ("absolute-valid-path", replaced("Scanning Cargo.lock for vulnerabilities", ABSOLUTE_SCAN), + "/tmp/absolute/path/Cargo.lock", 0), +] -def clean_text(): - return (LOGS / "clean-success.log").read_text() +# (name, argv after the guard path); every case is a usage error (exit 2). +USAGE_ERRORS = [ + *[(f"status-{rc or 'empty'}", [str(CLEAN_LOG), rc, CAPTURED_LOCKFILE]) for rc in ("abc", "", "-1", "0x0", "1.0")], + ("missing-log", [str(LOGS / "does-not-exist.log"), "0", CAPTURED_LOCKFILE]), + ("empty-expected-path", [str(CLEAN_LOG), "0", ""]), + ("zero-args", []), + ("one-arg", [str(CLEAN_LOG)]), + ("two-args", [str(CLEAN_LOG), "0"]), + ("four-args", ["a", "b", "c", "d"]), +] -def write_tmp_log(tmp, text): - path = Path(tmp) / "log.txt" - path.write_text(text) - return path +def guard(*args): + return subprocess.run(["bash", str(GUARD), *map(str, args)], capture_output=True, text=True, check=False) class ReportGuardTests(unittest.TestCase): - def assert_rejected(self, text, expected_lockfile=CAPTURED_LOCKFILE): - with tempfile.TemporaryDirectory() as tmp: - result = run_guard(write_tmp_log(tmp, text), 0, expected_lockfile) - self.assertEqual(result.returncode, 1, result.stdout + result.stderr) + def assert_exit(self, result, expected): + self.assertEqual(result.returncode, expected, result.stdout + result.stderr) - def test_captured_logs_guard_verdicts(self): - for name, rc, expected in CAPTURED: - with self.subTest(log=name, rc=rc): + def test_captured_logs(self): + for name, scanner_rc, expected in CAPTURED: + with self.subTest(name): log = LOGS / f"{name}.log" - self.assertTrue(log.is_file(), f"missing captured log: {log}") - result = run_guard(log, rc) - self.assertEqual(result.returncode, expected, result.stdout + result.stderr) - - def test_informational_warning_is_not_a_rejection(self): - # A zero-exit scan whose only output is informational warnings must pass, - # preserving the existing scanner warning policy. - log = LOGS / "warning-only-success.log" - self.assertIn("allowed warning found", log.read_text()) - self.assertEqual(run_guard(log, 0).returncode, 0) - - def test_clean_zero_exit_requires_every_marker(self): - text = clean_text() - for marker in MARKERS: - self.assertIn(marker, text) - for marker in MARKERS: - with self.subTest(removed=marker): - mutated = "\n".join( - line for line in text.splitlines() if marker not in line - ) - self.assert_rejected(mutated + "\n") - - def test_zero_exit_with_hidden_index_diagnostics_is_rejected(self): - for diagnostic in ( - "warning: couldn't update crates.io index: fixture failure\n", - "warning: couldn't open crates.io index: fixture failure\n", - "error: couldn't check if the package is yanked: fixture miss\n", - ): - with self.subTest(diagnostic=diagnostic): - self.assert_rejected(clean_text() + diagnostic) - - def test_empty_log_with_zero_exit_is_rejected(self): - self.assert_rejected("") - - def test_bare_marker_words_are_not_enough(self): - # The guard must require the substantive marker lines, not the words - # "Fetching", "Loaded", "Updating", and "Scanning" in unrelated text. - text = ( - "Fetching something else\n" - "Loaded a note\n" - "Updating something else\n" - "Scanning something else\n" - ) - self.assert_rejected(text) - - def test_zero_advisory_count_is_rejected(self): - text = clean_text().replace("Loaded 1380 security advisories", "Loaded 0 security advisories") - self.assert_rejected(text) - - def test_missing_advisory_count_is_rejected(self): - text = clean_text().replace("Loaded 1380 security advisories", "Loaded some security advisories") - self.assert_rejected(text) - - def test_zero_dependency_count_is_rejected(self): - text = clean_text().replace("(1 crate dependencies)", "(0 crate dependencies)") - self.assert_rejected(text) - - def test_missing_dependency_count_is_rejected(self): - text = clean_text().replace("(1 crate dependencies)", "(several crate dependencies)") - self.assert_rejected(text) - - def test_out_of_order_markers_are_rejected(self): - text = clean_text() - lines = text.splitlines() - fetch = next(line for line in lines if "Fetching advisory database" in line) - loaded = next(line for line in lines if "Loaded" in line) - reordered = [line for line in lines if line is not fetch] - reordered.insert(reordered.index(loaded) + 1, fetch) - self.assert_rejected("\n".join(reordered) + "\n") - - def test_wrong_expected_lockfile_is_rejected(self): - with tempfile.TemporaryDirectory() as tmp: - log = write_tmp_log(tmp, clean_text()) - result = run_guard(log, 0, "/expected/elsewhere/Cargo.lock") - self.assertEqual(result.returncode, 1, result.stdout + result.stderr) - self.assertIn("but the gate expected", result.stderr) - - def test_lockfile_path_suffix_is_not_accepted(self): - # A scan of /a/b/Cargo.lock must not satisfy an expectation of - # /b/Cargo.lock: the comparison is exact, not a suffix match. - text = clean_text().replace( - "Scanning Cargo.lock for vulnerabilities", - "Scanning /a/b/Cargo.lock for vulnerabilities", - ) - self.assert_rejected(text, "/b/Cargo.lock") - - def test_absolute_expected_lockfile_is_accepted(self): - text = clean_text().replace( - "Scanning Cargo.lock for vulnerabilities", - "Scanning /tmp/absolute/path/Cargo.lock for vulnerabilities", - ) + if name == "warning-only-success": + self.assertIn("allowed warning found", log.read_text()) + self.assert_exit(guard(log, scanner_rc, CAPTURED_LOCKFILE), expected) + + def test_synthetic_variants(self): with tempfile.TemporaryDirectory() as tmp: - self.assertEqual( - run_guard(write_tmp_log(tmp, text), 0, "/tmp/absolute/path/Cargo.lock").returncode, - 0, - ) - - def test_nonzero_scanner_exit_is_rejected_even_with_clean_log(self): - # The scanner exit code is authoritative and independent of the log: - # a complete, clean-looking log must not rescue a nonzero scan, including - # the 124/137 statuses produced by the bounded scanner timeout. - log = LOGS / "clean-success.log" + for name, text, expected_lockfile, expected in SYNTHETIC: + with self.subTest(name): + log = Path(tmp) / f"{name}.log" + log.write_text(text) + self.assert_exit(guard(log, 0, expected_lockfile), expected) + + def test_nonzero_scanner_status_rejects_clean_log(self): + # The exit code is authoritative: a clean log cannot rescue a failed scan, + # including the 124/137 statuses of the bounded scanner timeout. for rc in ("1", "2", "124", "137"): - with self.subTest(rc=rc): - result = run_guard(log, rc) - self.assertEqual(result.returncode, 1, result.stdout + result.stderr) + with self.subTest(rc): + result = guard(CLEAN_LOG, rc, CAPTURED_LOCKFILE) + self.assert_exit(result, 1) self.assertIn("release blocked", result.stderr) - def test_non_numeric_scanner_exit_code_is_rejected(self): - log = LOGS / "clean-success.log" - for bogus in ("abc", "", "-1", "0x0", "1.0"): - with self.subTest(rc=bogus): - self.assertEqual(run_guard(log, bogus).returncode, 2) - - def test_missing_log_is_a_usage_error(self): - self.assertEqual(run_guard(LOGS / "does-not-exist.log", 0).returncode, 2) - - def test_empty_expected_lockfile_is_a_usage_error(self): - self.assertEqual(run_guard(LOGS / "clean-success.log", 0, "").returncode, 2) - - def test_wrong_argument_count_is_a_usage_error(self): - for args in ( - [], - [str(LOGS / "clean-success.log")], - [str(LOGS / "clean-success.log"), "0"], - ["a", "b", "c", "d"], - ): - with self.subTest(args=args): - result = subprocess.run( - ["bash", str(GUARD), *args], - capture_output=True, - text=True, - check=False, - ) - self.assertEqual(result.returncode, 2) - - def test_rejection_prevents_following_command(self): - # A shell sequence must not reach its next command after a rejected scan - # and must reach it after an accepted scan. - with tempfile.TemporaryDirectory() as tmp: - tmp_path = Path(tmp) - sentinel = tmp_path / "sentinel" - failing = LOGS / "index-error-rc0.log" - passing = LOGS / "clean-success.log" - script = 'set -e; bash "$1" "$2" "$3" "$4"; touch "$5"' - - rejected = subprocess.run( - [ - "bash", "-c", script, "guard", - str(GUARD), str(failing), "0", CAPTURED_LOCKFILE, str(sentinel), - ], - capture_output=True, - text=True, - check=False, - ) - self.assertNotEqual(rejected.returncode, 0) - self.assertFalse(sentinel.exists()) - - accepted = subprocess.run( - [ - "bash", "-c", script, "guard", - str(GUARD), str(passing), "0", CAPTURED_LOCKFILE, str(sentinel), - ], - capture_output=True, - text=True, - check=False, - ) - self.assertEqual(accepted.returncode, 0, accepted.stdout + accepted.stderr) - self.assertTrue(sentinel.exists()) - - def test_guard_is_executable_and_offline_only(self): - self.assertTrue(os.access(GUARD, os.X_OK)) - text = GUARD.read_text() - for forbidden in ("curl", "mktemp", "npm publish", "action-gh-release", "aws "): - self.assertNotIn(forbidden, text) + def test_usage_errors(self): + for name, args in USAGE_ERRORS: + with self.subTest(name): + self.assert_exit(guard(*args), 2) if __name__ == "__main__": diff --git a/scripts/tests/audit-release-gate/test_workflows.py b/scripts/tests/audit-release-gate/test_workflows.py index 41610144..a7a0f88e 100644 --- a/scripts/tests/audit-release-gate/test_workflows.py +++ b/scripts/tests/audit-release-gate/test_workflows.py @@ -1,656 +1,251 @@ """Offline structural tests for the release-gate workflows. -These tests parse the real workflow YAML and assert that the strict cargo-audit -gate runs inside each protected job, before that job's first publication side -effect, against the exact source revision that produced the artifacts. They also -run a set of negative mutations to prove the checks reject an absent, reordered, -masked, or unlocked gate. - -PyYAML is a test-only dependency. `on:` is a YAML 1.1 boolean key, so the trigger -block is read through the `trigger()` helper rather than by key name. +These tests parse the real workflow YAML and check that the strict cargo-audit +gate runs unconditionally inside each protected job, before that job's first +publication effect, against the exact source revision that produced the +artifacts. One table of negative mutations proves the checks reject +representative regressions. Expected pins, refs, conditions, and commands are +literals reviewed together with the workflows, never values read from them. + +PyYAML is a test-only dependency. YAML 1.1 parses the `on:` key as boolean True, +so `load()` stores the trigger block back under "on". """ +import shlex import unittest from pathlib import Path import yaml -HERE = Path(__file__).resolve().parent -REPO = HERE.parents[2] -WORKFLOWS = REPO / ".github" / "workflows" -PUBLISH = WORKFLOWS / "publish.yml" -RELEASE = WORKFLOWS / "release-datadog-serverless-compat.yml" -BUILD = WORKFLOWS / "build-datadog-serverless-compat.yml" -TESTS = WORKFLOWS / "audit-release-gate-tests.yml" - -CHECKOUT_PIN = "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" -RUST_SETUP_PIN = "actions-rust-lang/setup-rust-toolchain@9399c7bb15d4c7d47b27263d024f0a4978346ba4" -GH_RELEASE_PIN = "softprops/action-gh-release@da05d552573ad5aba039eaac05058a918a7bf631" -GATE_CALL = "cargo-audit-release-gate.sh" -GATE_COMMAND = './scripts/cargo-audit-release-gate.sh "$GITHUB_SHA"' -SOURCE_REF = "${{ github.sha }}" -BUILD_ONLY_CONDITION = "${{ inputs.publish == 'Build and Publish' }}" -DOWNLOAD_ARTIFACT_PREFIX = "actions/download-artifact@" -EXTERNAL_ARTIFACT_SELECTORS = ("run-id", "repository", "github-token") - +WORKFLOWS = Path(__file__).resolve().parents[3] / ".github" / "workflows" +PUBLISH = "publish.yml" +RELEASE = "release-datadog-serverless-compat.yml" +BUILD = "build-datadog-serverless-compat.yml" -def load(path): - with path.open(encoding="utf-8") as handle: - return yaml.safe_load(handle) +CHECKOUT = "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" +RUST_SETUP = "actions-rust-lang/setup-rust-toolchain@9399c7bb15d4c7d47b27263d024f0a4978346ba4" +GH_RELEASE = "softprops/action-gh-release@da05d552573ad5aba039eaac05058a918a7bf631" +PINS = {pin.split("@")[0]: pin for pin in (CHECKOUT, RUST_SETUP, GH_RELEASE)} +SOURCE_REF = "${{ github.sha }}" +GATE_COMMAND = './scripts/cargo-audit-release-gate.sh "$GITHUB_SHA"' +BUILD_ONLY = "${{ inputs.publish == 'Build and Publish' }}" +CROSS_RUN_SELECTORS = {"run-id", "repository", "github-token"} +SHELL_SEPARATORS = {"&&", "||", ";", "|"} -def trigger(doc): - for key in ("on", True): - if key in doc: - return doc[key] - raise AssertionError("workflow has no trigger block") +def load(name): + doc = yaml.safe_load((WORKFLOWS / name).read_text(encoding="utf-8")) + if True in doc: + doc["on"] = doc.pop(True) + return doc -def step_run(step): +def run_of(step): return str(step.get("run", "")) -def find_step(steps, predicate): - for index, step in enumerate(steps): - if predicate(step): - return index, step - return None, None - - -def normalized_run(step): - return " ".join(step_run(step).split()) - - -def assert_no_failure_masks(doc): - for job_name, job in doc["jobs"].items(): - if job.get("continue-on-error"): - raise AssertionError(f"job '{job_name}' sets continue-on-error") - job_condition = str(job.get("if", "")) - if "always(" in job_condition or "cancelled(" in job_condition: - raise AssertionError(f"job '{job_name}' has an error-masking condition") - for step in job.get("steps", []): - if step.get("continue-on-error"): - raise AssertionError(f"job '{job_name}' has a step with continue-on-error") - condition = str(step.get("if", "")) - if "always(" in condition or "cancelled(" in condition: - raise AssertionError(f"job '{job_name}' has an error-masking step condition") - - -def assert_same_run_artifact_downloads(steps, where): - # Downloading artifacts with a run id, repository, or explicit token could - # pull artifacts produced by a different run than the gated source revision. - for step in steps: - if not str(step.get("uses", "")).startswith(DOWNLOAD_ARTIFACT_PREFIX): - continue - with_block = step.get("with", {}) or {} - for selector in EXTERNAL_ARTIFACT_SELECTORS: - if selector in with_block: - raise AssertionError( - f"{where}: artifact download must come from the current run, not '{selector}'" - ) - - -def assert_gate_step(step, where): - if step is None: - raise AssertionError(f"{where}: strict cargo-audit gate is missing") - if "if" in step: - raise AssertionError(f"{where}: gate step must be unconditional") - if "continue-on-error" in step: - raise AssertionError(f"{where}: gate step must not ignore failures") - run = normalized_run(step) - if run != GATE_COMMAND: - raise AssertionError( - f"{where}: gate step must run exactly '{GATE_COMMAND}', found '{run}'" - ) - shell = step.get("shell") - if shell is not None and str(shell) not in ("bash", "bash {0}"): - raise AssertionError(f"{where}: gate step must use the reviewed bash shell, found '{shell}'") - if step.get("working-directory") != "source": - raise AssertionError(f"{where}: gate step must run from the source checkout") - - -def assert_checkout_ref(step, where): - if step is None: - raise AssertionError(f"{where}: checkout is missing") - if step.get("uses") != CHECKOUT_PIN: - raise AssertionError(f"{where}: checkout is not pinned to the reviewed commit") - if step.get("with", {}).get("ref") != SOURCE_REF: - raise AssertionError(f"{where}: checkout does not pin ref {SOURCE_REF}") - - -def assert_source_checkout(step, where): - assert_checkout_ref(step, where) - if step.get("with", {}).get("path") != "source": - raise AssertionError(f"{where}: checkout does not use path 'source'") - - -def assert_rust_setup(step, where): - if step is None: - raise AssertionError(f"{where}: Rust toolchain setup is missing") - if step.get("uses") != RUST_SETUP_PIN: - raise AssertionError(f"{where}: Rust setup is not pinned to the reviewed commit") - if step.get("with", {}).get("cache") is not False: - raise AssertionError(f"{where}: Rust setup must disable the toolchain cache") - - -def assert_package_job(doc): - package = doc["jobs"]["package"] - steps = package["steps"] - _, checkout = find_step(steps, lambda s: str(s.get("uses", "")).startswith("actions/checkout@")) - assert_checkout_ref(checkout, "package") - index, upload = find_step( - steps, lambda s: str(s.get("uses", "")).startswith("actions/upload-artifact@") - ) - if index is None: - raise AssertionError("package job must upload the npm packages it produces") - if upload.get("with", {}).get("name") != "npm-packages": - raise AssertionError("package job must upload the 'npm-packages' artifact") - - -def assert_publish_workflow(doc): - assert_no_failure_masks(doc) - publish = doc["jobs"]["publish"] - if publish.get("environment") != "Production": - raise AssertionError("publish job must keep the Production environment approval") - if publish.get("needs") != "package": - raise AssertionError("publish job must depend on the package job") - condition = " ".join(str(publish.get("if", "")).split()) - if condition != BUILD_ONLY_CONDITION: - raise AssertionError("publish job must keep the build-only condition") - - assert_package_job(doc) - - steps = publish["steps"] - assert_same_run_artifact_downloads(steps, "publish") - artifact_names = [ - step.get("with", {}).get("name") - for step in steps - if str(step.get("uses", "")).startswith(DOWNLOAD_ARTIFACT_PREFIX) - ] - if "npm-packages" not in artifact_names: - raise AssertionError("publish job must download the artifact produced in the same run") - _, checkout = find_step(steps, lambda s: str(s.get("uses", "")).startswith("actions/checkout@")) - assert_source_checkout(checkout, "publish") - - _, rust = find_step( - steps, - lambda s: str(s.get("uses", "")).startswith("actions-rust-lang/setup-rust-toolchain@"), - ) - assert_rust_setup(rust, "publish") - - gate_index, gate_step = find_step(steps, lambda s: GATE_CALL in step_run(s)) - assert_gate_step(gate_step, "publish") - - publish_index, publisher = find_step(steps, lambda s: "npm publish" in step_run(s)) - if publish_index is None: - raise AssertionError("publish job has no npm publish step") - if "if" in publisher: - raise AssertionError("npm publish step must not be conditional") - if gate_index > publish_index: - raise AssertionError("strict cargo-audit gate must run before the first npm publish") - - permissions = doc.get("permissions", {}) - if permissions.get("contents") != "read": - raise AssertionError("publish workflow must grant only contents: read") - if permissions.get("id-token") != "write": - raise AssertionError("publish workflow must keep id-token: write for provenance") - - inputs = trigger(doc).get("workflow_dispatch", {}).get("inputs", {}) or {} - for name in inputs: - if "skip" in name.lower() or "bypass" in name.lower(): - raise AssertionError(f"publish workflow must not expose a bypass input: {name}") - - -def assert_release_workflow(doc): - assert_no_failure_masks(doc) - release = doc["jobs"]["release"] - if release.get("permissions", {}).get("contents") != "write": - raise AssertionError("release job must keep contents: write") - if release.get("needs") != "build": - raise AssertionError("release job must depend on the build job") - # This freezes the reviewed release flow: the strict cargo-audit gate plus a - # draft release is the release check, so no additional protected environment - # (and therefore no extra manual approval) is introduced. This is an - # invariant of the reviewed change, not a claim that environments are unsafe. - if "environment" in release: - raise AssertionError("release job must not add an extra protected environment approval") - - steps = release["steps"] - assert_same_run_artifact_downloads(steps, "release") - _, checkout = find_step(steps, lambda s: str(s.get("uses", "")).startswith("actions/checkout@")) - assert_source_checkout(checkout, "release") - - _, rust = find_step( - steps, - lambda s: str(s.get("uses", "")).startswith("actions-rust-lang/setup-rust-toolchain@"), - ) - assert_rust_setup(rust, "release") - - gate_index, gate_step = find_step(steps, lambda s: GATE_CALL in step_run(s)) - assert_gate_step(gate_step, "release") - - zip_index, _ = find_step(steps, lambda s: step_run(s).strip().startswith("zip ")) - if zip_index is None: - raise AssertionError("release job has no zip step") - release_index, gh_release = find_step( - steps, lambda s: str(s.get("uses", "")).startswith("softprops/action-gh-release@") - ) - if release_index is None: - raise AssertionError("release job has no action-gh-release step") - if gate_index > zip_index or gate_index > release_index: - raise AssertionError("strict cargo-audit gate must run before the zip and the draft release") - - if gh_release.get("uses") != GH_RELEASE_PIN: - raise AssertionError("action-gh-release pin changed") - if gh_release.get("with", {}).get("draft") is not True: - raise AssertionError("draft release behavior changed") - if "if" in gh_release: - raise AssertionError("the draft release step must not be conditional") - - if doc.get("permissions") != {}: - raise AssertionError("release workflow top-level permissions must stay empty") - - -def assert_build_workflow(doc): - builds = [] - checkouts = [] - for job_name, job in doc["jobs"].items(): - for step in job.get("steps", []): - run = step_run(step) - if "cargo build --release -p datadog-serverless-compat" in run: - builds.append((job_name, run)) - if "cargo-audit" in run: - raise AssertionError("the shared build workflow must not run audits") - if str(step.get("uses", "")).startswith("actions/checkout@"): - checkouts.append(step) - - if len(builds) != 5: - raise AssertionError(f"expected 5 release builds, found {len(builds)}") - for job_name, run in builds: - if "--locked" not in run: - raise AssertionError(f"release build in job '{job_name}' is not --locked") - - if len(checkouts) != 2: - raise AssertionError(f"expected 2 producer checkouts, found {len(checkouts)}") - for checkout in checkouts: - assert_checkout_ref(checkout, "build producer") - - -def assert_tests_workflow(doc, raw): - assert_no_failure_masks(doc) - if "if" in doc["jobs"]["offline"]: - raise AssertionError("offline gate tests must run unconditionally") - if doc.get("permissions") != {"contents": "read"}: - raise AssertionError("tests workflow must grant only contents: read") - if "id-token" in raw: - raise AssertionError("tests workflow must not request OIDC") - - for job_name, job in doc["jobs"].items(): - if "environment" in job: - raise AssertionError(f"tests job '{job_name}' must not use a protected environment") - - for forbidden in ("npm publish", "action-gh-release", "gh workflow run", "git push"): - if forbidden in raw: - raise AssertionError(f"tests workflow must not publish or dispatch: {forbidden}") - - triggers = trigger(doc) - if "workflow_dispatch" not in triggers: - raise AssertionError("tests workflow needs workflow_dispatch for the opt-in network smoke") - inputs = triggers["workflow_dispatch"].get("inputs", {}) or {} - if "network_smoke" not in inputs: - raise AssertionError("workflow_dispatch must expose the opt-in network_smoke input") - - pull_request = triggers.get("pull_request") - if not pull_request or "paths" not in pull_request: - raise AssertionError("pull_request trigger must be limited to relevant paths") - for required in ("scripts/cargo-audit-release-gate.sh", "scripts/tests/audit-release-gate/**"): - if required not in pull_request["paths"]: - raise AssertionError(f"pull_request paths must include {required}") - - if "unittest discover" not in raw: - raise AssertionError("tests workflow must run the deterministic offline suite") - - smoke = doc["jobs"].get("network-smoke") - if smoke is None: - raise AssertionError("tests workflow is missing the opt-in network smoke job") - condition = str(smoke.get("if", "")) - if "workflow_dispatch" not in condition or "inputs.network_smoke" not in condition: - raise AssertionError("network smoke job must be gated behind workflow_dispatch and inputs.network_smoke") - - -class WorkflowStructureTests(unittest.TestCase): - def test_publish_workflow(self): - assert_publish_workflow(load(PUBLISH)) - - def test_release_workflow(self): - assert_release_workflow(load(RELEASE)) - - def test_build_workflow(self): - assert_build_workflow(load(BUILD)) - - def test_tests_workflow(self): - assert_tests_workflow(load(TESTS), TESTS.read_text()) - - -class WorkflowMutationTests(unittest.TestCase): - def mutated(self, path, mutate): - doc = load(path) - mutate(doc) - return doc - - def assert_rejected(self, checker, doc, message): - with self.assertRaises(AssertionError, msg=message): - checker(doc) - - def test_disabled_or_error_masked_offline_tests_are_rejected(self): - raw = TESTS.read_text() - for mutation in ("job-error-mask", "step-error-mask", "conditional-skip"): - with self.subTest(mutation=mutation): - doc = load(TESTS) - offline = doc["jobs"]["offline"] - if mutation == "job-error-mask": - offline["continue-on-error"] = True - elif mutation == "step-error-mask": - offline["steps"][-1]["continue-on-error"] = True - else: - offline["if"] = False - with self.assertRaises(AssertionError): - assert_tests_workflow(doc, raw) - - def test_removing_gate_is_rejected(self): - def mutate(doc): - steps = doc["jobs"]["publish"]["steps"] - steps[:] = [step for step in steps if GATE_CALL not in step_run(step)] - - self.assert_rejected( - assert_publish_workflow, self.mutated(PUBLISH, mutate), "gate removal must be rejected" - ) - - def test_reordering_gate_after_publish_is_rejected(self): - def mutate(doc): - steps = doc["jobs"]["publish"]["steps"] - gate = next(step for step in steps if GATE_CALL in step_run(step)) - steps.remove(gate) - steps.append(gate) - - self.assert_rejected( - assert_publish_workflow, self.mutated(PUBLISH, mutate), "gate reorder must be rejected" - ) - - def test_continue_on_error_is_rejected(self): - def mutate(doc): - for step in doc["jobs"]["publish"]["steps"]: - if GATE_CALL in step_run(step): - step["continue-on-error"] = True - - self.assert_rejected( - assert_publish_workflow, self.mutated(PUBLISH, mutate), "continue-on-error must be rejected" - ) - - def test_always_condition_is_rejected(self): - def mutate(doc): - for step in doc["jobs"]["publish"]["steps"]: - if "npm publish" in step_run(step): - step["if"] = "always()" - - self.assert_rejected( - assert_publish_workflow, self.mutated(PUBLISH, mutate), "always() must be rejected" - ) - - def test_unpinned_source_ref_is_rejected(self): - def mutate(doc): - for step in doc["jobs"]["publish"]["steps"]: - if str(step.get("uses", "")).startswith("actions/checkout@"): - step["with"]["ref"] = "main" - - self.assert_rejected( - assert_publish_workflow, self.mutated(PUBLISH, mutate), "floating source ref must be rejected" - ) - - def test_missing_locked_flag_is_rejected(self): - def mutate(doc): - for job in doc["jobs"].values(): - for step in job.get("steps", []): - if "cargo build --release -p datadog-serverless-compat" in step_run(step): - step["run"] = step_run(step).replace(" --locked", "") - - self.assert_rejected( - assert_build_workflow, self.mutated(BUILD, mutate), "unlocked build must be rejected" - ) - - def test_missing_producer_source_ref_is_rejected(self): - def mutate(doc): - removed = False - for job in doc["jobs"].values(): - for step in job.get("steps", []): - if str(step.get("uses", "")).startswith("actions/checkout@"): - step.get("with", {}).pop("ref", None) - removed = True - self.assertTrue(removed) - - self.assert_rejected( - assert_build_workflow, self.mutated(BUILD, mutate), "unpinned producer checkout must be rejected" - ) - - def test_release_gate_after_draft_is_rejected(self): - def mutate(doc): - steps = doc["jobs"]["release"]["steps"] - gate = next(step for step in steps if GATE_CALL in step_run(step)) - steps.remove(gate) - steps.append(gate) - - self.assert_rejected( - assert_release_workflow, self.mutated(RELEASE, mutate), "late release gate must be rejected" - ) - - def test_non_draft_release_is_rejected(self): - def mutate(doc): - for step in doc["jobs"]["release"]["steps"]: - if str(step.get("uses", "")).startswith("softprops/action-gh-release@"): - step["with"]["draft"] = False - - self.assert_rejected( - assert_release_workflow, self.mutated(RELEASE, mutate), "draft change must be rejected" - ) - - def test_tests_workflow_oidc_is_rejected(self): - doc = load(TESTS) - doc["permissions"] = {"contents": "read", "id-token": "write"} - raw = TESTS.read_text() + "\n# id-token: write\n" - self.assert_rejected( - lambda candidate: assert_tests_workflow(candidate, raw), doc, "OIDC must be rejected" - ) - - # --- gate command masking --------------------------------------------- - - def mutate_gate_run(self, doc, replacement): - for step in doc["jobs"]["publish"]["steps"]: - if GATE_CALL in step_run(step): - step["run"] = replacement(step_run(step)) - return - raise AssertionError("gate step not found") - - def test_gate_or_true_masking_is_rejected(self): - self.assert_rejected( - assert_publish_workflow, - self.mutated(PUBLISH, lambda doc: self.mutate_gate_run(doc, lambda run: run + " || true")), - "|| true masking must be rejected", - ) - - def test_gate_exit_zero_masking_is_rejected(self): - self.assert_rejected( - assert_publish_workflow, - self.mutated(PUBLISH, lambda doc: self.mutate_gate_run(doc, lambda run: run + "; exit 0")), - "; exit 0 masking must be rejected", - ) - - def test_gate_set_plus_e_masking_is_rejected(self): - self.assert_rejected( - assert_publish_workflow, - self.mutated(PUBLISH, lambda doc: self.mutate_gate_run(doc, lambda run: "set +e\n" + run)), - "set +e masking must be rejected", - ) - - def test_gate_missing_expected_sha_is_rejected(self): - self.assert_rejected( - assert_publish_workflow, - self.mutated( - PUBLISH, - lambda doc: self.mutate_gate_run(doc, lambda run: "./scripts/cargo-audit-release-gate.sh"), - ), - "unbound gate command must be rejected", - ) - - def test_gate_conditional_false_is_rejected(self): - def mutate(doc): - for step in doc["jobs"]["publish"]["steps"]: - if GATE_CALL in step_run(step): - step["if"] = False - - self.assert_rejected( - assert_publish_workflow, self.mutated(PUBLISH, mutate), "conditional gate must be rejected" - ) - - def test_gate_condition_expression_is_rejected(self): - def mutate(doc): - for step in doc["jobs"]["publish"]["steps"]: - if GATE_CALL in step_run(step): - step["if"] = "${{ false }}" - - self.assert_rejected( - assert_publish_workflow, self.mutated(PUBLISH, mutate), "gate expression must be rejected" - ) - - def test_gate_shell_change_is_rejected(self): - def mutate(doc): - for step in doc["jobs"]["publish"]["steps"]: - if GATE_CALL in step_run(step): - step["shell"] = "sh" - - self.assert_rejected( - assert_publish_workflow, self.mutated(PUBLISH, mutate), "non-bash gate shell must be rejected" - ) - - def test_gate_working_directory_change_is_rejected(self): - def mutate(doc): - for step in doc["jobs"]["publish"]["steps"]: - if GATE_CALL in step_run(step): - step["working-directory"] = "." - - self.assert_rejected( - assert_publish_workflow, self.mutated(PUBLISH, mutate), "wrong gate directory must be rejected" - ) - - # --- job-level and artifact boundaries -------------------- - - def test_publish_job_continue_on_error_is_rejected(self): - def mutate(doc): - doc["jobs"]["publish"]["continue-on-error"] = True - - self.assert_rejected( - assert_publish_workflow, - self.mutated(PUBLISH, mutate), - "job-level continue-on-error must be rejected", - ) - - def test_publish_job_build_only_condition_removed_is_rejected(self): - def mutate(doc): - doc["jobs"]["publish"].pop("if", None) - - self.assert_rejected( - assert_publish_workflow, self.mutated(PUBLISH, mutate), "missing build-only condition must be rejected" - ) - - def test_publish_job_build_only_condition_replaced_is_rejected(self): - def mutate(doc): - doc["jobs"]["publish"]["if"] = "always()" - - self.assert_rejected( - assert_publish_workflow, self.mutated(PUBLISH, mutate), "always() build condition must be rejected" - ) - - def test_publisher_cancelled_condition_is_rejected(self): - def mutate(doc): - for step in doc["jobs"]["publish"]["steps"]: - if "npm publish" in step_run(step): - step["if"] = "cancelled()" - - self.assert_rejected( - assert_publish_workflow, self.mutated(PUBLISH, mutate), "cancelled() masking must be rejected" - ) - - def test_npm_publish_condition_is_rejected(self): - def mutate(doc): - for step in doc["jobs"]["publish"]["steps"]: - if "npm publish" in step_run(step): - step["if"] = "${{ always() }}" - - self.assert_rejected( - assert_publish_workflow, self.mutated(PUBLISH, mutate), "conditional publish must be rejected" - ) - - def test_external_artifact_selectors_are_rejected(self): - for selector in EXTERNAL_ARTIFACT_SELECTORS: - with self.subTest(selector=selector): - def mutate(doc, selector=selector): - for step in doc["jobs"]["publish"]["steps"]: - if str(step.get("uses", "")).startswith(DOWNLOAD_ARTIFACT_PREFIX): - step.setdefault("with", {})[selector] = "from-elsewhere" - return - raise AssertionError("no download-artifact step") - - self.assert_rejected( - assert_publish_workflow, - self.mutated(PUBLISH, mutate), - f"artifact selector {selector} must be rejected", - ) - - def test_release_external_artifact_selector_is_rejected(self): - def mutate(doc): - for step in doc["jobs"]["release"]["steps"]: - if str(step.get("uses", "")).startswith(DOWNLOAD_ARTIFACT_PREFIX): - step.setdefault("with", {})["run-id"] = "1234" - return - raise AssertionError("no download-artifact step") - - self.assert_rejected( - assert_release_workflow, self.mutated(RELEASE, mutate), "release artifact selector must be rejected" - ) - - def test_package_checkout_ref_change_is_rejected(self): - def mutate(doc): - for step in doc["jobs"]["package"]["steps"]: - if str(step.get("uses", "")).startswith("actions/checkout@"): - step["with"]["ref"] = "main" - - self.assert_rejected( - assert_publish_workflow, self.mutated(PUBLISH, mutate), "floating package checkout must be rejected" - ) - - def test_package_artifact_name_change_is_rejected(self): - def mutate(doc): - for step in doc["jobs"]["package"]["steps"]: - if str(step.get("uses", "")).startswith("actions/upload-artifact@"): - step["with"]["name"] = "other-packages" - - self.assert_rejected( - assert_publish_workflow, self.mutated(PUBLISH, mutate), "renamed package artifact must be rejected" - ) - - def test_release_needs_change_is_rejected(self): - def mutate(doc): - doc["jobs"]["release"]["needs"] = None - - self.assert_rejected( - assert_release_workflow, self.mutated(RELEASE, mutate), "release dependency change must be rejected" - ) - - def test_release_extra_environment_is_rejected(self): - def mutate(doc): - doc["jobs"]["release"]["environment"] = "Production" - - self.assert_rejected( - assert_release_workflow, self.mutated(RELEASE, mutate), "extra release approval must be rejected" - ) +def action_of(step): + return str(step.get("uses", "")).split("@")[0] + + +def runs(text): + return lambda step: text in run_of(step) + + +def using(action): + return lambda step: action_of(step) == action + + +def first(steps, predicate): + return next((index for index, step in enumerate(steps) if predicate(step)), None) + + +def withs(job, action): + return [step.get("with") or {} for step in job.get("steps", []) if action_of(step) == action] + + +def shell_commands(run): + """Yield the argv of each shell command in a run block, skipping comments.""" + for line in run.replace("\\\n", " ").splitlines(): + argv = [] + for token in shlex.split(line, comments=True) + [";"]: + if token in SHELL_SEPARATORS: + yield argv + argv = [] + else: + argv.append(token) + + +class WorkflowTests(unittest.TestCase): + def check_common(self, doc, where): + for name, job in doc["jobs"].items(): + for item in (job, *job.get("steps", [])): + self.assertNotIn("continue-on-error", item, f"{where}/{name} ignores errors") + action = action_of(item) + if action in PINS: + self.assertEqual(item["uses"], PINS[action], f"{where}: {action} pin changed") + if action == "actions/download-artifact": + selectors = CROSS_RUN_SELECTORS & set(item.get("with") or {}) + self.assertFalse(selectors, f"{where}: cross-run artifact download") + + def check_gated_job(self, job, is_effect, where): + steps = job["steps"] + gate = first(steps, runs("cargo-audit-release-gate.sh")) + self.assertIsNotNone(gate, f"{where}: gate missing") + allowed = {"name", "run", "working-directory"} + self.assertLessEqual(set(steps[gate]), allowed, f"{where}: gate must be unconditional and unmasked") + self.assertEqual(run_of(steps[gate]).strip(), GATE_COMMAND, f"{where}: gate command changed") + self.assertEqual(steps[gate].get("working-directory"), "source", f"{where}: gate directory") + + checkout = first(steps, using("actions/checkout")) + rust = first(steps, using("actions-rust-lang/setup-rust-toolchain")) + self.assertIsNotNone(checkout, f"{where}: source checkout missing") + self.assertIsNotNone(rust, f"{where}: Rust setup missing") + source = steps[checkout].get("with") or {} + self.assertEqual((source.get("ref"), source.get("path")), (SOURCE_REF, "source"), f"{where}: source checkout") + self.assertIs((steps[rust].get("with") or {}).get("cache"), False, f"{where}: Rust setup cache") + self.assertLess(max(checkout, rust), gate, f"{where}: checkout and Rust setup must precede the gate") + + effect = first(steps, is_effect) + self.assertIsNotNone(effect, f"{where}: publication effect missing") + self.assertLess(gate, effect, f"{where}: gate must precede the first publication effect") + for later in steps[gate:]: + self.assertNotIn("if", later, f"{where}: step conditional after the gate") + + def check_publish(self, doc): + self.check_common(doc, "publish") + self.assertEqual(doc["permissions"], {"contents": "read", "id-token": "write"}, "publish permissions") + inputs = doc["on"]["workflow_dispatch"]["inputs"] + self.assertEqual(set(inputs), {"publish", "version"}, "publish inputs changed") + self.assertEqual(inputs["publish"]["options"], ["Build", "Build and Publish"], "publish options") + + package, publish = doc["jobs"]["package"], doc["jobs"]["publish"] + self.assertEqual(publish.get("environment"), "Production", "publish lost Production approval") + self.assertEqual(publish.get("needs"), "package", "publish must need package") + self.assertEqual(publish.get("if"), BUILD_ONLY, "publish lost the build-only condition") + package_refs = [w.get("ref") for w in withs(package, "actions/checkout")] + self.assertEqual(package_refs, [SOURCE_REF], "package checkout ref") + self.assertIn("npm-packages", [w.get("name") for w in withs(package, "actions/upload-artifact")]) + self.assertIn("npm-packages", [w.get("name") for w in withs(publish, "actions/download-artifact")]) + self.check_gated_job(publish, runs("npm publish"), "publish") + + def check_release(self, doc): + self.check_common(doc, "release") + self.assertEqual(doc["permissions"], {}, "release top-level permissions must stay empty") + release = doc["jobs"]["release"] + # Reviewed flow: the strict gate plus a draft release is the release + # check, so no extra protected environment approval is introduced. + self.assertNotIn("environment", release, "release gained a protected environment") + self.assertNotIn("if", release, "release job became conditional") + self.assertEqual(release.get("needs"), "build", "release must need build") + self.assertEqual(release.get("permissions"), {"contents": "write"}, "release job permissions") + + def is_effect(step): + return run_of(step).lstrip().startswith("zip ") or action_of(step) == "softprops/action-gh-release" + + self.check_gated_job(release, is_effect, "release") + drafts = [w.get("draft") is True for w in withs(release, "softprops/action-gh-release")] + self.assertEqual(drafts, [True], "release must stay a single draft") + + def check_build(self, doc): + self.check_common(doc, "builder") + refs = [w.get("ref") for job in doc["jobs"].values() for w in withs(job, "actions/checkout")] + self.assertEqual(refs, [SOURCE_REF, SOURCE_REF], "builder checkouts must pin the run commit") + run_blocks = [run_of(step) for job in doc["jobs"].values() for step in job.get("steps", [])] + self.assertFalse([run for run in run_blocks if "audit" in run], "builder must not audit") + builds = [argv for run in run_blocks for argv in shell_commands(run) if argv[:2] == ["cargo", "build"]] + self.assertEqual(len(builds), 5, f"expected 5 release builds, found {builds}") + for argv in builds: + self.assertLessEqual({"--release", "--locked"}, set(argv), f"unlocked build: {argv}") + + def test_real_publish_workflow(self): + self.check_publish(load(PUBLISH)) + + def test_real_release_workflow(self): + self.check_release(load(RELEASE)) + + def test_real_build_workflow(self): + self.check_build(load(BUILD)) + + def test_mutations_are_rejected(self): + checks = {PUBLISH: self.check_publish, RELEASE: self.check_release, BUILD: self.check_build} + for context, name, mutate, reason in MUTATIONS: + with self.subTest(context): + doc = load(name) + mutate(doc) # LookupError or KeyError here means the mutation target moved. + with self.assertRaisesRegex(AssertionError, reason): + checks[name](doc) + + +# Mutation helpers fail loudly when their target is absent or already holds the +# requested value, so a case cannot pass by mutating nothing. +def step(doc, job, predicate): + matches = [s for s in doc["jobs"][job]["steps"] if predicate(s)] + if len(matches) != 1: + raise LookupError(f"mutation target in job '{job}' matched {len(matches)} steps") + return matches[0] + + +def put(target, key, value): + if target.get(key) == value: + raise LookupError(f"'{key}' already equals {value!r}") + target[key] = value + + +def edit_run(target, old, new): + if old not in run_of(target): + raise LookupError(f"'{old}' not found in run block") + target["run"] = run_of(target).replace(old, new) + + +def move_to_end(doc, job, predicate): + steps = doc["jobs"][job]["steps"] + steps.append(steps.pop(steps.index(step(doc, job, predicate)))) + + +GATE = runs("cargo-audit-release-gate.sh") +BUILDER = "build-datadog-serverless-compat" +MACOS_BUILD = runs("datadog-serverless-compat --locked") + +# (context, workflow, mutation, expected rejection reason) +MUTATIONS = [ + ("publish gate removed", PUBLISH, + lambda d: d["jobs"]["publish"]["steps"].remove(step(d, "publish", GATE)), "gate missing"), + ("publish gate moved after npm publish", PUBLISH, + lambda d: move_to_end(d, "publish", GATE), "gate must precede"), + ("release gate moved after the zip and draft release", RELEASE, + lambda d: move_to_end(d, "release", GATE), "gate must precede"), + ("publish gate masked with || true", PUBLISH, + lambda d: edit_run(step(d, "publish", GATE), '"$GITHUB_SHA"', '"$GITHUB_SHA" || true'), "gate command"), + ("publish gate skipped with if: false", PUBLISH, + lambda d: put(step(d, "publish", GATE), "if", False), "unconditional and unmasked"), + ("release gate ignores its own failure", RELEASE, + lambda d: put(step(d, "release", GATE), "continue-on-error", True), "ignores errors"), + ("publish gate no longer bound to the run commit", PUBLISH, + lambda d: edit_run(step(d, "publish", GATE), ' "$GITHUB_SHA"', ""), "gate command"), + ("publish source checkout floats to main", PUBLISH, + lambda d: put(step(d, "publish", using("actions/checkout"))["with"], "ref", "main"), "source checkout"), + ("builder setup checkout floats to main", BUILD, + lambda d: put(step(d, "setup", using("actions/checkout"))["with"], "ref", "main"), "builder checkouts"), + ("publish downloads npm packages from another run", PUBLISH, + lambda d: put(step(d, "publish", using("actions/download-artifact"))["with"], "run-id", "1234"), + "cross-run artifact"), + ("publish loses Production approval", PUBLISH, + lambda d: d["jobs"]["publish"].pop("environment"), "Production approval"), + ("publish also runs on Build-only dispatches", PUBLISH, + lambda d: put(d["jobs"]["publish"], "if", "${{ always() }}"), "build-only condition"), + ("npm publish step made conditional", PUBLISH, + lambda d: put(step(d, "publish", runs("npm publish")), "if", "${{ always() }}"), "conditional after the gate"), + ("draft release becomes public", RELEASE, + lambda d: put(step(d, "release", using("softprops/action-gh-release"))["with"], "draft", False), + "single draft"), + ("i686 backslash-continued build drops --locked", BUILD, + lambda d: edit_run(step(d, BUILDER, runs("--target i686-pc-windows-msvc")), "--locked", ""), + "unlocked build"), + ("macOS build keeps --locked only inside a shell comment", BUILD, + lambda d: edit_run(step(d, BUILDER, MACOS_BUILD), "--locked", "# --locked"), "unlocked build"), + ("macOS build commented out", BUILD, + lambda d: edit_run(step(d, BUILDER, MACOS_BUILD), "cargo", "# cargo"), "expected 5 release builds"), +] if __name__ == "__main__":