diff --git a/.github/base-tier-skips-iouring.txt b/.github/base-tier-skips-iouring.txt new file mode 100644 index 000000000..63a6af39d --- /dev/null +++ b/.github/base-tier-skips-iouring.txt @@ -0,0 +1,26 @@ +# Tests that skip at CELERIS_MAX_IOURING_TIER=base on purpose, for .github/scripts/base-tier-tally.sh +# (celeris#890 item 87): the full test or subtest name, then why. A name covers its own +# subtests. Measured on the run of the job iouring-base-tier cited in the pull request: +# 554 top-level tests passed, 4 skipped at the top level, 12 SKIP lines in all. On main +# 8936e89 (with #1123 and #1127) a docker run (kernel without an fe80:: address) gave 577 +# passed and 13 SKIP lines, the 13th being TestListenOnAZonedLinkLocalServes885, which +# the job does not allow: it sets CELERIS_REQUIRE_ZONED_PEER=1, so the test fails rather +# than skips on a runner without one. +min-pass 540 + +# Tier base has no multishot recv with provided buffers (the tests say so in their skip). +TestSpentDetectBudgetReturnsItsRingBuffer/check no provided buffer ring at the base tier +TestSpentDetectBudgetReturnsItsRingBuffer/recheck no provided buffer ring at the base tier +TestSpentDetectBudgetReturnsItsRingBuffer/check-async no provided buffer ring at the base tier +TestSpentDetectBudgetReturnsItsRingBuffer/recheck-async no provided buffer ring at the base tier +TestStartupLineReportsTheMultishotRecvThatIsInUse/unset tier base has no multishot recv +TestStartupLineReportsTheMultishotRecvThatIsInUse/zero tier base has no multishot recv +TestStartupLineReportsTheMultishotRecvThatIsInUse/true-is-not-1 tier base has no multishot recv +TestStartupLineReportsTheMultishotRecvThatIsInUse/one tier base has no multishot recv +TestMultishotRecvDecidedOnceAtNew/on_at_New,_off_after tier base has no multishot recv +TestMultishotRecvDecidedOnceAtNew/off_at_New,_on_after tier base has no multishot recv + +# They need net.ipv4.tcp_synack_retries=0; the runner has 5. The step "celeris#662 +# tcp_synack_retries=0 guard" of the job adaptive sets it and forbids the skip. +TestPauseAcceptServesDeferredIdleConnectionsSynackRetriesZero needs net.ipv4.tcp_synack_retries=0 +TestPauseAcceptNoGuardLosesAtSynackRetriesZero needs net.ipv4.tcp_synack_retries=0 diff --git a/.github/base-tier-skips-root.txt b/.github/base-tier-skips-root.txt new file mode 100644 index 000000000..93e24be4e --- /dev/null +++ b/.github/base-tier-skips-root.txt @@ -0,0 +1,25 @@ +# Tests that skip at CELERIS_MAX_IOURING_TIER=base on purpose, for .github/scripts/base-tier-tally.sh +# (celeris#890 item 87): the full test or subtest name, then why. A name covers its own +# subtests. Measured on the run of the job iouring-base-tier cited in the pull request: +# 107 top-level tests passed, 3 skipped at the top level, 19 SKIP lines in all (and +# 117 passed with 46 SKIP lines once main had brought in nine more tests of the kind below). +# On main 8936e89 (docker, merged tree): 118 passed, 46 SKIP lines, all allowed. +min-pass 105 + +# The probes of the tests that gate their io_uring arms (c714ProbeIOUring, +# keptProbeIOUring, h2_handler_return_889's) call io_uring "usable" only at the high +# tier with provided buffers, so with the cap their io_uring and adaptive arms skip with +# "io_uring tier=base kernel=...: no usable io_uring". The std and epoll arms of the +# same tests run. A new test of this kind joins them without an edit here, because the +# entry names the arms, not the tests (run 40419578127 met nine new ones that a merge +# with main brought in: the TestH1Unclosed...889 family); the arm must have logged the +# probe's own skip message ("io_uring tier=base kernel=...") while it was the current test, so an +# arm that starts an engine and skips for another reason still fails the step. +pattern:^.+/(io_uring|adaptive|adaptive-iouring)$ the io_uring, adaptive and adaptive-iouring arms of probe-gated tests + +# Tier base has no multishot recv, and these two tests are about it. +TestHijackCopiesRequestValuesUnderMultishotRecv tier base has no multishot recv +TestMultishotRingSurvivesHijacks tier base has no multishot recv + +# Its probe (keptProbeIOUring) calls io_uring usable only at the high tier: "no usable io_uring". +TestServerAsyncRouteDetachThenGoexitOnIOUring no usable io_uring at the base tier diff --git a/.github/dependabot.yml b/.github/dependabot.yml index d5efb6158..03e7b1a2a 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -92,6 +92,26 @@ updates: commit-message: prefix: "deps" + # ----- CI Python tools ----- + # zizmor, the workflow security audit of the Lint job, is the one CI tool + # that is not a `tool` directive of .github/tools/go.mod. Its pin is + # .github/tools/requirements.txt, installed by `pip install -r`. The Lint step + # "Tool pins agree" (.github/scripts/tool-pins-check.sh, check 4) fails when a + # workflow names a `name==version` of its own, and when a requirements file + # under .github/ has no entry like this one. No `allow` rule: the file lists + # direct dependencies only. A bump of zizmor can add or change findings of the + # Lint job; the pull request shows them. + - package-ecosystem: pip + directory: /.github/tools + schedule: + interval: daily + groups: + ci-python-tools: + patterns: + - "*" + commit-message: + prefix: "deps" + # ----- GitHub Actions ----- # Watches every workflow under .github/workflows. Group all action # version bumps into ONE PR per scheduled run. diff --git a/.github/scripts/base-tier-tally-test.sh b/.github/scripts/base-tier-tally-test.sh new file mode 100755 index 000000000..62bc9e041 --- /dev/null +++ b/.github/scripts/base-tier-tally-test.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash +# Self-test of base-tier-tally.sh on fixture logs (celeris#890 item 87). Run from +# the repository root; the Lint job runs it, so a change to the tally that stops +# it failing on a skip nobody allowed, on a cap that did not take effect or on a +# test that is missing is a red Lint. +# +# BASE_TIER_TALLY=path the tally to test (default .github/scripts/base-tier-tally.sh) +set -uo pipefail + +tally=${BASE_TIER_TALLY:-.github/scripts/base-tier-tally.sh} +dir=$(mktemp -d) +trap 'rm -rf "$dir"' EXIT +fail=0 +n=0 +export BASE_TIER_SKIPS_DIR=$dir +printf '# header\nmin-pass 2\nTestNeedsHighTier needs a provided buffer ring\nTestDot.Name a name with a regular expression character\n' >"$dir/base-tier-skips-iouring.txt" +printf '# header\nmin-pass 1\npattern:^.+/(io_uring|adaptive)$ the arms of probe-gated tests\nTestNeedsHighTier needs a provided buffer ring\n' >"$dir/base-tier-skips-root.txt" + +# expect NAME WANT_STATUS WANT_STDERR_SUBSTRING LABEL LOG [TESTS...] +expect() { + local name=$1 want=$2 sub=$3 status=0 + shift 3 + "$tally" "$@" >"$dir/out" 2>"$dir/err" || status=$? + n=$((n + 1)) + if [ "$status" -ne "$want" ]; then + echo "FAIL $name: exit $status, want $want"; cat "$dir/out" "$dir/err"; fail=1 + elif [ -n "$sub" ] && ! grep -qF -e "$sub" "$dir/err" "$dir/out"; then + echo "FAIL $name: output does not say '$sub':"; cat "$dir/out" "$dir/err"; fail=1 + else + echo "ok $name" + fi +} + +# good.log: the shape of an iouring log at the base tier. 806 and the tier test passed, +# one allowed test skipped (with a subtest). +good() { + printf '%s\n' \ + '=== RUN TestShutdownDetachedPartialWriteIsFlushed806' '--- PASS: TestShutdownDetachedPartialWriteIsFlushed806 (0.10s)' \ + '=== RUN TestReportTier844' \ + ' async_abort_844_engine_linux_test.go:160: celeris844 PROBE kernel=6.11.0 io_uring_tier=base CELERIS_MAX_IOURING_TIER="base"' \ + '--- PASS: TestReportTier844 (0.00s)' \ + '=== RUN TestNeedsHighTier' '=== RUN TestNeedsHighTier/sub' ' --- SKIP: TestNeedsHighTier/sub (0.00s)' '--- SKIP: TestNeedsHighTier (0.00s)' \ + ' async_abort_844_linux_test.go:518: celeris844 RESULT engine=io_uring tier=base arm=park inject=true' \ + 'PASS' +} +good >"$dir/good.log" +expect "good iouring log" 0 "" iouring "$dir/good.log" TestShutdownDetachedPartialWriteIsFlushed806 TestReportTier844 +sed 's/NeedsHighTier/NeedsSomething/' "$dir/good.log" >"$dir/unlisted.log" +expect "a skip nobody allowed fails" 1 "TestNeedsSomething" iouring "$dir/unlisted.log" TestShutdownDetachedPartialWriteIsFlushed806 +sed 's/tier=base arm=park/tier=high arm=park/' "$dir/good.log" >"$dir/high.log" +expect "an arm at tier=high fails" 1 "the cap did not take effect" iouring "$dir/high.log" TestShutdownDetachedPartialWriteIsFlushed806 +# A name is literal: the dot of an allowed name is no wildcard. +{ cat "$dir/good.log"; printf '%s\n' '--- SKIP: TestDot.Name (0.00s)' '--- SKIP: TestDotXName (0.00s)'; } >"$dir/dot.log" +expect "a regular expression character in an allowed name is literal" 1 "TestDotXName" iouring "$dir/dot.log" +grep -v 'TestDotXName' "$dir/dot.log" >"$dir/dot-ok.log" +expect "the allowed name itself passes" 0 "" iouring "$dir/dot-ok.log" +grep -v 'celeris844 PROBE' "$dir/good.log" >"$dir/noprobe.log" +expect "a missing capped probe line fails" 1 "the cap did not take effect" iouring "$dir/noprobe.log" TestShutdownDetachedPartialWriteIsFlushed806 +sed 's/CELERIS_MAX_IOURING_TIER="base"/CELERIS_MAX_IOURING_TIER=""/' "$dir/good.log" >"$dir/nocap.log" +expect "a probe line of an uncapped run fails" 1 "the cap did not take effect" iouring "$dir/nocap.log" TestShutdownDetachedPartialWriteIsFlushed806 +grep -v 'TestShutdownDetachedPartialWriteIsFlushed806' "$dir/good.log" >"$dir/no806.log" +expect "a named test that never ran fails" 1 "TestShutdownDetachedPartialWriteIsFlushed806 ran 0" iouring "$dir/no806.log" TestShutdownDetachedPartialWriteIsFlushed806 +sed 's/^--- PASS: TestShutdownDetachedPartialWriteIsFlushed806/--- SKIP: TestShutdownDetachedPartialWriteIsFlushed806/' "$dir/good.log" >"$dir/skip806.log" +expect "a named test that skipped fails" 1 "TestShutdownDetachedPartialWriteIsFlushed806" iouring "$dir/skip806.log" TestShutdownDetachedPartialWriteIsFlushed806 +# min-pass: a run that lost its tests (a -run pattern that stopped matching). +grep -v 'TestShutdownDetachedPartialWriteIsFlushed806' "$dir/good.log" | sed 's/^--- PASS: TestReportTier844/--- XXX: TestReportTier844/' >"$dir/shrunk.log" +expect "fewer top-level PASS lines than min-pass fails" 1 "fewer than the 2" iouring "$dir/shrunk.log" +# The root label has no tier lines. A pattern entry allows an arm only while the log +# shows tier=base under that arm's RUN; a top-level test, or an arm that skips for +# another reason, is not allowed by it. +printf '%s\n' '=== RUN TestA' '--- PASS: TestA (0.00s)' 'PASS' >"$dir/root.log" +expect "root: a clean log" 0 "" root "$dir/root.log" +printf '%s\n' '=== RUN TestA' '--- SKIP: TestA (0.00s)' '--- PASS: TestB (0.00s)' 'PASS' >"$dir/rootskip.log" +expect "root: a skip no entry names fails" 1 "TestA" root "$dir/rootskip.log" +arm() { # arm NAME MESSAGE: a probe-gated test whose NAME arm skipped with MESSAGE + printf '%s\n' '=== RUN TestP' "=== RUN TestP/$1" " p_test.go:9: $2" '=== RUN TestP/std' '--- PASS: TestP (0.00s)' \ + " --- SKIP: TestP/$1 (0.00s)" ' --- PASS: TestP/std (0.00s)' 'PASS' +} +arm io_uring 'io_uring tier=base kernel=6.17: no usable io_uring' >"$dir/arm.log" +expect "root: an arm that logged tier=base is allowed by the pattern" 0 "" root "$dir/arm.log" +# An arm that started an engine (the engine's INFO line says tier=base) and then skipped +# for another reason is not excused by the pattern. +printf '%s\n' '=== RUN TestP' '=== RUN TestP/io_uring' '2026/10/26 23:34:23 INFO io_uring engine selected tier=base multishot_accept=false' \ + ' p_test.go:9: the peer never connected' '--- PASS: TestP (0.00s)' ' --- SKIP: TestP/io_uring (0.00s)' >"$dir/arm-engine.log" +expect "root: an arm that started an engine and skipped for another reason fails" 1 "TestP/io_uring" root "$dir/arm-engine.log" +# Parallel subtests interleave their output and go test marks the switch with `=== NAME` +# (and `=== CONT` after a pause): a probe message printed under one arm must not be +# credited to another arm that skipped for a different reason (review r1 of #1124, m1). +printf '%s\n' '=== RUN TestP' '=== RUN TestP/io_uring' '=== PAUSE TestP/io_uring' '=== RUN TestQ' '=== RUN TestQ/io_uring' '=== PAUSE TestQ/io_uring' \ + '=== CONT TestP/io_uring' ' p_test.go:9: io_uring tier=base kernel=6.17: no usable io_uring' \ + '=== NAME TestQ/io_uring' ' q_test.go:12: listener never became ready' \ + '--- PASS: TestP (0.00s)' ' --- SKIP: TestP/io_uring (0.00s)' '--- PASS: TestQ (0.00s)' ' --- SKIP: TestQ/io_uring (0.00s)' >"$dir/arm-interleaved.log" +expect "root: a probe message of one parallel arm does not excuse another's skip" 1 "TestQ/io_uring" root "$dir/arm-interleaved.log" +printf '%s\n' '=== RUN TestP' '=== RUN TestP/io_uring' '=== PAUSE TestP/io_uring' '=== RUN TestP/adaptive' '=== PAUSE TestP/adaptive' \ + '=== CONT TestP/io_uring' ' p_test.go:9: io_uring tier=base kernel=6.17: no usable io_uring' \ + '=== CONT TestP/adaptive' ' p_test.go:9: io_uring tier=base kernel=6.17: no usable io_uring' \ + '--- PASS: TestP (0.00s)' ' --- SKIP: TestP/io_uring (0.00s)' ' --- SKIP: TestP/adaptive (0.00s)' >"$dir/arm-interleaved-ok.log" +expect "root: parallel arms that each logged the probe message are allowed" 0 "" root "$dir/arm-interleaved-ok.log" +# The pattern's evidence is the message the comments name, not any tier=base line of a probe: +# a Logf that merely says so, or another tier=base message, is no excuse (m2). +arm io_uring 'io_uring tier=base kernel=6.17: io_uring arms not run' >"$dir/arm-notrun.log" +expect "root: another tier=base message is not the probe's skip message" 1 "TestP/io_uring" root "$dir/arm-notrun.log" +arm io_uring 'the kernel has no io_uring' >"$dir/arm-other.log" +expect "root: an arm that skipped for another reason fails" 1 "TestP/io_uring" root "$dir/arm-other.log" +arm epoll 'io_uring tier=base kernel=6.17: no usable io_uring' >"$dir/arm-epoll.log" +expect "root: an arm the pattern does not name fails" 1 "TestP/epoll" root "$dir/arm-epoll.log" +printf '%s\n' '=== RUN io_uring' ' p_test.go:9: io_uring tier=base: no usable io_uring' '--- SKIP: io_uring (0.00s)' '--- PASS: TestB (0.00s)' >"$dir/arm-top.log" +expect "root: a top-level test named like an arm fails" 1 "SKIP: io_uring" root "$dir/arm-top.log" +expect "a label that is neither fails with usage" 2 "neither iouring nor root" other "$dir/root.log" +expect "an unreadable log is a usage error" 2 "cannot read" root "$dir/absent.log" + +echo "$n cases" +exit "$fail" diff --git a/.github/scripts/base-tier-tally.sh b/.github/scripts/base-tier-tally.sh new file mode 100755 index 000000000..82940b4a3 --- /dev/null +++ b/.github/scripts/base-tier-tally.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +# base-tier-tally.sh LABEL LOG [NAME...]: the tally of a `go test -v` log run +# with CELERIS_MAX_IOURING_TIER=base (celeris#890 item 87). +# +# LABEL is "iouring" (the log of ./internal/engine/iouring) or "root" (the log of +# the root package and ./test/server). Three checks, each failing the step: +# +# 1. No test skips except the ones the allow-list of that label names, each +# with the reason it may skip at the base tier. A SKIP line is matched by +# the test's full name, a subtest included (`--- SKIP: Test/sub (`), and +# an entry covers its own subtests. A skip is not coverage; a test that +# needs the high tier says so here. +# 2. Every NAME is a top-level test that ran once and PASSED, with no SKIP or +# FAIL line (named-tally.sh). +# 3. For "iouring": the cap took effect. TestReportTier844 printed the capped +# probe once as io_uring_tier=base with the variable set to "base", and no +# engine arm of the celeris#844 tests reported tier=high. A value the +# probe does not know counts as "none", which would skip every io_uring +# test and leave check 1 as the only witness; this one names the cause. +# +# 4. At least `min-pass` top-level tests passed (the data file of the label). +# +# Prints one summary line on stdout. Exit status 0 all hold, 1 some do not, +# 2 usage or an unreadable log. +set -euo pipefail + +[ "$#" -ge 2 ] || { echo "usage: base-tier-tally.sh LABEL LOG [NAME...]" >&2; exit 2; } +label=$1 +log=$2 +shift 2 +[ -r "$log" ] || { echo "base-tier-tally: cannot read $log" >&2; exit 2; } +here=$(cd "$(dirname "$0")" && pwd) + +# Allowed skips: .github/base-tier-skips-LABEL.txt, one entry per line, then the reason +# (blank lines and # lines are ignored; BASE_TIER_SKIPS_DIR points elsewhere for the +# self-test). An entry is +# NAME a test or subtest by its full name; it covers its own subtests, or +# pattern:REGEX a skipped test or subtest whose full name matches REGEX, and only +# when the test itself logged the probe's skip message while it was the +# current test ("file_test.go:N: io_uring tier=base kernel=...: no +# usable io_uring"). The engine's own startup line ("INFO io_uring engine +# selected tier=base ...") does not count: an arm that started an +# engine and then skipped for another reason must still fail. +# min-pass N the top-level tests that must PASS at least, so that a -run pattern +# that stops matching (a rename) or a package that stops compiling its +# tests in cannot shrink the run unseen; the figure is measured. +case "$label" in + iouring | root) ;; + *) echo "base-tier-tally: label '$label' is neither iouring nor root" >&2; exit 2 ;; +esac +list_file=${BASE_TIER_SKIPS_DIR:-$here/..}/base-tier-skips-$label.txt +[ -r "$list_file" ] || { echo "base-tier-tally: cannot read $list_file" >&2; exit 2; } +# Names are literal (compared as strings below, not as regular expressions). +names=$(awk 'NF && $1 !~ /^#/ && $1 != "min-pass" && $1 !~ /^pattern:/ { print $1 }' "$list_file" | paste -sd'|' -) +patterns=$(awk 'NF && $1 ~ /^pattern:/ { sub(/^pattern:/, "", $1); print $1 }' "$list_file" | paste -sd'|' -) +minpass=$(awk '$1 == "min-pass" { print $2 }' "$list_file") + +fail=0 +skipped=$("$here"/skip-count.sh "$log") || fail=1 +# Each SKIP line is judged by the name of the test: allowed by an exact entry (or as a +# subtest of one), or by a pattern while the test logged the probe's tier=base skip message. +unlisted=$(awk -v names="$names" -v pats="$patterns" 'BEGIN { nn = (names == "") ? 0 : split(names, nm, "|") } + # Parallel subtests interleave their output; go test names the owner of the lines that + # follow with RUN, and with NAME or CONT when it switches back to a test. + /^=== (RUN|NAME|CONT) / { cur = $3; next } + /^[[:space:]]+[A-Za-z0-9_.]+\.go:[0-9]+: .*io_uring tier=base kernel=.*: no usable io_uring/ && cur != "" { base[cur] = 1 } + /^[[:space:]]*--- SKIP: / { + n = $3 + ok = 0 + for (i = 1; i <= nn; i++) if (n == nm[i] || index(n, nm[i] "/") == 1) ok = 1 + if (!ok && pats != "" && n ~ pats && base[n]) ok = 1 + if (!ok) print + } +' "$log") +n_unlisted=0 +[ -n "$unlisted" ] && n_unlisted=$(printf '%s\n' "$unlisted" | wc -l | tr -d '[:space:]') + +tiers= +if [ "$label" = iouring ]; then + probe=$(grep -acE 'celeris844 PROBE .* io_uring_tier=base CELERIS_MAX_IOURING_TIER="base"$' "$log" || true) + high=$(grep -acE 'celeris844 RESULT engine=io_uring tier=high ' "$log" || true) + base=$(grep -acE 'celeris844 RESULT engine=io_uring tier=base ' "$log" || true) + tiers=", capped probe lines $probe (want 1), engine arms at tier=base $base (want > 0) and tier=high $high (want 0)" + if [ "$probe" -ne 1 ] || [ "$high" -ne 0 ] || [ "$base" -eq 0 ]; then + fail=1 + echo "base-tier-tally: the cap did not take effect: CELERIS_MAX_IOURING_TIER=base is not the tier the io_uring arms ran on" >&2 + fi +fi + +passed=$(grep -acE '^--- PASS: Test' "$log" || true) +if [ -n "$minpass" ] && [ "$passed" -lt "$minpass" ]; then + fail=1 + echo "base-tier-tally: $passed top-level tests passed, fewer than the $minpass of $list_file: did the run shrink (a renamed test, a -run pattern that no longer matches)?" >&2 +fi +echo "base-tier-tally $label: top-level PASS $passed (min ${minpass:-none}), SKIP lines $skipped, not in the allow-list $n_unlisted$tiers" +if [ "$n_unlisted" -ne 0 ]; then + fail=1 + printf '%s\n' "$unlisted" >&2 + echo "base-tier-tally: tests skipped at the base tier that the allow-list of '$label' does not name;" >&2 + echo "add a name only with the reason it must skip at this tier (a skip is not coverage)" >&2 +fi +if [ "$#" -gt 0 ]; then + "$here"/named-tally.sh "base tier $label" "$log" "$@" || fail=1 +fi +exit "$fail" diff --git a/.github/scripts/tool-pins-check-test.sh b/.github/scripts/tool-pins-check-test.sh index 67102c701..bc12418b4 100755 --- a/.github/scripts/tool-pins-check-test.sh +++ b/.github/scripts/tool-pins-check-test.sh @@ -24,6 +24,11 @@ fixture() { mkdir -p "$dir/$1/.github/actions/probe" printf 'name: probe\nruns:\n using: composite\n steps:\n - run: echo ok\n shell: bash\n' >"$dir/$1/.github/actions/probe/action.yml" cp "$repo/CONTRIBUTING.md" "$dir/$1/" + # The requirements files of the Python tools (check 4), where there are any. + find "$repo/.github" -name 'requirements*.txt' -type f | while IFS= read -r f; do + mkdir -p "$dir/$1/$(dirname "${f#"$repo"/}")" + cp "$f" "$dir/$1/${f#"$repo"/}" + done } # expect NAME WANT_STATUS WANT_STDERR_SUBSTRING: runs the check on $dir/NAME. @@ -142,5 +147,84 @@ fixture go-run-local printf ' - run: go run ./cmd/tool -mail me@example.com\n' >>"$dir/go-run-local/.github/workflows/ci.yml" expect go-run-local 0 +# 4. Python tools: a version of their own in a workflow, and a requirements +# file that no Dependabot pip entry watches (celeris#890 item 93). +fixture pipx-spec +printf ' - run: pipx run --spec zizmor==1.30.0 zizmor .github/workflows\n' >>"$dir/pipx-spec/.github/workflows/ci.yml" +expect pipx-spec 1 'ci.yml' +fixture pip-install-pin +printf ' - run: python3 -m pip install zizmor==1.30.0\n' >>"$dir/pip-install-pin/.github/workflows/ci.yml" +expect pip-install-pin 1 'ci.yml' +fixture pip-install-extras +printf ' - run: pip3 install --quiet "zizmor[x]==1.30.0"\n' >>"$dir/pip-install-extras/.github/workflows/ci.yml" +expect pip-install-extras 1 'ci.yml' +fixture uvx-pin +printf ' - run: uvx --from zizmor==1.30.0 zizmor .\n' >>"$dir/uvx-pin/.github/workflows/ci.yml" +expect uvx-pin 1 'ci.yml' +fixture pip-install-continuation +printf ' - run: |\n pipx run \\\n --spec zizmor==1.30.0 zizmor .\n' >>"$dir/pip-install-continuation/.github/workflows/ci.yml" +expect pip-install-continuation 1 'ci.yml' +fixture pip-install-composite +printf ' - run: pipx install zizmor==1.30.0\n shell: bash\n' >>"$dir/pip-install-composite/.github/actions/probe/action.yml" +expect pip-install-composite 1 'actions/probe/action.yml' +# Pin forms other than ==: uv's name@version, and range pins (review r1 of #1124, m4). +fixture uvx-at-pin +printf ' - run: uvx zizmor@1.30.0 .github/workflows\n' >>"$dir/uvx-at-pin/.github/workflows/ci.yml" +expect uvx-at-pin 1 'ci.yml' +fixture uv-tool-run-at-pin +printf ' - run: uv tool run zizmor@v1.30.0 .\n' >>"$dir/uv-tool-run-at-pin/.github/workflows/ci.yml" +expect uv-tool-run-at-pin 1 'ci.yml' +fixture pip-range-pin +printf ' - run: pip install "zizmor>=1.30,<1.31"\n' >>"$dir/pip-range-pin/.github/workflows/ci.yml" +expect pip-range-pin 1 'ci.yml' +fixture pip-compatible-pin +printf ' - run: pipx run --spec zizmor~=1.30 zizmor .\n' >>"$dir/pip-compatible-pin/.github/workflows/ci.yml" +expect pip-compatible-pin 1 'ci.yml' +fixture pip-upper-pin +printf ' - run: python3 -m pip install "zizmor<2"\n' >>"$dir/pip-upper-pin/.github/workflows/ci.yml" +expect pip-upper-pin 1 'ci.yml' +# An @ that is no version (a path, a mail address) and a plain redirection are fine. +fixture uvx-at-path +printf ' - run: uvx zizmor .github/workflows 2>&1 | tee log@host.txt\n' >>"$dir/uvx-at-path/.github/workflows/ci.yml" +expect uvx-at-path 0 +fixture pip-redirect +printf ' - run: pip install -r .github/tools/requirements.txt >/dev/null\n' >>"$dir/pip-redirect/.github/workflows/ci.yml" +expect pip-redirect 0 + +# Reading the pins from a requirements file, and a comment, are fine. +fixture pip-install-requirements +printf ' - run: pip install -r .github/tools/requirements.txt\n' >>"$dir/pip-install-requirements/.github/workflows/ci.yml" +expect pip-install-requirements 0 +fixture pip-install-comment +printf ' # was: pipx run --spec zizmor==1.30.0 zizmor .\n' >>"$dir/pip-install-comment/.github/workflows/ci.yml" +expect pip-install-comment 0 +# An == that is no pin of a Python tool is fine. +fixture eq-in-condition +printf ' - run: if [ "$x" == "pip install" ]; then echo ok; fi\n' >>"$dir/eq-in-condition/.github/workflows/ci.yml" +expect eq-in-condition 0 + +# A requirements file in a directory no pip entry names, and the entry added +# in either spelling. +fixture requirements-unwatched +mkdir -p "$dir/requirements-unwatched/.github/ci-extra" +printf 'zizmor==1.30.0\n' >"$dir/requirements-unwatched/.github/ci-extra/requirements.txt" +expect requirements-unwatched 1 "no 'package-ecosystem: pip' entry" +fixture requirements-watched +mkdir -p "$dir/requirements-watched/.github/ci-extra" +printf 'zizmor==1.30.0\n' >"$dir/requirements-watched/.github/ci-extra/requirements.txt" +printf ' - package-ecosystem: pip\n directory: /.github/ci-extra\n schedule:\n interval: daily\n' >>"$dir/requirements-watched/.github/dependabot.yml" +expect requirements-watched 0 +fixture requirements-watched-list +mkdir -p "$dir/requirements-watched-list/.github/ci-extra" +printf 'zizmor==1.30.0\n' >"$dir/requirements-watched-list/.github/ci-extra/requirements.txt" +printf ' - package-ecosystem: pip\n directories:\n - /somewhere\n - /.github/ci-extra\n schedule:\n interval: daily\n' >>"$dir/requirements-watched-list/.github/dependabot.yml" +expect requirements-watched-list 0 +# An entry of another ecosystem for the same directory does not watch pip. +fixture requirements-wrong-ecosystem +mkdir -p "$dir/requirements-wrong-ecosystem/.github/ci-extra" +printf 'zizmor==1.30.0\n' >"$dir/requirements-wrong-ecosystem/.github/ci-extra/requirements.txt" +printf ' - package-ecosystem: gomod\n directory: /.github/ci-extra\n schedule:\n interval: daily\n' >>"$dir/requirements-wrong-ecosystem/.github/dependabot.yml" +expect requirements-wrong-ecosystem 1 "no 'package-ecosystem: pip' entry" + echo "$n cases" exit "$fail" diff --git a/.github/scripts/tool-pins-check.sh b/.github/scripts/tool-pins-check.sh index b3218c77c..ec756ea17 100755 --- a/.github/scripts/tool-pins-check.sh +++ b/.github/scripts/tool-pins-check.sh @@ -22,6 +22,15 @@ # with # is a comment and is skipped; a # later in a line is not (it can # sit in an echo string), so a trailing comment that quotes such a command # is flagged too. A YAML folded scalar (run: >) is not joined. +# 4. The Python tools of CI (zizmor) are pinned in a requirements file under +# .github/, which Dependabot's pip ecosystem reads. (a) No workflow or +# composite action installs or runs a Python tool with a `name==version` +# (or `>=`, `~=`, `<`, `uvx name@1`) of its own (`pipx run --spec zizmor==1.30.0`, +# `pip install x==1`, `uvx x==1`): that pin is read by no Dependabot entry, and stayed on its +# version until somebody remembered (celeris#890 item 93). The same line +# rules as check 3 apply. (b) Every directory under .github/ that holds a +# requirements*.txt has a `package-ecosystem: pip` entry in dependabot.yml +# for it (`directory:` or one of `directories:`), or nothing watches it. # # Prints one line per failed check and exits 1 if any failed. The Lint job # runs it, and tool-pins-check-test.sh runs it on broken fixtures. @@ -111,5 +120,90 @@ if [ -n "$hits" ]; then echo " ${hits//$'\n'/$'\n' }" >&2 fi +# 4. Python tools: pinned in a requirements file that Dependabot watches. +pip_hits() { + python3 - "$1" <<'PY' +import glob, os, re, sys +root = sys.argv[1] +pats = ['.github/workflows/*.yml', '.github/workflows/*.yaml', + '.github/actions/**/*.yml', '.github/actions/**/*.yaml'] +# A Python tool's installer or runner, then (flags) name==version or another +# specifier with a version (>=1.30,<1.31), or uv's name@version. pip's -r FILE +# form has none of them, so reading the pins from the file is not flagged. +installer = r'(?:\bpipx\s+(?:run|install)\b|\bpip3?\s+install\b|\bpython3?\s+-m\s+pip\s+install\b|\buvx\b|\buv\s+tool\s+(?:run|install)\b)' +# name==version, and the other specifiers of PEP 440 (~=, >=, <=, !=, <, >) with a version. +spec = r'[A-Za-z0-9_.-]+(?:\[[^]]*\])?(?:==|~=|>=|<=|!=|<|>)\s*[0-9]' +# uv's own pin, name@version, after uvx or uv tool (a path or a mail address has no digit or v after the @). +at = r'\b(?:uvx|uv\s+tool\s+(?:run|install))\b.*?\s[A-Za-z0-9_.-]+@v?[0-9]' +pat = re.compile(installer + r'.*?' + spec + '|' + at) +files = sorted({f for p in pats for f in glob.glob(os.path.join(root, p), recursive=True)}) +for f in files: + lines = open(f).read().splitlines() + i = 0 + while i < len(lines): + start, text = i, lines[i] + while text.rstrip().endswith('\\') and i + 1 < len(lines): + i += 1 + text = text.rstrip()[:-1] + ' ' + lines[i].strip() + i += 1 + if text.lstrip().startswith('#'): + continue + if pat.search(text): + print("%s:%d:%s" % (os.path.relpath(f, root), start + 1, lines[start].strip())) +PY +} +hits=$(pip_hits "$root") +if [ -n "$hits" ]; then + bad "a workflow or action installs or runs a Python tool with its own name==version; pin it in a requirements file under .github/ (Dependabot's pip entry watches it) and install it with pip install -r:" + echo " ${hits//$'\n'/$'\n' }" >&2 +fi +# The pip entries of dependabot.yml, as "/dir" lines: an entry runs from its +# "- package-ecosystem:" line to the next one; its `directory:` and the items +# of its `directories:` list count. +pip_dirs() { + python3 - "$1" <<'PY' +import re, sys +dirs = [] +eco, found, in_list = None, [], False +def end_entry(): + if eco == "pip": + dirs.extend(found) +for line in open(sys.argv[1]).read().splitlines(): + m = re.match(r'^\s*-\s*package-ecosystem:\s*["\']?([^\s"\'#]+)', line) + if m: + end_entry() + eco, found, in_list = m.group(1), [], False + continue + m = re.match(r'^\s*directory:\s*["\']?([^\s"\'#]+)', line) + if m: + found.append(m.group(1)); in_list = False + continue + if re.match(r'^\s*directories:\s*(#.*)?$', line): + in_list = True + continue + m = re.match(r'^\s*-\s*["\']?(/[^\s"\'#]*)', line) + if m and in_list: + found.append(m.group(1)) + continue + if in_list and not re.match(r'^\s*(#.*)?$', line): + in_list = False +end_entry() +print("\n".join(sorted(set(dirs)))) +PY +} +watched=$(pip_dirs "$root/.github/dependabot.yml") +unwatched= +while IFS= read -r f; do + d=/$(dirname "${f#"$root"/}") + case " +$watched +" in *" +$d +"*) ;; *) unwatched="$unwatched $d" ;; esac +done < <(find "$root/.github" -name 'requirements*.txt' -type f | sort) +if [ -n "$unwatched" ]; then + bad ".github/dependabot.yml has no 'package-ecosystem: pip' entry for the directory of a requirements file:$unwatched" +fi + [ "$fail" = 0 ] && echo "tool-pins-check: ok (tools: $(echo "$tools" | paste -sd' ' -); golangci-lint $ci)" exit "$fail" diff --git a/.github/tools/requirements.txt b/.github/tools/requirements.txt new file mode 100644 index 000000000..ac452646c --- /dev/null +++ b/.github/tools/requirements.txt @@ -0,0 +1,5 @@ +# The Python tools CI runs, pinned here so that Dependabot's pip entry for +# /.github/tools (.github/dependabot.yml) proposes their bumps (celeris#890 +# item 93). The Lint job installs this file with `pip install -r`; no workflow +# names a version of its own (.github/scripts/tool-pins-check.sh, check 4). +zizmor==1.30.0 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0b13f30a5..3b595e8e0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -84,11 +84,19 @@ jobs: # and with a result line a test's own output joined (celeris#890). - name: named-tally.sh self-test run: .github/scripts/named-tally-test.sh - # The tools CI runs are pinned in one place, .github/tools/go.mod, and - # three other places name them: the allow list of its Dependabot entry + # The tally of the job `iouring-base-tier` (base-tier-tally.sh) on fixture + # logs: an allowed skip, a skip nobody allowed, an arm that ran at tier=high, + # a probe line of an uncapped run, and a named test missing or skipped + # (celeris#890 item 87). + - name: base-tier-tally.sh self-test + run: .github/scripts/base-tier-tally-test.sh + # The tools CI runs are pinned in one place, .github/tools/go.mod (the Go + # ones) or .github/tools/requirements.txt (zizmor), and three other places + # name them: the allow list of its Dependabot entry # (a tool without an entry, or one that names a package path instead # of the module path, silently stops getting bump PRs), CONTRIBUTING's - # golangci-lint version, and the workflows (no `go install ...@v`). + # golangci-lint version, and the workflows (no `go install ...@v`, no + # `name==version` of a Python tool). # tool-pins-check.sh compares them; its self-test runs it on broken # fixtures, so a check that stops failing is a red Lint (celeris#890). - name: Tool pins agree @@ -138,10 +146,15 @@ jobs: # template injection, excessive permissions). Complements # actionlint, which checks syntax rather than security posture. # Needs only the ephemeral job token (read) for the online audits; - # no repository secrets. Pinned to an exact version. + # no repository secrets. Pinned to an exact version in + # .github/tools/requirements.txt, which Dependabot's pip entry bumps + # (celeris#890 item 93); the step names no version of its own. env: GH_TOKEN: ${{ github.token }} - run: pipx run --spec zizmor==1.30.0 zizmor --color always .github/workflows + run: | + python3 -m venv "$RUNNER_TEMP/zizmor-venv" + "$RUNNER_TEMP/zizmor-venv/bin/pip" install --quiet --disable-pip-version-check -r .github/tools/requirements.txt + "$RUNNER_TEMP/zizmor-venv/bin/zizmor" --color always .github/workflows # The required check "Unit (root + middleware sub-modules)" is the `unit-gate` # job below: it passes only when this job AND every root-package shard passed. @@ -1253,16 +1266,20 @@ jobs: # with -v, and tallies by name per package: every listed test must run and # PASS, and no SKIP line is allowed. # - # Which of them can fail a pull request: celeristest, middleware/timeout and - # middleware/ratelimit fail on any count over their budget. The rest (the - # drivers, middleware/{cache,idempotency,overload}) only log an over-budget - # count unless TESTING_STRICT_ALLOC_BUDGETS=1, which this step does not set, - # so for them it proves the guard ran (PASS, no skip) and not that the count - # held. Setting it is the stricter choice; see the pull request for the - # numbers behind it. The sse and websocket tests fail on their own bounds. + # Every one of them fails a pull request on a count over its budget. + # celeristest, middleware/timeout and middleware/ratelimit always did; the + # rest (the drivers, middleware/{cache,idempotency,overload}) only logged an + # over-budget count unless TESTING_STRICT_ALLOC_BUDGETS=1, which this step + # now sets (celeris#890 item 91), so for them a PASS says the count held and + # not only that the guard ran. The one guard that was over budget when the + # variable was first set, driver/postgres Query_1col_1row (14 allocs/op + # against 4), has the budget the driver costs. The sse and websocket tests + # fail on their own bounds. - name: celeris#890 allocation guards and alloc-count tests, without -race (skipping forbidden) if: ${{ !cancelled() }} shell: bash + env: + TESTING_STRICT_ALLOC_BUDGETS: "1" run: | set -o pipefail # " " @@ -1793,6 +1810,22 @@ jobs: echo "engines to report a refused steal -- was one renamed or removed, or did an io_uring arm skip?" exit 1 fi + # celeris#890 item 96: ./test/integration is not run by this workflow (its + # tests need a host; `mage` runs them), but its adaptive_*_test.go files keep + # a copy of ./internal/adaptive's CELERIS_REQUIRE_UPSWITCH gate + # (skipOrFailIfStandbyAborted886). This guard, the counterpart of + # TestAdaptiveSkipRouting684, fails on a raw t.Skip in those files that + # bypasses it and is not registered with a reason, and its self-test + # plants one. It is the only test of the package that runs here, by name. + - name: celeris#890 test/integration adaptive skips go through the gate + if: ${{ !cancelled() }} + shell: bash + run: | + set -o pipefail + go test -count=1 -timeout=300s -v -run '^(TestAdaptiveSkipRoutingIntegration890|TestAdaptiveSkipRoutingGuardSelfTest890)$' ./test/integration/ 2>&1 | tee /tmp/integration-skip-guard.log + "$GITHUB_WORKSPACE"/.github/scripts/named-tally.sh 'test/integration skip guard (celeris#890 item 96)' /tmp/integration-skip-guard.log \ + TestAdaptiveSkipRoutingIntegration890 \ + TestAdaptiveSkipRoutingGuardSelfTest890 # NOT a required check: the repository's ruleset ("Main — CI") requires Lint, # "Unit (root + middleware sub-modules)" (the `unit-gate` job), Conformance, @@ -2321,6 +2354,162 @@ jobs: if [ "$skipped" -ne 0 ]; then echo "a SKIP is not a result ($skipped SKIP lines)"; bad=1; fi exit "$bad" + # NOT a required check for merging (see the note above `iouring`), but a gate of + # releases: release.yml calls this workflow and tag-submodules needs its result to + # be success, so a red run of this job blocks the tag. celeris#890 item 87: + # nothing ran the io_uring base tier. CELERIS_MAX_IOURING_TIER=base caps the + # probe's tier at "base" (no provided buffer ring, single-shot accept, no + # multishot recv), the tier a kernel without those features gets and the one + # PR #1026 made non-blocking on accept (item 53, celeris#884): a partial + # inline write of a detached stream leaves a remainder in writeBuf there + # (TestShutdownDetachedPartialWriteIsFlushed806), a state the other tiers do + # not reach the same way. Until this job only local runs covered it. + # + # Two steps, each with memlock raised (the tests that ask for several workers + # need it) and the cap in the environment: + # * ./internal/engine/iouring under -race. The cap must have taken effect: + # TestReportTier844 must print io_uring_tier=base for the capped probe + # and no engine arm may report tier=high (a misspelled value counts as + # "none" and would skip everything, which the tally also refuses); + # * the root package and ./test/server, the tests whose names say Shutdown, + # Detach, SSE, Stream, WebSocket, Hijack or Drain, under -race. + # The -timeout of each step is about twice its measured time (473 s and 239 s), so a + # run with the cap lost (a misspelled value made 47 tests fail slowly and kept the + # second step busy for 23 minutes before it was cancelled, run 40416230192) ends + # in minutes. + # CELERIS_REQUIRE_ZONED_PEER=1, as in the unit job: the zoned link-local tests (celeris#885, + # #1127) fail rather than skip on a runner without an fe80:: address, so they need no + # allow-list entry. + # Tests that need the high tier skip at this tier on purpose; each skip that + # is allowed is listed by name with its reason in .github/base-tier-skips-iouring.txt + # and -root.txt, and any other SKIP line fails the step (base-tier-tally.sh, + # whose self-test is a Lint step). First run on a hosted runner (kernel + # 6.17.0-1022-azure, run 40414278698): the package took 473 s under -race (504 s + # for the step), 554 top-level tests passed and 12 SKIP lines are the allowed + # ones; the root package and ./test/server took 4.6 s and 239 s (258 s for the + # step), 107 top-level tests passed and 19 SKIP lines are allowed. Those 19 are + # the io_uring and adaptive arms of tests whose probe (c714ProbeIOUring, + # keptProbeIOUring) calls io_uring usable only at the high tier: with the cap + # they skip with "no usable io_uring", so what this step adds for the root tests + # is the tests that start an io_uring engine without such a probe: they run at + # the base tier here and at the high tier in every other job. The root list names + # those arms by pattern (a new probe-gated test needs no edit; the arm must have + # logged the probe's own "io_uring tier=base kernel=..." skip message) after run + # 40419578127, on a merge with main, + # met nine new such tests (117 passed, 46 SKIP lines) and failed a list of names. + iouring-base-tier: + name: io_uring base tier (CELERIS_MAX_IOURING_TIER=base, celeris#890) + runs-on: ubuntu-latest + timeout-minutes: 30 + env: + CELERIS_MAX_IOURING_TIER: base + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version: "1.27.3" + - name: internal/engine/iouring at the base tier (race, tier asserted, skips accounted for) + shell: bash + run: | + set -o pipefail + sudo prlimit --pid "$$" --memlock=unlimited:unlimited + echo "memlock (KiB): $(ulimit -l), CELERIS_MAX_IOURING_TIER=$CELERIS_MAX_IOURING_TIER" + start=$SECONDS + CELERIS_REQUIRE_DUALSTACK=1 CELERIS_REQUIRE_ZONED_PEER=1 go test -race -count=1 -timeout=900s -v ./internal/engine/iouring/ 2>&1 | tee /tmp/base-iouring.log + echo "internal/engine/iouring at the base tier took $((SECONDS - start)) s" + .github/scripts/base-tier-tally.sh iouring /tmp/base-iouring.log TestShutdownDetachedPartialWriteIsFlushed806 TestReportTier844 + - name: Root package and test/server, Shutdown/Detach/SSE/Stream/WebSocket/Hijack/Drain tests at the base tier (race) + if: ${{ !cancelled() }} + shell: bash + run: | + set -o pipefail + sudo prlimit --pid "$$" --memlock=unlimited:unlimited + echo "memlock (KiB): $(ulimit -l), CELERIS_MAX_IOURING_TIER=$CELERIS_MAX_IOURING_TIER" + start=$SECONDS + # -p 1: one test process at a time, so the runner's one memlock budget never funds two. + CELERIS_REQUIRE_DUALSTACK=1 CELERIS_REQUIRE_ZONED_PEER=1 go test -p 1 -race -count=1 -timeout=600s -v -run 'Shutdown|Detach|SSE|Stream|WebSocket|Hijack|Drain' . ./test/server 2>&1 | tee /tmp/base-root.log + echo "root and test/server at the base tier took $((SECONDS - start)) s" + .github/scripts/base-tier-tally.sh root /tmp/base-root.log + + # NOT a required check for merging, but a gate of releases (release.yml calls this + # workflow and tag-submodules needs its result to be success). celeris#890 item 92: the large-response tests of + # ./test/server (celeris#761, #817, #836) have a lean shape and a full-size + # one. lean761() (large_response_linux_test.go) is true under -race and + # under coverage, which are the only ways the other jobs run ./test/server, + # so their 16 MiB / 64 MiB bodies, their request counts and the h2c + # AsyncHandlers arm of TestH2CUnmatchedThroughTimeoutSingleflight836 + # (AsyncHandlers: !raceOn761) never ran in CI. This job builds the test binary + # without -race and without coverage (race761_off_test.go makes raceOn761 + # false) and runs the eight tests that read lean761(), n836() or raceOn761, + # by name, with memlock raised (they start io_uring engines), -v, and a tally + # by name: each must run once and PASS, with no SKIP line. The script prints + # the wall time and the peak resident set of the test process, so a log says + # what the full-size shape costs on a runner. + large-response: + name: Large responses at full size, no race (./test/server, celeris#890) + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version: "1.27.3" + - name: test/server large-response tests at full size (no race, no coverage) + shell: bash + run: | + set -o pipefail + sudo prlimit --pid "$$" --memlock=unlimited:unlimited + echo "memlock (KiB): $(ulimit -l), CPUs: $(nproc), memory: $(free -m | awk '/^Mem:/ {print $2}') MiB" + tests=( + TestLargeResponseIsDelivered + TestLargeFileResponseIsDelivered + TestLargeResponseIsDeliveredH2 + TestConcurrentResponsesOwnTheirBodies + TestH2DerivedContextDoesNotPanic836 + TestH2DetachedContextIsItsRequests836 + TestH2InlineCleanupDoesNotReleaseAReusedStream947 + TestH2CUnmatchedThroughTimeoutSingleflight836 + ) + regex=$(IFS='|'; echo "${tests[*]}") + # Discovery: a Test function of ./test/server that reads lean761(), raceOn761, n836() or + # dur836() runs a lean shape everywhere else, so it must be in the list above or its + # full-size shape runs nowhere (the failure this job exists for). A helper that is not a + # Test function and reads them is not seen: its callers are not found this way. + used=$(awk 'FNR == 1 { fn = "" } /^func / { fn = $2; sub(/\(.*/, "", fn) } /lean761\(\)|raceOn761|n836\(|dur836\(/ && fn ~ /^Test/ { print fn }' test/server/*_test.go | sort -u) + listed=$(printf '%s\n' "${tests[@]}" | sort -u) + if [ "$used" != "$listed" ]; then + echo "tests of ./test/server that read the lean-shape helpers (left) against the list of this step (right):" + diff <(echo "$used") <(echo "$listed") || true + echo "add the test to the list (or take it out when it no longer reads a helper)" + exit 1 + fi + go test -c -o /tmp/server.test ./test/server + python3 - /tmp/server.test "^(${regex})\$" <<'PY' 2>&1 | tee /tmp/large.log + import resource, subprocess, sys, time + binary, run = sys.argv[1], sys.argv[2] + t0 = time.monotonic() + rc = subprocess.run([binary, "-test.run", run, "-test.v", "-test.count=1", "-test.timeout=1500s"]).returncode + rss = resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss # KiB on Linux + print("full-size shape: wall %.1f s, peak RSS of the test process %.0f MiB, exit %d" % (time.monotonic() - t0, rss / 1024, rc)) + sys.exit(rc) + PY + .github/scripts/named-tally.sh 'test/server full-size shape (celeris#890 item 92)' /tmp/large.log "${tests[@]}" + # The shape itself: the 64 MiB HTTP/1 bodies (16 MiB under lean761) and the 16 MiB + # HTTP/2 ones (8 MiB under lean761) ran, on all four engines. + full1=$(grep -cE '^=== RUN TestLargeResponseIsDelivered/[a-z_]+/[a-z-]+/67108864/' /tmp/large.log || true) + full2=$(grep -cE '^=== RUN TestLargeResponseIsDeliveredH2/[a-z_]+/[a-z-]+/16777216$' /tmp/large.log || true) + lean1=$(grep -cE '^=== RUN TestLargeResponseIsDelivered/.*/16777216/' /tmp/large.log || true) + lean2=$(grep -cE '^=== RUN TestLargeResponseIsDeliveredH2/.*/8388608$' /tmp/large.log || true) + echo "full-size bodies: HTTP/1 64 MiB runs $full1 (want 12), HTTP/2 16 MiB runs $full2 (want 8), lean-size runs $lean1 and $lean2 (want 0)" + if [ "$full1" -ne 12 ] || [ "$full2" -ne 8 ] || [ "$lean1" -ne 0 ] || [ "$lean2" -ne 0 ]; then + echo "expected the full-size bodies, not the lean ones -- did lean761() become true here (race or coverage), or did a size change?" + exit 1 + fi + conformance: name: Conformance runs-on: ubuntu-latest diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 47b34928d..beccb529e 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -149,8 +149,52 @@ # 189 minutes rounded up per job (celeris 21 jobs and 101 minutes, loadgen 22 # and 88), two thirds of it pull request runs. At that rate the 600 minutes # last 3.6 to 4.1 days (rounded or raw job time; CodSpeed's own meter is not -# public), and pushes to main alone would use them in about two weeks. Whether to gate pull request runs on the `performance` label, -# or more, is an open decision (celeris#754). +# public), and pushes to main alone would use them in about two weeks. +# +# What happened (celeris#890 items 7 and 97). The 163 jobs of this workflow +# that got a macro runner between 2026-09-26 and 2026-10-12 add up to 880 +# minutes rounded up per job (786 raw); 644 of them fall in October, and the +# count passes 600 at 13:45Z on 2026-10-12. The last job ran at 20:37Z that +# day (run 38305334372). None of the 76 jobs created after it got a runner: +# 35 were cancelled within 3 hours (replaced by the next run of their group), +# 41 after waiting the 24 hours GitHub allows a queued job, and 7 runs are +# still queued as this is written. That fits an exhausted monthly quota (the +# meter here is a count of job minutes, CodSpeed's own is not public, and the +# organization's settings page in CodSpeed holds the real figure: a maintainer +# reads it there). It is also why the main baseline went stale: a push to main +# queued behind the one before it, and a group keeps one pending run. +# +# So, since celeris#890 item 7: +# - a pull request runs the benchmarks only while it carries the +# `performance` label (CONTRIBUTING.md lists it among the release-note +# labels, and a benchmark-path change that is labelled for the notes is +# the one to measure). Adding the label starts a run, every push to a +# labelled pull request starts another, and a draft waits until it is +# marked ready, as before. A pull request without the label costs no +# macro-runner minute: its run exists and its job is skipped, which is the +# result a reviewer sees; +# - main is measured once a day (the schedule below) and no longer on every +# push, which makes the baseline a day old at most. A job took 4.8 minutes +# on average (786 raw minutes over 163 jobs), so that is about 144 minutes +# a month, against about 440 for the 92 pushes to main of the 30 days to +# 2026-10-26 (all of them started a run); +# - a scheduled run replaces an older one that is still waiting for a +# runner, so a stall cannot pile up a queue behind it. +# The label gate in loadgen's workflow is the other half of the decision and +# is not in this repository. +# +# Unproven, and to be checked after the merge: CodSpeed's GitHub Actions page +# (https://codspeed.io/docs/integrations/ci/github-actions) says that pushes to +# the default branch record the baseline and names workflow_dispatch for +# backtests; it does not mention `schedule`. Look at the first scheduled run in +# the CodSpeed dashboard (not before the quota resets on 2026-10-31; until then +# no run gets a runner); if it is not recorded as the main baseline, either +# bring `push: branches: [main]` (with the paths below) back beside the schedule, +# or keep the cron but have it start `gh workflow run codspeed.yml --ref main` +# (a `workflow_dispatch` run, which CodSpeed documents), for example from a small +# hosted workflow with `actions: write`. Either way a pull request then compares +# with a main run of the day, not with an older report. +# # The trigger paths below are the non-test code of every package the # benchmark set executes (measured: the set run once with -coverpkg over the # whole module, on Linux and macOS), the benchmark files, the root @@ -165,10 +209,18 @@ name: CodSpeed on: - push: + # Main, once a day (celeris#890 item 7). A schedule has no path filter, so + # it runs whether or not the benchmark paths changed; it replaces the push + # trigger, which started a run for each push that touched them. + schedule: + - cron: "17 3 * * *" + pull_request: branches: [main] - # One list for both events: the pull_request trigger below aliases it. - paths: &benchmark-paths + # `labeled` starts the run when `performance` is added; the job's `if:` + # below skips every other label. `unlabeled` is not listed: taking the + # label off does not stop a run in flight and starts none. + types: [opened, synchronize, reopened, ready_for_review, labeled] + paths: - "*.go" - "celeristest/**" - "internal/ctxkit/**" @@ -207,10 +259,6 @@ on: - "go.mod" - "go.sum" - ".github/workflows/codspeed.yml" - pull_request: - branches: [main] - types: [opened, synchronize, reopened, ready_for_review] - paths: *benchmark-paths # Backtests: CodSpeed dispatches this to measure older commits, and it is # the manual way to take a fresh baseline or to run a draft pull request's # branch (gh workflow run codspeed.yml --ref ). @@ -220,25 +268,36 @@ permissions: contents: read # One run per pull request: a new push cancels the one in flight, so a -# burst of pushes costs one run. Pushes to main are never cancelled -- each -# is a baseline -- but a burst still collapses to the newest pending one. -# Every dispatch gets a group of its own, so the backtests CodSpeed -# dispatches together never cancel one another. +# burst of pushes costs one run. A scheduled run cancels an older scheduled +# run that is still waiting for a runner (a stalled queue must not hold the +# baseline back: celeris#890 item 97). Every dispatch gets a group of its +# own, so the backtests CodSpeed dispatches together never cancel one +# another. So does the run that adding any label other than `performance` +# starts: its job is skipped, and without a group of its own it would cancel +# the benchmark run that the label `performance` started a minute before. concurrency: - group: codspeed-${{ github.event.pull_request.number || (github.event_name == 'workflow_dispatch' && github.run_id) || github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + group: codspeed-${{ (github.event.action == 'labeled' && github.event.label.name != 'performance' && github.run_id) || github.event.pull_request.number || (github.event_name == 'workflow_dispatch' && github.run_id) || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || github.event_name == 'schedule' }} jobs: benchmarks: name: Benchmarks (arm64 macro runner) - # Skips fork pull requests (as this file is written; the approval policy - # in the header is what holds against a fork that edits it) and drafts, - # which wait until they are marked ready (ready_for_review above), to - # save minutes. + # A pull request runs only with the `performance` label, and the label's + # own event (celeris#890 item 7): the run that adding another label starts + # skips the job, and so does a label added to a closed or merged pull request + # (a release-notes clean-up after the merge must not spend the quota). Fork pull + # requests are skipped as this file is written + # (the approval policy in the header is what holds against a fork that + # edits it), and so are drafts, which wait until they are marked ready + # (ready_for_review above), to save minutes. Every other event (the + # schedule, a dispatch) runs. if: >- github.event_name != 'pull_request' || (github.event.pull_request.head.repo.full_name == github.repository && - !github.event.pull_request.draft) + github.event.pull_request.state == 'open' && + !github.event.pull_request.draft && + contains(github.event.pull_request.labels.*.name, 'performance') && + (github.event.action != 'labeled' || github.event.label.name == 'performance')) runs-on: codspeed-macro-arm64-graviton-ubuntu-22-04 # A hang must not burn the month's minutes. A full run is a few minutes. timeout-minutes: 20 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 940810040..7f6ada74a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -171,6 +171,11 @@ category it belongs to so it lands in the right section: Anything else lands under "Other". If a change is breaking, label it `breaking` *and* say so in the PR description. +The `performance` label has a second job: a pull request that touches the +benchmarked paths starts the CodSpeed run (`.github/workflows/codspeed.yml`) +only while it carries that label, because the run spends macro-runner minutes +from a monthly budget shared with loadgen. Main is measured once a day. + ### Commit sign-off We recommend signing off your commits with `git commit -s`, which adds a diff --git a/driver/postgres/alloc_guard_test.go b/driver/postgres/alloc_guard_test.go index d991f3efb..083770369 100644 --- a/driver/postgres/alloc_guard_test.go +++ b/driver/postgres/alloc_guard_test.go @@ -43,7 +43,12 @@ func TestAllocBudgets(t *testing.T) { } guards := []guard{ - {name: "Query_1col_1row", budget: 4, bench: benchQuery1col1row}, + // 14, what the driver costs: celeris#890 item 91 measured 14 allocs/op on a + // linux/arm64 container (20 of 20 runs) and on the GitHub amd64 runner (run + // 39074958353) against the aspirational 4 this row held until then, which + // never failed because no CI step set TESTING_STRICT_ALLOC_BUDGETS=1. The + // allocation-guard step of ci.yml sets it now, so 14 is the bound. + {name: "Query_1col_1row", budget: 14, bench: benchQuery1col1row}, {name: "Pool_AcquireRelease", budget: 0, bench: benchPoolAcquireRelease}, // Prepared exec is intentionally omitted: it requires the extended // protocol (Parse/Bind/Execute/Sync) handshake which the existing diff --git a/internal/engine/epoll/conns_slot_bench_linux_test.go b/internal/engine/epoll/conns_slot_bench_linux_test.go index 1163fe6b9..d73b9ac6f 100644 --- a/internal/engine/epoll/conns_slot_bench_linux_test.go +++ b/internal/engine/epoll/conns_slot_bench_linux_test.go @@ -38,6 +38,13 @@ func BenchmarkCloseConnSlotClear(b *testing.B) { // has counted every close. The kernel's connect and close dominate each op, so // the figure is read against an A/A floor; BenchmarkCloseConnSlotClear // isolates the lock itself. +// +// Warning: this instrument has extreme outlier samples. One A/A pair (two runs +// of the same code) differed by up to about 9x in the review of the change that +// added it (celeris#960, tracked in celeris#885), a spread the median floor of +// 0.032 does not show. Do not read one pair, or one run: compare medians over +// many interleaved rounds (.github/scripts/bench-ab.sh) and report the spread +// beside the median. The 9x figure is that review's, not measured here. func BenchmarkCloseChurn(b *testing.B) { eng, stop := newTestEngine(b) defer stop() diff --git a/middleware/adapters/retained_strings_linux_test.go b/middleware/adapters/retained_strings_linux_test.go index 0cde78f5f..a375fc77b 100644 --- a/middleware/adapters/retained_strings_linux_test.go +++ b/middleware/adapters/retained_strings_linux_test.go @@ -132,7 +132,7 @@ func keptArms(t *testing.T) []keptArm { {"epoll", celeris.Epoll, false}, {"epoll-async", celeris.Epoll, true}, } - if ok, p := keptProbeIOUring(); ok { + if ok, p := keptProbeIOUring(t); ok { arms = append(arms, keptArm{"io_uring", celeris.IOUring, false}, keptArm{"io_uring-async", celeris.IOUring, true}) } else if os.Getenv("CELERIS_REQUIRE_IOURING_WORKERS") == "1" { t.Fatalf("io_uring tier=%s kernel=%s, and CELERIS_REQUIRE_IOURING_WORKERS=1 forbids dropping the io_uring arms", p.IOUringTier, p.KernelVersion) @@ -144,17 +144,21 @@ func keptArms(t *testing.T) []keptArm { // keptProbeIOUring probes the kernel's io_uring support. With // CELERIS_REQUIRE_IOURING_WORKERS=1 a probe that finds no usable ring is -// retried for up to 10 s: the probe's ring can fail with ENOMEM against +// retried for up to 10 s, logging each round through tb: the probe's ring can fail with ENOMEM against // RLIMIT_MEMLOCK while the rings of engines stopped moments ago, or of // another test binary of the same user, are still charged // (internal/engine/iouring/ring_budget_linux_test.go). -func keptProbeIOUring() (usable bool, p celerisengine.CapabilityProfile) { +func keptProbeIOUring(tb testing.TB) (usable bool, p celerisengine.CapabilityProfile) { p = probe.Probe() usable = p.IOUringTier >= celerisengine.High && p.ProvidedBuffers if os.Getenv("CELERIS_REQUIRE_IOURING_WORKERS") != "1" { return usable, p } - for deadline := time.Now().Add(10 * time.Second); !usable && time.Now().Before(deadline); { + deadline := time.Now().Add(10 * time.Second) + for round := 1; !usable && time.Now().Before(deadline); round++ { + // One line per round, so a CI log shows that the probe fired and how long it + // kept failing. Rounds are 10 ms apart, so this is at most about 1000 lines. + tb.Logf("keptProbeIOUring: retry round %d, no usable io_uring ring yet (tier %v, provided buffers %t)", round, p.IOUringTier, p.ProvidedBuffers) time.Sleep(10 * time.Millisecond) p = probe.Probe() usable = p.IOUringTier >= celerisengine.High && p.ProvidedBuffers diff --git a/middleware/logger/retained_attrs_linux_test.go b/middleware/logger/retained_attrs_linux_test.go index c346d997c..cbf9d8d93 100644 --- a/middleware/logger/retained_attrs_linux_test.go +++ b/middleware/logger/retained_attrs_linux_test.go @@ -171,7 +171,7 @@ func keptArms(t *testing.T) []keptArm { {"epoll", celeris.Epoll, false}, {"epoll-async", celeris.Epoll, true}, } - if ok, p := keptProbeIOUring(); ok { + if ok, p := keptProbeIOUring(t); ok { arms = append(arms, keptArm{"io_uring", celeris.IOUring, false}, keptArm{"io_uring-async", celeris.IOUring, true}) } else if os.Getenv("CELERIS_REQUIRE_IOURING_WORKERS") == "1" { t.Fatalf("io_uring tier=%s kernel=%s, and CELERIS_REQUIRE_IOURING_WORKERS=1 forbids dropping the io_uring arms", p.IOUringTier, p.KernelVersion) @@ -183,17 +183,21 @@ func keptArms(t *testing.T) []keptArm { // keptProbeIOUring probes the kernel's io_uring support. With // CELERIS_REQUIRE_IOURING_WORKERS=1 a probe that finds no usable ring is -// retried for up to 10 s: the probe's ring can fail with ENOMEM against +// retried for up to 10 s, logging each round through tb: the probe's ring can fail with ENOMEM against // RLIMIT_MEMLOCK while the rings of engines stopped moments ago, or of // another test binary of the same user, are still charged // (internal/engine/iouring/ring_budget_linux_test.go). -func keptProbeIOUring() (usable bool, p celerisengine.CapabilityProfile) { +func keptProbeIOUring(tb testing.TB) (usable bool, p celerisengine.CapabilityProfile) { p = probe.Probe() usable = p.IOUringTier >= celerisengine.High && p.ProvidedBuffers if os.Getenv("CELERIS_REQUIRE_IOURING_WORKERS") != "1" { return usable, p } - for deadline := time.Now().Add(10 * time.Second); !usable && time.Now().Before(deadline); { + deadline := time.Now().Add(10 * time.Second) + for round := 1; !usable && time.Now().Before(deadline); round++ { + // One line per round, so a CI log shows that the probe fired and how long it + // kept failing. Rounds are 10 ms apart, so this is at most about 1000 lines. + tb.Logf("keptProbeIOUring: retry round %d, no usable io_uring ring yet (tier %v, provided buffers %t)", round, p.IOUringTier, p.ProvidedBuffers) time.Sleep(10 * time.Millisecond) p = probe.Probe() usable = p.IOUringTier >= celerisengine.High && p.ProvidedBuffers diff --git a/middleware/metrics/retained_labels_linux_test.go b/middleware/metrics/retained_labels_linux_test.go index 381ed37d4..3e4811f0b 100644 --- a/middleware/metrics/retained_labels_linux_test.go +++ b/middleware/metrics/retained_labels_linux_test.go @@ -182,7 +182,7 @@ func keptArms(t *testing.T) []keptArm { {"epoll", celeris.Epoll, false}, {"epoll-async", celeris.Epoll, true}, } - if ok, p := keptProbeIOUring(); ok { + if ok, p := keptProbeIOUring(t); ok { arms = append(arms, keptArm{"io_uring", celeris.IOUring, false}, keptArm{"io_uring-async", celeris.IOUring, true}) } else if os.Getenv("CELERIS_REQUIRE_IOURING_WORKERS") == "1" { t.Fatalf("io_uring tier=%s kernel=%s, and CELERIS_REQUIRE_IOURING_WORKERS=1 forbids dropping the io_uring arms", p.IOUringTier, p.KernelVersion) @@ -194,17 +194,21 @@ func keptArms(t *testing.T) []keptArm { // keptProbeIOUring probes the kernel's io_uring support. With // CELERIS_REQUIRE_IOURING_WORKERS=1 a probe that finds no usable ring is -// retried for up to 10 s: the probe's ring can fail with ENOMEM against +// retried for up to 10 s, logging each round through tb: the probe's ring can fail with ENOMEM against // RLIMIT_MEMLOCK while the rings of engines stopped moments ago, or of // another test binary of the same user, are still charged // (internal/engine/iouring/ring_budget_linux_test.go). -func keptProbeIOUring() (usable bool, p celerisengine.CapabilityProfile) { +func keptProbeIOUring(tb testing.TB) (usable bool, p celerisengine.CapabilityProfile) { p = probe.Probe() usable = p.IOUringTier >= celerisengine.High && p.ProvidedBuffers if os.Getenv("CELERIS_REQUIRE_IOURING_WORKERS") != "1" { return usable, p } - for deadline := time.Now().Add(10 * time.Second); !usable && time.Now().Before(deadline); { + deadline := time.Now().Add(10 * time.Second) + for round := 1; !usable && time.Now().Before(deadline); round++ { + // One line per round, so a CI log shows that the probe fired and how long it + // kept failing. Rounds are 10 ms apart, so this is at most about 1000 lines. + tb.Logf("keptProbeIOUring: retry round %d, no usable io_uring ring yet (tier %v, provided buffers %t)", round, p.IOUringTier, p.ProvidedBuffers) time.Sleep(10 * time.Millisecond) p = probe.Probe() usable = p.IOUringTier >= celerisengine.High && p.ProvidedBuffers diff --git a/middleware/otel/retained_attrs_linux_test.go b/middleware/otel/retained_attrs_linux_test.go index d8bab5409..a8875df5f 100644 --- a/middleware/otel/retained_attrs_linux_test.go +++ b/middleware/otel/retained_attrs_linux_test.go @@ -187,7 +187,7 @@ func keptArms(t *testing.T) []keptArm { {"epoll", celeris.Epoll, false}, {"epoll-async", celeris.Epoll, true}, } - if ok, p := keptProbeIOUring(); ok { + if ok, p := keptProbeIOUring(t); ok { arms = append(arms, keptArm{"io_uring", celeris.IOUring, false}, keptArm{"io_uring-async", celeris.IOUring, true}) } else if os.Getenv("CELERIS_REQUIRE_IOURING_WORKERS") == "1" { t.Fatalf("io_uring tier=%s kernel=%s, and CELERIS_REQUIRE_IOURING_WORKERS=1 forbids dropping the io_uring arms", p.IOUringTier, p.KernelVersion) @@ -199,17 +199,21 @@ func keptArms(t *testing.T) []keptArm { // keptProbeIOUring probes the kernel's io_uring support. With // CELERIS_REQUIRE_IOURING_WORKERS=1 a probe that finds no usable ring is -// retried for up to 10 s: the probe's ring can fail with ENOMEM against +// retried for up to 10 s, logging each round through tb: the probe's ring can fail with ENOMEM against // RLIMIT_MEMLOCK while the rings of engines stopped moments ago, or of // another test binary of the same user, are still charged // (internal/engine/iouring/ring_budget_linux_test.go). -func keptProbeIOUring() (usable bool, p celerisengine.CapabilityProfile) { +func keptProbeIOUring(tb testing.TB) (usable bool, p celerisengine.CapabilityProfile) { p = probe.Probe() usable = p.IOUringTier >= celerisengine.High && p.ProvidedBuffers if os.Getenv("CELERIS_REQUIRE_IOURING_WORKERS") != "1" { return usable, p } - for deadline := time.Now().Add(10 * time.Second); !usable && time.Now().Before(deadline); { + deadline := time.Now().Add(10 * time.Second) + for round := 1; !usable && time.Now().Before(deadline); round++ { + // One line per round, so a CI log shows that the probe fired and how long it + // kept failing. Rounds are 10 ms apart, so this is at most about 1000 lines. + tb.Logf("keptProbeIOUring: retry round %d, no usable io_uring ring yet (tier %v, provided buffers %t)", round, p.IOUringTier, p.ProvidedBuffers) time.Sleep(10 * time.Millisecond) p = probe.Probe() usable = p.IOUringTier >= celerisengine.High && p.ProvidedBuffers diff --git a/middleware/recovery/retained_attrs_linux_test.go b/middleware/recovery/retained_attrs_linux_test.go index 1a487da70..188e41023 100644 --- a/middleware/recovery/retained_attrs_linux_test.go +++ b/middleware/recovery/retained_attrs_linux_test.go @@ -207,7 +207,7 @@ func mwArms(t *testing.T) []keptArm { {"epoll", celeris.Epoll, false}, {"epoll-async", celeris.Epoll, true}, } - if ok, p := keptProbeIOUring(); ok { + if ok, p := keptProbeIOUring(t); ok { arms = append(arms, keptArm{"io_uring", celeris.IOUring, false}, keptArm{"io_uring-async", celeris.IOUring, true}) } else if os.Getenv("CELERIS_REQUIRE_IOURING_WORKERS") == "1" { t.Fatalf("io_uring tier=%s kernel=%s, and CELERIS_REQUIRE_IOURING_WORKERS=1 forbids dropping the io_uring arms", p.IOUringTier, p.KernelVersion) @@ -219,17 +219,21 @@ func mwArms(t *testing.T) []keptArm { // keptProbeIOUring probes the kernel's io_uring support. With // CELERIS_REQUIRE_IOURING_WORKERS=1 a probe that finds no usable ring is -// retried for up to 10 s: the probe's ring can fail with ENOMEM against +// retried for up to 10 s, logging each round through tb: the probe's ring can fail with ENOMEM against // RLIMIT_MEMLOCK while the rings of engines stopped moments ago, or of // another test binary of the same user, are still charged // (internal/engine/iouring/ring_budget_linux_test.go). -func keptProbeIOUring() (usable bool, p celerisengine.CapabilityProfile) { +func keptProbeIOUring(tb testing.TB) (usable bool, p celerisengine.CapabilityProfile) { p = probe.Probe() usable = p.IOUringTier >= celerisengine.High && p.ProvidedBuffers if os.Getenv("CELERIS_REQUIRE_IOURING_WORKERS") != "1" { return usable, p } - for deadline := time.Now().Add(10 * time.Second); !usable && time.Now().Before(deadline); { + deadline := time.Now().Add(10 * time.Second) + for round := 1; !usable && time.Now().Before(deadline); round++ { + // One line per round, so a CI log shows that the probe fired and how long it + // kept failing. Rounds are 10 ms apart, so this is at most about 1000 lines. + tb.Logf("keptProbeIOUring: retry round %d, no usable io_uring ring yet (tier %v, provided buffers %t)", round, p.IOUringTier, p.ProvidedBuffers) time.Sleep(10 * time.Millisecond) p = probe.Probe() usable = p.IOUringTier >= celerisengine.High && p.ProvidedBuffers diff --git a/test/integration/adaptive_standby_886_linux_test.go b/test/integration/adaptive_standby_886_linux_test.go index b8aaafd2c..105875fab 100644 --- a/test/integration/adaptive_standby_886_linux_test.go +++ b/test/integration/adaptive_standby_886_linux_test.go @@ -38,7 +38,7 @@ func TestAdaptiveSwitchVerdictOnAFailedStandby886(t *testing.T) { t.Skip("integration") } if !probe.Probe().IOUringTier.Available() { - t.Skip("io_uring unavailable: the standby refusal is never reached") + skipOrFailUpswitch890(t, "io_uring unavailable: the standby refusal is never reached") } run := func(require bool) (string, int) { t.Helper() diff --git a/test/integration/adaptive_standby_guard_886_linux_test.go b/test/integration/adaptive_standby_guard_886_linux_test.go index dd10f7d4e..82b6cd21e 100644 --- a/test/integration/adaptive_standby_guard_886_linux_test.go +++ b/test/integration/adaptive_standby_guard_886_linux_test.go @@ -56,7 +56,18 @@ func skipOrFailIfStandbyAborted886(t *testing.T, switches uint64, logs *syncBuf8 if switches != 0 || !standbyAborted886(logs) { return } - const msg = "the io_uring standby did not come up here: the engine aborted the forced switch and stayed put" + skipOrFailUpswitch890(t, "the io_uring standby did not come up here: the engine aborted the forced switch and stayed put") +} + +// skipOrFailUpswitch890 is this package's one definition of the gate that +// ./internal/adaptive keeps in skipOrFailUpswitch662 (the two cannot share a +// function: separate packages, and the test files of internal/adaptive cannot be +// imported). A test of the adaptive_*_test.go files that cannot run where the +// io_uring standby is unavailable skips through it, and fails instead under +// CELERIS_REQUIRE_UPSWITCH=1; TestAdaptiveSkipRoutingIntegration890 fails on a +// raw t.Skip in those files that bypasses it (celeris#890 item 96). +func skipOrFailUpswitch890(t *testing.T, msg string) { + t.Helper() if os.Getenv("CELERIS_REQUIRE_UPSWITCH") == "1" { t.Fatal(msg + " -- CELERIS_REQUIRE_UPSWITCH=1 forbids skipping") } diff --git a/test/integration/pipeline_test.go b/test/integration/pipeline_test.go index 05e50037b..23a088adf 100644 --- a/test/integration/pipeline_test.go +++ b/test/integration/pipeline_test.go @@ -37,7 +37,9 @@ func TestHTTP1PipeliningAsync(t *testing.T) { cfg.AsyncHandlers = tc.async eng, err := epoll.New(cfg, &echoHandler{}) if err != nil { - t.Skipf("epoll engine unavailable: %v", err) + // epoll.New fails on config validation only, so a failure here is a + // defect, not a host without the engine (celeris#890 item 95). + t.Fatalf("epoll.New: %v", err) } startEngine(t, eng) diff --git a/test/integration/skip_routing_890_test.go b/test/integration/skip_routing_890_test.go new file mode 100644 index 000000000..483b3aa80 --- /dev/null +++ b/test/integration/skip_routing_890_test.go @@ -0,0 +1,332 @@ +package integration + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "go/types" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "testing" +) + +// celeris#890 item 96: a test of the adaptive files of this package that cannot +// run where the io_uring standby is unavailable skips through +// skipOrFailUpswitch890 (adaptive_standby_guard_886_linux_test.go, which +// skipOrFailIfStandbyAborted886 calls too), which fails instead under +// CELERIS_REQUIRE_UPSWITCH=1. It repeats, in this +// package, the gate that internal/adaptive keeps in skipOrFailUpswitch662: the +// two cannot share a function (separate packages, and the test files of +// internal/adaptive cannot be imported), so what keeps this package's copy +// honest is this guard, the counterpart of internal/adaptive's +// TestAdaptiveSkipRouting684. +// +// It parses every adaptive_*_test.go file of the package, and helpers_test.go, and lists each t.Skip, +// t.Skipf and t.SkipNow call that is none of: +// +// - inside a gate helper (gateHelpers890); +// - under an `if` whose condition is true only when testing.Short() is +// (testing.Short() itself, or a && with it): a ||, a negation or an else +// branch is not exempt; +// - registered in unrouted890 with the reason it has not been routed. +// +// A new raw skip fails the guard until it goes through the gate or is +// registered. Nothing runs this package's tests in CI except this guard (the +// adaptive job runs it by name): the package needs a host, see mage. + +// gateHelpers890 are the functions whose job is to skip or fail; a raw skip +// inside one is the gate itself. +var gateHelpers890 = map[string]bool{ + "skipOrFailUpswitch890": true, // the gate (CELERIS_REQUIRE_UPSWITCH=1) +} + +// unrouted890 are the raw skips that stay for now, by file and the string +// literal that starts the skip's message, each with how many sites carry that +// message and why they stay. The count is exact: a further site with a +// registered message fails the guard as a stale entry would. +// +// Ten are in adaptive_hybrid_test.go and three in helpers_test.go (startEngine). #890 item 95 leaves the adaptive +// skips of this package to the lane that owns that file; the table is the count +// of what is left (the guard listed these ten, startEngine's three and one more, in +// adaptive_standby_886_linux_test.go, which now goes through the gate), and a +// new one cannot join them unseen. +type unrouted890Entry struct { + sites int + why string +} + +const ( + newFails890 = "adaptive.New fails on config validation or a port hold (internal/adaptive/engine.go New), not on a host without io_uring: a failure there is a defect, as for epoll.New in item 95" + startFail890 = "an engine that does not start, bind or answer: this skip cannot tell a host without io_uring from a defect" +) + +var unrouted890 = map[string]unrouted890Entry{ + "adaptive_hybrid_test.go|adaptive engine not available: %v": {5, newFails890}, + "adaptive_hybrid_test.go|engine failed to start: %v": {2, startFail890}, + "adaptive_hybrid_test.go|engine Listen returned nil without addr": {1, startFail890}, + "adaptive_hybrid_test.go|engine did not start in time": {1, startFail890}, + "adaptive_hybrid_test.go|engine not functional: %v": {1, startFail890}, + // startEngine, which the adaptive tests and the epoll arms of the slowloris and pipeline tests share. + "helpers_test.go|engine failed to start: %v": {2, startFail890}, + "helpers_test.go|engine not functional (skipping): %v": {1, startFail890}, +} + +// rawSkipViolations890 lists the t.Skip* calls of one Go source that the rules +// above do not admit, as "file:line: call(message)". It returns an error for a +// source that does not parse: the guard must fail on input it cannot read, not +// report no violation. +func rawSkipViolations890(name string, src []byte) ([]string, error) { + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, name, src, parser.SkipObjectResolution) + if err != nil { + return nil, err + } + var out []string + called := map[*ast.SelectorExpr]bool{} // Skip selectors seen as the function of a call + report := func(pos token.Pos, what string) { + out = append(out, fmt.Sprintf("%s:%d: %s", filepath.Base(name), fset.Position(pos).Line, what)) + } + var walk func(n ast.Node, fn string, short bool) + walk = func(n ast.Node, fn string, short bool) { + ast.Inspect(n, func(c ast.Node) bool { + switch x := c.(type) { + case *ast.FuncDecl: + if x != n { + if x.Body != nil { // a bodyless declaration holds no call + name := x.Name.Name + if x.Recv != nil { // a method of that name is not the gate function + name = "(method)" + name + } + walk(x.Body, name, short) + } + return false + } + case *ast.IfStmt: + if x != n && onlyWhenShort890(x.Cond) { + walk(x.Body, fn, true) + if x.Else != nil { + walk(x.Else, fn, short) + } + return false + } + case *ast.CallExpr: + // Any receiver: a subtest closure may name its *testing.T anything. + sel, ok := x.Fun.(*ast.SelectorExpr) + if !ok || !isSkipSelector890(sel) { + return true + } + called[sel] = true + if short || gateHelpers890[fn] { + return true + } + lit := skipMessage890(x) + if _, ok := unrouted890[filepath.Base(name)+"|"+lit]; ok { + return true + } + report(x.Pos(), fmt.Sprintf("%s.%s(%q)", types.ExprString(sel.X), sel.Sel.Name, lit)) + case *ast.SelectorExpr: + // A method value (s := t.Skip; s(...), run(t.SkipNow)) skips without a + // call the rules can see. A call's own selector was recorded above, as + // the CallExpr is visited before its Fun. + if isSkipSelector890(x) && !called[x] && !short && !gateHelpers890[fn] { + report(x.Pos(), fmt.Sprintf("%s.%s as a method value", types.ExprString(x.X), x.Sel.Name)) + } + } + return true + }) + } + walk(f, "", false) + return out, nil +} + +// isSkipSelector890 reports whether the selector names Skip, Skipf or SkipNow. +func isSkipSelector890(s *ast.SelectorExpr) bool { + switch s.Sel.Name { + case "Skip", "Skipf", "SkipNow": + return true + } + return false +} + +// onlyWhenShort890 reports whether the condition is true only when +// testing.Short() is: the call itself, parenthesised, or a && with such an +// operand. A ||, a negation or any other use of testing.Short() is not: the +// body would skip on a run that is not -short. +func onlyWhenShort890(e ast.Expr) bool { + switch v := e.(type) { + case *ast.ParenExpr: + return onlyWhenShort890(v.X) + case *ast.BinaryExpr: + return v.Op == token.LAND && (onlyWhenShort890(v.X) || onlyWhenShort890(v.Y)) + case *ast.CallExpr: + if s, ok := v.Fun.(*ast.SelectorExpr); ok && s.Sel.Name == "Short" { + id, ok := s.X.(*ast.Ident) + return ok && id.Name == "testing" && len(v.Args) == 0 + } + } + return false +} + +// skipMessage890 is the string literal a Skip call's message starts with: the +// first argument, with the operands of a "+" concatenation of literals joined. +func skipMessage890(c *ast.CallExpr) string { + if len(c.Args) == 0 { + return "" + } + var lit func(e ast.Expr) string + lit = func(e ast.Expr) string { + switch v := e.(type) { + case *ast.BasicLit: + if v.Kind == token.STRING { + if s, err := strconv.Unquote(v.Value); err == nil { + return s + } + } + case *ast.BinaryExpr: + if v.Op == token.ADD { + return lit(v.X) + lit(v.Y) + } + } + return "" + } + return lit(c.Args[0]) +} + +// TestAdaptiveSkipRoutingIntegration890 runs the guard on the adaptive files of +// this package. +func TestAdaptiveSkipRoutingIntegration890(t *testing.T) { + files, err := filepath.Glob("adaptive_*_test.go") + if err != nil { + t.Fatal(err) + } + // helpers_test.go: startEngine skips when Listen fails, and the adaptive tests call it + // (review r1 of #1124, m3). + files = append(files, "helpers_test.go") + // A glob that finds nothing, or not the files this guard exists for, passes + // vacuously. + have := map[string]bool{} + for _, f := range files { + have[f] = true + } + for _, must := range []string{"adaptive_standby_guard_886_linux_test.go", "adaptive_hybrid_test.go", "adaptive_standby_886_linux_test.go", "helpers_test.go"} { + if !have[must] { + t.Fatalf("the guard did not see %s among %d adaptive test files: it would pass without checking it", must, len(files)) + } + } + sort.Strings(files) + var bad []string + seen := map[string]int{} + for _, f := range files { + src, err := os.ReadFile(f) + if err != nil { + t.Fatal(err) + } + v, err := rawSkipViolations890(f, src) + if err != nil { + t.Fatalf("%s does not parse: %v", f, err) + } + bad = append(bad, v...) + fset := token.NewFileSet() + af, err := parser.ParseFile(fset, f, src, parser.SkipObjectResolution) + if err != nil { + t.Fatalf("%s does not parse: %v", f, err) + } + ast.Inspect(af, func(n ast.Node) bool { + if c, ok := n.(*ast.CallExpr); ok { + if s, ok := c.Fun.(*ast.SelectorExpr); ok && isSkipSelector890(s) { + seen[f+"|"+skipMessage890(c)]++ + } + } + return true + }) + } + t.Logf("%d scanned test files, %d raw skips not admitted, %d registered in unrouted890", len(files), len(bad), len(unrouted890)) + if len(bad) > 0 { + t.Errorf("t.Skip calls that bypass skipOrFailUpswitch890 (adaptive_standby_guard_886_linux_test.go) and are not registered in unrouted890:\n %s", + strings.Join(bad, "\n ")) + } + // Every registered skip must still exist, as many times as registered, or + // the table keeps a reason for nothing or admits a site unseen. + for k, e := range unrouted890 { + if seen[k] != e.sites { + t.Errorf("unrouted890 registers %q at %d sites, but the adaptive files have %d t.Skip calls with it: fix the entry", k, e.sites, seen[k]) + } + } +} + +// TestAdaptiveSkipRoutingGuardSelfTest890 checks the guard on fixtures, and on +// the real gate file with a planted regression. +func TestAdaptiveSkipRoutingGuardSelfTest890(t *testing.T) { + const head = "package integration\n\nimport \"testing\"\n\n" + for _, c := range []struct { + name string + src string + want int // violations; -1 = the guard must report a parse error + }{ + {"raw skip for a missing io_uring fails", head + `func TestX(t *testing.T) { t.Skip("io_uring unavailable here") }`, 1}, + {"raw Skipf fails", head + `func TestX(t *testing.T) { t.Skipf("adaptive engine not available: %v", 1) }`, 1}, + {"raw SkipNow fails", head + `func TestX(t *testing.T) { t.SkipNow() }`, 1}, + {"a raw skip in a subtest closure fails", head + `func TestX(t *testing.T) { t.Run("a", func(t *testing.T) { t.Skip("io_uring unavailable") }) }`, 1}, + {"the gate passes", head + `func TestX(t *testing.T) { skipOrFailUpswitch890(t, "io_uring unavailable") }`, 0}, + {"a skip inside the gate helper passes", head + `func skipOrFailUpswitch890(t *testing.T, m string) { t.Skip(m) }`, 0}, + {"a registered message is not exempt in an unregistered file", head + `func TestX(t *testing.T) { t.Skipf("engine failed to start: %v", 1) }`, 1}, + {"a -short gate passes", head + `func TestX(t *testing.T) { if testing.Short() { t.Skip("integration") } }`, 0}, + {"a -short gate with another clause passes", head + `func TestX(t *testing.T) { if x && testing.Short() { t.Skip("integration") } }`, 0}, + {"a raw skip after a -short gate fails", head + `func TestX(t *testing.T) { if testing.Short() { t.Skip("integration") }; t.Skip("io_uring unavailable") }`, 1}, + {"a raw skip in the else of a -short gate fails", head + `func TestX(t *testing.T) { if testing.Short() { } else { t.Skip("io_uring unavailable") } }`, 1}, + {"an || with testing.Short() still skips when not short and fails", head + `func TestX(t *testing.T) { if testing.Short() || !avail() { t.Skip("io_uring unavailable") } }`, 1}, + {"a negated testing.Short() gate fails", head + `func TestX(t *testing.T) { if !testing.Short() { t.Skip("io_uring unavailable") } }`, 1}, + {"a skip through a receiver named otherwise fails", head + `func TestX(t *testing.T) { t.Run("a", func(st *testing.T) { st.Skip("io_uring unavailable") }) }`, 1}, + {"a Skip method value fails", head + `func TestX(t *testing.T) { s := t.Skip; s("io_uring unavailable") }`, 1}, + {"a SkipNow method value passed as an argument fails", head + `func TestX(t *testing.T) { run(t.SkipNow) }`, 1}, + {"a Skip method value under a -short gate passes", head + `func TestX(t *testing.T) { if testing.Short() { s := t.Skip; s("integration") } }`, 0}, + {"a method named like the gate is not the gate", head + `func (s *S) skipOrFailUpswitch890(t *testing.T, m string) { t.Skip(m) }`, 1}, + {"a bodyless function declaration is not a violation or a panic", head + `func foo()`, 0}, + {"input that does not parse is an error, not a pass", head + `func TestX(t *testing.T) { t.Skip(`, -1}, + {"empty input is an error, not a pass", ``, -1}, + } { + t.Run(c.name, func(t *testing.T) { + v, err := rawSkipViolations890("fixture_test.go", []byte(c.src)) + if c.want == -1 { + if err == nil { + t.Fatalf("a source that does not parse gave no error (violations %v)", v) + } + return + } + if err != nil { + t.Fatal(err) + } + if len(v) != c.want { + t.Fatalf("violations %v, want %d", v, c.want) + } + }) + } + + // The real gate file with the regression this guard is for: the gate's own + // skip, copied out of the helper into a test body, must be listed, and the + // file as it is must not. + const file = "adaptive_standby_guard_886_linux_test.go" + src, err := os.ReadFile(file) + if err != nil { + t.Fatal(err) + } + if v, err := rawSkipViolations890(file, src); err != nil || len(v) != 0 { + t.Fatalf("%s as it is: violations %v, err %v, want none", file, v, err) + } + const gate = "\tt.Skip(msg)\n}" + if strings.Count(string(src), gate) != 1 { + t.Fatalf("%s no longer ends the helper with %q: update the control", file, gate) + } + mutant := strings.Replace(string(src), gate, "\tt.Skip(msg)\n}\n\nfunc TestPlanted890(t *testing.T) { t.Skip(\"io_uring unavailable\") }", 1) + v, err := rawSkipViolations890(file, []byte(mutant)) + if err != nil { + t.Fatal(err) + } + if len(v) != 1 { + t.Fatalf("the planted raw skip was listed %d times (%v), want 1", len(v), v) + } +} diff --git a/test/integration/slowloris_synthetic_test.go b/test/integration/slowloris_synthetic_test.go index b8e6a2ce1..80e75bd9c 100644 --- a/test/integration/slowloris_synthetic_test.go +++ b/test/integration/slowloris_synthetic_test.go @@ -14,6 +14,18 @@ import ( "github.com/goceleris/celeris/internal/resource" ) +// engineNewFailed890 handles a failed engine constructor in a table of engines +// (celeris#890 item 95). epoll.New fails on config validation only, so on the +// epoll arm a failure is a defect and the test fails; the io_uring arm skips, +// as before (io_uring skips are celeris#888's). +func engineNewFailed890(t *testing.T, name string, err error) { + t.Helper() + if name == "epoll" { + t.Fatalf("epoll.New: %v", err) + } + t.Skipf("%s engine unavailable on this host: %v", name, err) +} + // TestSlowlorisSynthetic is the focused per-engine reproducer for the // v1.4.11 slowloris-defence work. Replaces the 1h+ probatorium nightly // cycle with an in-process test that logs every relevant step. @@ -39,9 +51,8 @@ import ( // (FIN-vs-RST issue — RST would force immediate fail). func TestSlowlorisSynthetic(t *testing.T) { for _, tc := range []struct { - name string - newEng func(cfg resource.Config) (engine.Engine, error) - minSkip string // skip message if engine unsupported + name string + newEng func(cfg resource.Config) (engine.Engine, error) }{ { name: "iouring", @@ -68,10 +79,7 @@ func TestSlowlorisSynthetic(t *testing.T) { } eng, err := tc.newEng(cfg) if err != nil { - if tc.minSkip != "" { - t.Skip(tc.minSkip) - } - t.Skipf("%s engine unavailable on this host: %v", tc.name, err) + engineNewFailed890(t, tc.name, err) } startEngine(t, eng) @@ -210,7 +218,7 @@ func TestSlowlorisSyntheticConcurrent(t *testing.T) { } eng, err := tc.newEng(cfg) if err != nil { - t.Skipf("%s engine unavailable: %v", tc.name, err) + engineNewFailed890(t, tc.name, err) } startEngine(t, eng) addr := eng.(addrGetter).Addr().String() diff --git a/test/server/adapt_request_linux_test.go b/test/server/adapt_request_linux_test.go index e0baacf5d..7a313fe3c 100644 --- a/test/server/adapt_request_linux_test.go +++ b/test/server/adapt_request_linux_test.go @@ -174,7 +174,7 @@ func keptArms(t *testing.T) []keptArm { {"epoll", celeris.Epoll, false}, {"epoll-async", celeris.Epoll, true}, } - if ok, p := keptProbeIOUring(); ok { + if ok, p := keptProbeIOUring(t); ok { arms = append(arms, keptArm{"io_uring", celeris.IOUring, false}, keptArm{"io_uring-async", celeris.IOUring, true}) } else if os.Getenv("CELERIS_REQUIRE_IOURING_WORKERS") == "1" { t.Fatalf("io_uring tier=%s kernel=%s, and CELERIS_REQUIRE_IOURING_WORKERS=1 forbids dropping the io_uring arms", p.IOUringTier, p.KernelVersion) @@ -186,17 +186,21 @@ func keptArms(t *testing.T) []keptArm { // keptProbeIOUring probes the kernel's io_uring support. With // CELERIS_REQUIRE_IOURING_WORKERS=1 a probe that finds no usable ring is -// retried for up to 10 s: the probe's ring can fail with ENOMEM against +// retried for up to 10 s, logging each round through tb: the probe's ring can fail with ENOMEM against // RLIMIT_MEMLOCK while the rings of engines stopped moments ago, or of // another test binary of the same user, are still charged // (internal/engine/iouring/ring_budget_linux_test.go). -func keptProbeIOUring() (usable bool, p celerisengine.CapabilityProfile) { +func keptProbeIOUring(tb testing.TB) (usable bool, p celerisengine.CapabilityProfile) { p = probe.Probe() usable = p.IOUringTier >= celerisengine.High && p.ProvidedBuffers if os.Getenv("CELERIS_REQUIRE_IOURING_WORKERS") != "1" { return usable, p } - for deadline := time.Now().Add(10 * time.Second); !usable && time.Now().Before(deadline); { + deadline := time.Now().Add(10 * time.Second) + for round := 1; !usable && time.Now().Before(deadline); round++ { + // One line per round, so a CI log shows that the probe fired and how long it + // kept failing. Rounds are 10 ms apart, so this is at most about 1000 lines. + tb.Logf("keptProbeIOUring: retry round %d, no usable io_uring ring yet (tier %v, provided buffers %t)", round, p.IOUringTier, p.ProvidedBuffers) time.Sleep(10 * time.Millisecond) p = probe.Probe() usable = p.IOUringTier >= celerisengine.High && p.ProvidedBuffers diff --git a/test/server/server_async_goexit_791_linux_test.go b/test/server/server_async_goexit_791_linux_test.go index ff3918fdb..d945ffae3 100644 --- a/test/server/server_async_goexit_791_linux_test.go +++ b/test/server/server_async_goexit_791_linux_test.go @@ -97,7 +97,7 @@ const ( func runServerAsyncAbort791(t *testing.T, eng celeris.EngineType, mode string) { if eng == celeris.IOUring { - if ok, p := keptProbeIOUring(); !ok { + if ok, p := keptProbeIOUring(t); !ok { if os.Getenv("CELERIS_REQUIRE_IOURING_WORKERS") == "1" { t.Fatalf("io_uring tier=%s kernel=%s, and CELERIS_REQUIRE_IOURING_WORKERS=1 forbids skipping", p.IOUringTier, p.KernelVersion) } diff --git a/test/server/server_sync_goexit_848_linux_test.go b/test/server/server_sync_goexit_848_linux_test.go index fbf9d3110..7588349dc 100644 --- a/test/server/server_sync_goexit_848_linux_test.go +++ b/test/server/server_sync_goexit_848_linux_test.go @@ -141,7 +141,7 @@ func need848(t *testing.T, eng celeris.EngineType) { if eng != celeris.IOUring && eng != celeris.Adaptive { return } - if ok, p := keptProbeIOUring(); !ok { + if ok, p := keptProbeIOUring(t); !ok { if os.Getenv("CELERIS_REQUIRE_IOURING_WORKERS") == "1" { t.Fatalf("io_uring tier=%s kernel=%s, and CELERIS_REQUIRE_IOURING_WORKERS=1 forbids skipping", p.IOUringTier, p.KernelVersion) }