From a61d9104319b2caa782afb53f56a6aa88c0b05df Mon Sep 17 00:00:00 2001 From: Albert Bausili Date: Mon, 26 Oct 2026 23:38:38 +0100 Subject: [PATCH 01/14] ci: gates that fail on main: strict alloc budgets, Python tool pins, the test/integration skip guard (Refs #890) Failing-first arm of three items of #890. Each gate is added with the fix left out, so that CI shows what it catches: - item 91: TESTING_STRICT_ALLOC_BUDGETS=1 on the allocation-guard step. The postgres Query_1col_1row guard measures 14 allocs/op against a budget of 4. - item 93: tool-pins-check.sh check 4. No workflow installs or runs a Python tool with a name==version of its own, and every requirements file under .github/ is in a Dependabot pip entry. ci.yml still runs `pipx run --spec zizmor==1.30.0`, so Lint is red, and the "good" fixtures of the self-test, which copy the real tree, are red with it. - item 96: TestAdaptiveSkipRoutingIntegration890 with an empty registry lists the 11 raw skips of test/integration/adaptive_*_test.go that bypass the gate; a CI step in the adaptive job runs it by name. --- .github/scripts/tool-pins-check-test.sh | 60 +++++ .github/scripts/tool-pins-check.sh | 88 +++++++ .github/workflows/ci.yml | 18 ++ test/integration/skip_routing_890_test.go | 295 ++++++++++++++++++++++ 4 files changed, 461 insertions(+) create mode 100644 test/integration/skip_routing_890_test.go diff --git a/.github/scripts/tool-pins-check-test.sh b/.github/scripts/tool-pins-check-test.sh index 67102c701..5c80f216a 100755 --- a/.github/scripts/tool-pins-check-test.sh +++ b/.github/scripts/tool-pins-check-test.sh @@ -24,6 +24,11 @@ fixture() { mkdir -p "$dir/$1/.github/actions/probe" printf 'name: probe\nruns:\n using: composite\n steps:\n - run: echo ok\n shell: bash\n' >"$dir/$1/.github/actions/probe/action.yml" cp "$repo/CONTRIBUTING.md" "$dir/$1/" + # The requirements files of the Python tools (check 4), where there are any. + find "$repo/.github" -name 'requirements*.txt' -type f | while IFS= read -r f; do + mkdir -p "$dir/$1/$(dirname "${f#"$repo"/}")" + cp "$f" "$dir/$1/${f#"$repo"/}" + done } # expect NAME WANT_STATUS WANT_STDERR_SUBSTRING: runs the check on $dir/NAME. @@ -142,5 +147,60 @@ fixture go-run-local printf ' - run: go run ./cmd/tool -mail me@example.com\n' >>"$dir/go-run-local/.github/workflows/ci.yml" expect go-run-local 0 +# 4. Python tools: a version of their own in a workflow, and a requirements +# file that no Dependabot pip entry watches (celeris#890 item 93). +fixture pipx-spec +printf ' - run: pipx run --spec zizmor==1.30.0 zizmor .github/workflows\n' >>"$dir/pipx-spec/.github/workflows/ci.yml" +expect pipx-spec 1 'ci.yml' +fixture pip-install-pin +printf ' - run: python3 -m pip install zizmor==1.30.0\n' >>"$dir/pip-install-pin/.github/workflows/ci.yml" +expect pip-install-pin 1 'ci.yml' +fixture pip-install-extras +printf ' - run: pip3 install --quiet "zizmor[x]==1.30.0"\n' >>"$dir/pip-install-extras/.github/workflows/ci.yml" +expect pip-install-extras 1 'ci.yml' +fixture uvx-pin +printf ' - run: uvx --from zizmor==1.30.0 zizmor .\n' >>"$dir/uvx-pin/.github/workflows/ci.yml" +expect uvx-pin 1 'ci.yml' +fixture pip-install-continuation +printf ' - run: |\n pipx run \\\n --spec zizmor==1.30.0 zizmor .\n' >>"$dir/pip-install-continuation/.github/workflows/ci.yml" +expect pip-install-continuation 1 'ci.yml' +fixture pip-install-composite +printf ' - run: pipx install zizmor==1.30.0\n shell: bash\n' >>"$dir/pip-install-composite/.github/actions/probe/action.yml" +expect pip-install-composite 1 'actions/probe/action.yml' +# Reading the pins from a requirements file, and a comment, are fine. +fixture pip-install-requirements +printf ' - run: pip install -r .github/tools/requirements.txt\n' >>"$dir/pip-install-requirements/.github/workflows/ci.yml" +expect pip-install-requirements 0 +fixture pip-install-comment +printf ' # was: pipx run --spec zizmor==1.30.0 zizmor .\n' >>"$dir/pip-install-comment/.github/workflows/ci.yml" +expect pip-install-comment 0 +# An == that is no pin of a Python tool is fine. +fixture eq-in-condition +printf ' - run: if [ "$x" == "pip install" ]; then echo ok; fi\n' >>"$dir/eq-in-condition/.github/workflows/ci.yml" +expect eq-in-condition 0 + +# A requirements file in a directory no pip entry names, and the entry added +# in either spelling. +fixture requirements-unwatched +mkdir -p "$dir/requirements-unwatched/.github/ci-extra" +printf 'zizmor==1.30.0\n' >"$dir/requirements-unwatched/.github/ci-extra/requirements.txt" +expect requirements-unwatched 1 "no 'package-ecosystem: pip' entry" +fixture requirements-watched +mkdir -p "$dir/requirements-watched/.github/ci-extra" +printf 'zizmor==1.30.0\n' >"$dir/requirements-watched/.github/ci-extra/requirements.txt" +printf ' - package-ecosystem: pip\n directory: /.github/ci-extra\n schedule:\n interval: daily\n' >>"$dir/requirements-watched/.github/dependabot.yml" +expect requirements-watched 0 +fixture requirements-watched-list +mkdir -p "$dir/requirements-watched-list/.github/ci-extra" +printf 'zizmor==1.30.0\n' >"$dir/requirements-watched-list/.github/ci-extra/requirements.txt" +printf ' - package-ecosystem: pip\n directories:\n - /somewhere\n - /.github/ci-extra\n schedule:\n interval: daily\n' >>"$dir/requirements-watched-list/.github/dependabot.yml" +expect requirements-watched-list 0 +# An entry of another ecosystem for the same directory does not watch pip. +fixture requirements-wrong-ecosystem +mkdir -p "$dir/requirements-wrong-ecosystem/.github/ci-extra" +printf 'zizmor==1.30.0\n' >"$dir/requirements-wrong-ecosystem/.github/ci-extra/requirements.txt" +printf ' - package-ecosystem: gomod\n directory: /.github/ci-extra\n schedule:\n interval: daily\n' >>"$dir/requirements-wrong-ecosystem/.github/dependabot.yml" +expect requirements-wrong-ecosystem 1 "no 'package-ecosystem: pip' entry" + echo "$n cases" exit "$fail" diff --git a/.github/scripts/tool-pins-check.sh b/.github/scripts/tool-pins-check.sh index b3218c77c..fe5f04fe4 100755 --- a/.github/scripts/tool-pins-check.sh +++ b/.github/scripts/tool-pins-check.sh @@ -22,6 +22,15 @@ # with # is a comment and is skipped; a # later in a line is not (it can # sit in an echo string), so a trailing comment that quotes such a command # is flagged too. A YAML folded scalar (run: >) is not joined. +# 4. The Python tools of CI (zizmor) are pinned in a requirements file under +# .github/, which Dependabot's pip ecosystem reads. (a) No workflow or +# composite action installs or runs a Python tool with a `name==version` +# of its own (`pipx run --spec zizmor==1.30.0`, `pip install x==1`, `uvx +# x==1`): that pin is read by no Dependabot entry, and stayed on its +# version until somebody remembered (celeris#890 item 93). The same line +# rules as check 3 apply. (b) Every directory under .github/ that holds a +# requirements*.txt has a `package-ecosystem: pip` entry in dependabot.yml +# for it (`directory:` or one of `directories:`), or nothing watches it. # # Prints one line per failed check and exits 1 if any failed. The Lint job # runs it, and tool-pins-check-test.sh runs it on broken fixtures. @@ -111,5 +120,84 @@ if [ -n "$hits" ]; then echo " ${hits//$'\n'/$'\n' }" >&2 fi +# 4. Python tools: pinned in a requirements file that Dependabot watches. +pip_hits() { + python3 - "$1" <<'PY' +import glob, os, re, sys +root = sys.argv[1] +pats = ['.github/workflows/*.yml', '.github/workflows/*.yaml', + '.github/actions/**/*.yml', '.github/actions/**/*.yaml'] +# A Python tool's installer or runner, then (flags) name==version. pip's -r +# FILE form has no ==, so reading the pins from the file is not flagged. +pat = re.compile(r'(?:\bpipx\s+(?:run|install)\b|\bpip3?\s+install\b|\bpython3?\s+-m\s+pip\s+install\b|\buvx\b|\buv\s+tool\s+(?:run|install)\b).*?[A-Za-z0-9_.-]+(?:\[[^]]*\])?==\S') +files = sorted({f for p in pats for f in glob.glob(os.path.join(root, p), recursive=True)}) +for f in files: + lines = open(f).read().splitlines() + i = 0 + while i < len(lines): + start, text = i, lines[i] + while text.rstrip().endswith('\\') and i + 1 < len(lines): + i += 1 + text = text.rstrip()[:-1] + ' ' + lines[i].strip() + i += 1 + if text.lstrip().startswith('#'): + continue + if pat.search(text): + print("%s:%d:%s" % (os.path.relpath(f, root), start + 1, lines[start].strip())) +PY +} +hits=$(pip_hits "$root") +if [ -n "$hits" ]; then + bad "a workflow or action installs or runs a Python tool with its own name==version; pin it in a requirements file under .github/ (Dependabot's pip entry watches it) and install it with pip install -r:" + echo " ${hits//$'\n'/$'\n' }" >&2 +fi +# The pip entries of dependabot.yml, as "/dir" lines: an entry runs from its +# "- package-ecosystem:" line to the next one; its `directory:` and the items +# of its `directories:` list count. +pip_dirs() { + python3 - "$1" <<'PY' +import re, sys +dirs = [] +eco, found, in_list = None, [], False +def end_entry(): + if eco == "pip": + dirs.extend(found) +for line in open(sys.argv[1]).read().splitlines(): + m = re.match(r'^\s*-\s*package-ecosystem:\s*["\']?([^\s"\'#]+)', line) + if m: + end_entry() + eco, found, in_list = m.group(1), [], False + continue + m = re.match(r'^\s*directory:\s*["\']?([^\s"\'#]+)', line) + if m: + found.append(m.group(1)); in_list = False + continue + if re.match(r'^\s*directories:\s*(#.*)?$', line): + in_list = True + continue + m = re.match(r'^\s*-\s*["\']?(/[^\s"\'#]*)', line) + if m and in_list: + found.append(m.group(1)) + continue + if in_list and not re.match(r'^\s*(#.*)?$', line): + in_list = False +end_entry() +print("\n".join(sorted(set(dirs)))) +PY +} +watched=$(pip_dirs "$root/.github/dependabot.yml") +unwatched= +while IFS= read -r f; do + d=/$(dirname "${f#"$root"/}") + case " +$watched +" in *" +$d +"*) ;; *) unwatched="$unwatched $d" ;; esac +done < <(find "$root/.github" -name 'requirements*.txt' -type f | sort) +if [ -n "$unwatched" ]; then + bad ".github/dependabot.yml has no 'package-ecosystem: pip' entry for the directory of a requirements file:$unwatched" +fi + [ "$fail" = 0 ] && echo "tool-pins-check: ok (tools: $(echo "$tools" | paste -sd' ' -); golangci-lint $ci)" exit "$fail" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b5183977c..86a447d3d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1262,6 +1262,8 @@ jobs: - name: celeris#890 allocation guards and alloc-count tests, without -race (skipping forbidden) if: ${{ !cancelled() }} shell: bash + env: + TESTING_STRICT_ALLOC_BUDGETS: "1" run: | set -o pipefail # " " @@ -1792,6 +1794,22 @@ jobs: echo "engines to report a refused steal -- was one renamed or removed, or did an io_uring arm skip?" exit 1 fi + # celeris#890 item 96: ./test/integration is not run by this workflow (its + # tests need a host; `mage` runs them), but its adaptive_*_test.go files keep + # a copy of ./internal/adaptive's CELERIS_REQUIRE_UPSWITCH gate + # (skipOrFailIfStandbyAborted886). This guard, the counterpart of + # TestAdaptiveSkipRouting684, fails on a raw t.Skip in those files that + # bypasses it and is not registered with a reason, and its self-test + # plants one. It is the only test of the package that runs here, by name. + - name: celeris#890 test/integration adaptive skips go through the gate + if: ${{ !cancelled() }} + shell: bash + run: | + set -o pipefail + go test -count=1 -timeout=300s -v -run '^(TestAdaptiveSkipRoutingIntegration890|TestAdaptiveSkipRoutingGuardSelfTest890)$' ./test/integration/ 2>&1 | tee /tmp/integration-skip-guard.log + "$GITHUB_WORKSPACE"/.github/scripts/named-tally.sh 'test/integration skip guard (celeris#890 item 96)' /tmp/integration-skip-guard.log \ + TestAdaptiveSkipRoutingIntegration890 \ + TestAdaptiveSkipRoutingGuardSelfTest890 # NOT a required check: the repository's ruleset ("Main — CI") requires Lint, # "Unit (root + middleware sub-modules)" (the `unit-gate` job), Conformance, diff --git a/test/integration/skip_routing_890_test.go b/test/integration/skip_routing_890_test.go new file mode 100644 index 000000000..bafb6886e --- /dev/null +++ b/test/integration/skip_routing_890_test.go @@ -0,0 +1,295 @@ +package integration + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "go/types" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "testing" +) + +// celeris#890 item 96: a test of the adaptive files of this package that cannot +// run where the io_uring standby is unavailable skips through +// skipOrFailIfStandbyAborted886 (adaptive_standby_guard_886_linux_test.go), +// which fails instead under CELERIS_REQUIRE_UPSWITCH=1. It repeats, in this +// package, the gate that internal/adaptive keeps in skipOrFailUpswitch662: the +// two cannot share a function (separate packages, and the test files of +// internal/adaptive cannot be imported), so what keeps this package's copy +// honest is this guard, the counterpart of internal/adaptive's +// TestAdaptiveSkipRouting684. +// +// It parses every adaptive_*_test.go file of the package and lists each t.Skip, +// t.Skipf and t.SkipNow call that is none of: +// +// - inside a gate helper (gateHelpers890); +// - under an `if` whose condition is true only when testing.Short() is +// (testing.Short() itself, or a && with it): a ||, a negation or an else +// branch is not exempt; +// - registered in unrouted890 with the reason it has not been routed. +// +// A new raw skip fails the guard until it goes through the gate or is +// registered. Nothing runs this package's tests in CI except this guard (the +// adaptive job runs it by name): the package needs a host, see mage. + +// gateHelpers890 are the functions whose job is to skip or fail; a raw skip +// inside one is the gate itself. +var gateHelpers890 = map[string]bool{ + "skipOrFailIfStandbyAborted886": true, // the standby gate (CELERIS_REQUIRE_UPSWITCH=1) +} + +// unrouted890 are the raw skips that stay for now, by file and the string +// literal that starts the skip's message, each with why. +var unrouted890 = map[string]string{} + +// rawSkipViolations890 lists the t.Skip* calls of one Go source that the rules +// above do not admit, as "file:line: call(message)". It returns an error for a +// source that does not parse: the guard must fail on input it cannot read, not +// report no violation. +func rawSkipViolations890(name string, src []byte) ([]string, error) { + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, name, src, parser.SkipObjectResolution) + if err != nil { + return nil, err + } + var out []string + called := map[*ast.SelectorExpr]bool{} // Skip selectors seen as the function of a call + report := func(pos token.Pos, what string) { + out = append(out, fmt.Sprintf("%s:%d: %s", filepath.Base(name), fset.Position(pos).Line, what)) + } + var walk func(n ast.Node, fn string, short bool) + walk = func(n ast.Node, fn string, short bool) { + ast.Inspect(n, func(c ast.Node) bool { + switch x := c.(type) { + case *ast.FuncDecl: + if x != n { + if x.Body != nil { // a bodyless declaration holds no call + walk(x.Body, x.Name.Name, short) + } + return false + } + case *ast.IfStmt: + if x != n && onlyWhenShort890(x.Cond) { + walk(x.Body, fn, true) + if x.Else != nil { + walk(x.Else, fn, short) + } + return false + } + case *ast.CallExpr: + // Any receiver: a subtest closure may name its *testing.T anything. + sel, ok := x.Fun.(*ast.SelectorExpr) + if !ok || !isSkipSelector890(sel) { + return true + } + called[sel] = true + if short || gateHelpers890[fn] { + return true + } + lit := skipMessage890(x) + if _, ok := unrouted890[filepath.Base(name)+"|"+lit]; ok { + return true + } + report(x.Pos(), fmt.Sprintf("%s.%s(%q)", types.ExprString(sel.X), sel.Sel.Name, lit)) + case *ast.SelectorExpr: + // A method value (s := t.Skip; s(...), run(t.SkipNow)) skips without a + // call the rules can see. A call's own selector was recorded above, as + // the CallExpr is visited before its Fun. + if isSkipSelector890(x) && !called[x] && !short && !gateHelpers890[fn] { + report(x.Pos(), fmt.Sprintf("%s.%s as a method value", types.ExprString(x.X), x.Sel.Name)) + } + } + return true + }) + } + walk(f, "", false) + return out, nil +} + +// isSkipSelector890 reports whether the selector names Skip, Skipf or SkipNow. +func isSkipSelector890(s *ast.SelectorExpr) bool { + switch s.Sel.Name { + case "Skip", "Skipf", "SkipNow": + return true + } + return false +} + +// onlyWhenShort890 reports whether the condition is true only when +// testing.Short() is: the call itself, parenthesised, or a && with such an +// operand. A ||, a negation or any other use of testing.Short() is not: the +// body would skip on a run that is not -short. +func onlyWhenShort890(e ast.Expr) bool { + switch v := e.(type) { + case *ast.ParenExpr: + return onlyWhenShort890(v.X) + case *ast.BinaryExpr: + return v.Op == token.LAND && (onlyWhenShort890(v.X) || onlyWhenShort890(v.Y)) + case *ast.CallExpr: + if s, ok := v.Fun.(*ast.SelectorExpr); ok && s.Sel.Name == "Short" { + id, ok := s.X.(*ast.Ident) + return ok && id.Name == "testing" && len(v.Args) == 0 + } + } + return false +} + +// skipMessage890 is the string literal a Skip call's message starts with: the +// first argument, with the operands of a "+" concatenation of literals joined. +func skipMessage890(c *ast.CallExpr) string { + if len(c.Args) == 0 { + return "" + } + var lit func(e ast.Expr) string + lit = func(e ast.Expr) string { + switch v := e.(type) { + case *ast.BasicLit: + if v.Kind == token.STRING { + if s, err := strconv.Unquote(v.Value); err == nil { + return s + } + } + case *ast.BinaryExpr: + if v.Op == token.ADD { + return lit(v.X) + lit(v.Y) + } + } + return "" + } + return lit(c.Args[0]) +} + +// TestAdaptiveSkipRoutingIntegration890 runs the guard on the adaptive files of +// this package. +func TestAdaptiveSkipRoutingIntegration890(t *testing.T) { + files, err := filepath.Glob("adaptive_*_test.go") + if err != nil { + t.Fatal(err) + } + // A glob that finds nothing, or not the files this guard exists for, passes + // vacuously. + have := map[string]bool{} + for _, f := range files { + have[f] = true + } + for _, must := range []string{"adaptive_standby_guard_886_linux_test.go", "adaptive_hybrid_test.go", "adaptive_standby_886_linux_test.go"} { + if !have[must] { + t.Fatalf("the guard did not see %s among %d adaptive test files: it would pass without checking it", must, len(files)) + } + } + sort.Strings(files) + var bad []string + seen := map[string]bool{} + for _, f := range files { + src, err := os.ReadFile(f) + if err != nil { + t.Fatal(err) + } + v, err := rawSkipViolations890(f, src) + if err != nil { + t.Fatalf("%s does not parse: %v", f, err) + } + bad = append(bad, v...) + fset := token.NewFileSet() + af, err := parser.ParseFile(fset, f, src, parser.SkipObjectResolution) + if err != nil { + t.Fatalf("%s does not parse: %v", f, err) + } + ast.Inspect(af, func(n ast.Node) bool { + if c, ok := n.(*ast.CallExpr); ok { + if s, ok := c.Fun.(*ast.SelectorExpr); ok && isSkipSelector890(s) { + seen[f+"|"+skipMessage890(c)] = true + } + } + return true + }) + } + t.Logf("%d adaptive test files, %d raw skips not admitted, %d registered in unrouted890", len(files), len(bad), len(unrouted890)) + if len(bad) > 0 { + t.Errorf("t.Skip calls that bypass skipOrFailIfStandbyAborted886 (adaptive_standby_guard_886_linux_test.go) and are not registered in unrouted890:\n %s", + strings.Join(bad, "\n ")) + } + // Every registered skip must still exist, or the table keeps a reason for + // nothing. + for k := range unrouted890 { + if !seen[k] { + t.Errorf("unrouted890 registers %q, which no t.Skip in the adaptive files has any more: delete the entry", k) + } + } +} + +// TestAdaptiveSkipRoutingGuardSelfTest890 checks the guard on fixtures, and on +// the real gate file with a planted regression. +func TestAdaptiveSkipRoutingGuardSelfTest890(t *testing.T) { + const head = "package integration\n\nimport \"testing\"\n\n" + for _, c := range []struct { + name string + src string + want int // violations; -1 = the guard must report a parse error + }{ + {"raw skip for a missing io_uring fails", head + `func TestX(t *testing.T) { t.Skip("io_uring unavailable here") }`, 1}, + {"raw Skipf fails", head + `func TestX(t *testing.T) { t.Skipf("adaptive engine not available: %v", 1) }`, 1}, + {"raw SkipNow fails", head + `func TestX(t *testing.T) { t.SkipNow() }`, 1}, + {"a raw skip in a subtest closure fails", head + `func TestX(t *testing.T) { t.Run("a", func(t *testing.T) { t.Skip("io_uring unavailable") }) }`, 1}, + {"the gate passes", head + `func TestX(t *testing.T) { skipOrFailIfStandbyAborted886(t, 0, nil) }`, 0}, + {"a skip inside the gate helper passes", head + `func skipOrFailIfStandbyAborted886(t *testing.T) { t.Skip("x") }`, 0}, + {"a -short gate passes", head + `func TestX(t *testing.T) { if testing.Short() { t.Skip("integration") } }`, 0}, + {"a -short gate with another clause passes", head + `func TestX(t *testing.T) { if x && testing.Short() { t.Skip("integration") } }`, 0}, + {"a raw skip after a -short gate fails", head + `func TestX(t *testing.T) { if testing.Short() { t.Skip("integration") }; t.Skip("io_uring unavailable") }`, 1}, + {"a raw skip in the else of a -short gate fails", head + `func TestX(t *testing.T) { if testing.Short() { } else { t.Skip("io_uring unavailable") } }`, 1}, + {"an || with testing.Short() still skips when not short and fails", head + `func TestX(t *testing.T) { if testing.Short() || !avail() { t.Skip("io_uring unavailable") } }`, 1}, + {"a negated testing.Short() gate fails", head + `func TestX(t *testing.T) { if !testing.Short() { t.Skip("io_uring unavailable") } }`, 1}, + {"a skip through a receiver named otherwise fails", head + `func TestX(t *testing.T) { t.Run("a", func(st *testing.T) { st.Skip("io_uring unavailable") }) }`, 1}, + {"a Skip method value fails", head + `func TestX(t *testing.T) { s := t.Skip; s("io_uring unavailable") }`, 1}, + {"a SkipNow method value passed as an argument fails", head + `func TestX(t *testing.T) { run(t.SkipNow) }`, 1}, + {"a Skip method value under a -short gate passes", head + `func TestX(t *testing.T) { if testing.Short() { s := t.Skip; s("integration") } }`, 0}, + {"a bodyless function declaration is not a violation or a panic", head + `func foo()`, 0}, + {"input that does not parse is an error, not a pass", head + `func TestX(t *testing.T) { t.Skip(`, -1}, + {"empty input is an error, not a pass", ``, -1}, + } { + t.Run(c.name, func(t *testing.T) { + v, err := rawSkipViolations890("fixture_test.go", []byte(c.src)) + if c.want == -1 { + if err == nil { + t.Fatalf("a source that does not parse gave no error (violations %v)", v) + } + return + } + if err != nil { + t.Fatal(err) + } + if len(v) != c.want { + t.Fatalf("violations %v, want %d", v, c.want) + } + }) + } + + // The real gate file with the regression this guard is for: the gate's own + // skip, copied out of the helper into a test body, must be listed, and the + // file as it is must not. + const file = "adaptive_standby_guard_886_linux_test.go" + src, err := os.ReadFile(file) + if err != nil { + t.Fatal(err) + } + if v, err := rawSkipViolations890(file, src); err != nil || len(v) != 0 { + t.Fatalf("%s as it is: violations %v, err %v, want none", file, v, err) + } + const gate = "\tt.Skip(msg)\n}" + if strings.Count(string(src), gate) != 1 { + t.Fatalf("%s no longer ends the helper with %q: update the control", file, gate) + } + mutant := strings.Replace(string(src), gate, "\tt.Skip(msg)\n}\n\nfunc TestPlanted890(t *testing.T) { t.Skip(\"io_uring unavailable\") }", 1) + v, err := rawSkipViolations890(file, []byte(mutant)) + if err != nil { + t.Fatal(err) + } + if len(v) != 1 { + t.Fatalf("the planted raw skip was listed %d times (%v), want 1", len(v), v) + } +} From e0379d271e4a0818da15b14c29553edb0a3e21f4 Mon Sep 17 00:00:00 2001 From: Albert Bausili Date: Mon, 26 Oct 2026 23:40:39 +0100 Subject: [PATCH 02/14] ci: CodSpeed runs on a pull request only with the performance label, and on main once a day (Refs #890) Items 7 and 97 of #890. The workflow's job minutes passed a 600-minute monthly count on 2026-10-12 and no job has had a macro runner since; the header now carries the numbers. A pull request runs the benchmarks while it carries the `performance` label (the run that adding any other label starts has a concurrency group of its own, so it cannot cancel the one in flight); main is measured by a daily schedule instead of every push, and a scheduled run replaces an older one still waiting for a runner. Adds a temporary hosted twin of the trigger, group and condition (l18-gate-proof.yml, label `spike` in place of `performance`) so that the gate can be shown on GitHub without a macro-runner minute. It is deleted in a later commit. --- .github/workflows/codspeed.yml | 84 +++++++++++++++++++++------- .github/workflows/l18-gate-proof.yml | 41 ++++++++++++++ 2 files changed, 105 insertions(+), 20 deletions(-) create mode 100644 .github/workflows/l18-gate-proof.yml diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index dfd47edbd..87d48e2a1 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -149,8 +149,40 @@ # 189 minutes rounded up per job (celeris 21 jobs and 101 minutes, loadgen 22 # and 88), two thirds of it pull request runs. At that rate the 600 minutes # last 3.6 to 4.1 days (rounded or raw job time; CodSpeed's own meter is not -# public), and pushes to main alone would use them in about two weeks. Whether to gate pull request runs on the `performance` label, -# or more, is an open decision (celeris#754). +# public), and pushes to main alone would use them in about two weeks. +# +# What happened (celeris#890 items 7 and 97). The 163 jobs of this workflow +# that got a macro runner between 2026-09-26 and 2026-10-12 add up to 880 +# minutes rounded up per job (786 raw); 644 of them fall in October, and the +# count passes 600 at 13:45Z on 2026-10-12. The last job ran at 20:37Z that +# day (run 38305334372). None of the 76 jobs created after it got a runner: +# 35 were cancelled within 3 hours (replaced by the next run of their group), +# 41 after waiting the 24 hours GitHub allows a queued job, and 7 runs are +# still queued as this is written. That fits an exhausted monthly quota (the +# meter here is a count of job minutes, CodSpeed's own is not public, and the +# organization's settings page in CodSpeed holds the real figure: a maintainer +# reads it there). It is also why the main baseline went stale: a push to main +# queued behind the one before it, and a group keeps one pending run. +# +# So, since celeris#890 item 7: +# - a pull request runs the benchmarks only while it carries the +# `performance` label (CONTRIBUTING.md lists it among the release-note +# labels, and a benchmark-path change that is labelled for the notes is +# the one to measure). Adding the label starts a run, every push to a +# labelled pull request starts another, and a draft waits until it is +# marked ready, as before. A pull request without the label costs no +# macro-runner minute: its run exists and its job is skipped, which is the +# result a reviewer sees; +# - main is measured once a day (the schedule below) and no longer on every +# push, which makes the baseline a day old at most. A job took 4.8 minutes +# on average (786 raw minutes over 163 jobs), so that is about 144 minutes +# a month, against about 440 for the 92 pushes to main of the 30 days to +# 2026-10-26 (all of them started a run); +# - a scheduled run replaces an older one that is still waiting for a +# runner, so a stall cannot pile up a queue behind it. +# The label gate in loadgen's workflow is the other half of the decision and +# is not in this repository. +# # The trigger paths below are the non-test code of every package the # benchmark set executes (measured: the set run once with -coverpkg over the # whole module, on Linux and macOS), the benchmark files, the root @@ -165,10 +197,18 @@ name: CodSpeed on: - push: + # Main, once a day (celeris#890 item 7). A schedule has no path filter, so + # it runs whether or not the benchmark paths changed; it replaces the push + # trigger, which started a run for each push that touched them. + schedule: + - cron: "17 3 * * *" + pull_request: branches: [main] - # One list for both events: the pull_request trigger below aliases it. - paths: &benchmark-paths + # `labeled` starts the run when `performance` is added; the job's `if:` + # below skips every other label. `unlabeled` is not listed: taking the + # label off does not stop a run in flight and starts none. + types: [opened, synchronize, reopened, ready_for_review, labeled] + paths: - "*.go" - "celeristest/**" - "internal/ctxkit/**" @@ -207,10 +247,6 @@ on: - "go.mod" - "go.sum" - ".github/workflows/codspeed.yml" - pull_request: - branches: [main] - types: [opened, synchronize, reopened, ready_for_review] - paths: *benchmark-paths # Backtests: CodSpeed dispatches this to measure older commits, and it is # the manual way to take a fresh baseline or to run a draft pull request's # branch (gh workflow run codspeed.yml --ref ). @@ -220,25 +256,33 @@ permissions: contents: read # One run per pull request: a new push cancels the one in flight, so a -# burst of pushes costs one run. Pushes to main are never cancelled -- each -# is a baseline -- but a burst still collapses to the newest pending one. -# Every dispatch gets a group of its own, so the backtests CodSpeed -# dispatches together never cancel one another. +# burst of pushes costs one run. A scheduled run cancels an older scheduled +# run that is still waiting for a runner (a stalled queue must not hold the +# baseline back: celeris#890 item 97). Every dispatch gets a group of its +# own, so the backtests CodSpeed dispatches together never cancel one +# another. So does the run that adding any label other than `performance` +# starts: its job is skipped, and without a group of its own it would cancel +# the benchmark run that the label `performance` started a minute before. concurrency: - group: codspeed-${{ github.event.pull_request.number || (github.event_name == 'workflow_dispatch' && github.run_id) || github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + group: codspeed-${{ (github.event.action == 'labeled' && github.event.label.name != 'performance' && github.run_id) || github.event.pull_request.number || (github.event_name == 'workflow_dispatch' && github.run_id) || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || github.event_name == 'schedule' }} jobs: benchmarks: name: Benchmarks (arm64 macro runner) - # Skips fork pull requests (as this file is written; the approval policy - # in the header is what holds against a fork that edits it) and drafts, - # which wait until they are marked ready (ready_for_review above), to - # save minutes. + # A pull request runs only with the `performance` label, and the label's + # own event (celeris#890 item 7): the run that adding another label starts + # skips the job. Fork pull requests are skipped as this file is written + # (the approval policy in the header is what holds against a fork that + # edits it), and so are drafts, which wait until they are marked ready + # (ready_for_review above), to save minutes. Every other event (the + # schedule, a dispatch) runs. if: >- github.event_name != 'pull_request' || (github.event.pull_request.head.repo.full_name == github.repository && - !github.event.pull_request.draft) + !github.event.pull_request.draft && + contains(github.event.pull_request.labels.*.name, 'performance') && + (github.event.action != 'labeled' || github.event.label.name == 'performance')) runs-on: codspeed-macro-arm64-graviton-ubuntu-22-04 # A hang must not burn the month's minutes. A full run is a few minutes. timeout-minutes: 20 diff --git a/.github/workflows/l18-gate-proof.yml b/.github/workflows/l18-gate-proof.yml new file mode 100644 index 000000000..5366331c5 --- /dev/null +++ b/.github/workflows/l18-gate-proof.yml @@ -0,0 +1,41 @@ +# TEMPORARY (celeris#890 item 7, lane L18). A hosted twin of codspeed.yml's +# trigger, concurrency group and job condition, with the label `spike` in place +# of `performance` and no draft clause, so that the label gate can be shown on +# GitHub without a macro-runner minute. Deleted before the pull request is +# marked ready. +name: L18 gate proof + +on: + pull_request: + branches: [main] + types: [opened, synchronize, reopened, ready_for_review, labeled] + paths: + - ".github/workflows/l18-gate-proof.yml" + +permissions: + contents: read + +concurrency: + group: l18proof-${{ (github.event.action == 'labeled' && github.event.label.name != 'spike' && github.run_id) || github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + benchmarks: + name: Gate twin + if: >- + github.event_name != 'pull_request' || + (github.event.pull_request.head.repo.full_name == github.repository && + contains(github.event.pull_request.labels.*.name, 'spike') && + (github.event.action != 'labeled' || github.event.label.name == 'spike')) + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Hold the slot + env: + ACTION: ${{ github.event.action }} + LABEL: ${{ github.event.label.name }} + SHA: ${{ github.event.pull_request.head.sha }} + run: | + echo "twin ran: action=$ACTION label=$LABEL head=$SHA" + sleep 90 + echo "twin finished" From b90001a23028bbc95b06734d91bbc8cfb0a8bf95 Mon Sep 17 00:00:00 2001 From: Albert Bausili Date: Mon, 26 Oct 2026 23:40:39 +0100 Subject: [PATCH 03/14] ci, test: io_uring base-tier job, full-size large-response job, probe retry logs, epoll arms fail, BenchmarkCloseChurn warning (Refs #890 #885) - 890.87: job iouring-base-tier runs ./internal/engine/iouring and the root package's Shutdown/Detach/SSE/Stream/WebSocket/Hijack/Drain tests with CELERIS_MAX_IOURING_TIER=base, asserts the cap took effect and tallies skips against an allow-list (empty in this commit: the first run lists them). - 890.92: job large-response builds ./test/server without -race and coverage and runs the eight tests that read lean761(), n836() or raceOn761 by name, printing wall time and peak RSS. - 890.88: the six keptProbeIOUring copies take a testing.TB and log one line per retry round, as the c714ProbeIOUring copies do. - 890.95: the epoll arms of pipeline_test.go and slowloris_synthetic_test.go fail where they skipped when epoll.New fails. - 885.21: the warning about the outlier samples on BenchmarkCloseChurn. --- .github/scripts/base-tier-tally.sh | 82 +++++++++++++ .github/workflows/ci.yml | 109 ++++++++++++++++++ .../epoll/conns_slot_bench_linux_test.go | 7 ++ .../adapters/retained_strings_linux_test.go | 12 +- .../logger/retained_attrs_linux_test.go | 12 +- .../metrics/retained_labels_linux_test.go | 12 +- middleware/otel/retained_attrs_linux_test.go | 12 +- .../recovery/retained_attrs_linux_test.go | 12 +- test/integration/pipeline_test.go | 4 +- test/integration/slowloris_synthetic_test.go | 24 ++-- test/server/adapt_request_linux_test.go | 12 +- .../server_async_goexit_791_linux_test.go | 2 +- .../server_sync_goexit_848_linux_test.go | 2 +- 13 files changed, 267 insertions(+), 35 deletions(-) create mode 100755 .github/scripts/base-tier-tally.sh diff --git a/.github/scripts/base-tier-tally.sh b/.github/scripts/base-tier-tally.sh new file mode 100755 index 000000000..cdce8c222 --- /dev/null +++ b/.github/scripts/base-tier-tally.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +# base-tier-tally.sh LABEL LOG [NAME...]: the tally of a `go test -v` log run +# with CELERIS_MAX_IOURING_TIER=base (celeris#890 item 87). +# +# LABEL is "iouring" (the log of ./internal/engine/iouring) or "root" (the log of +# the root package and ./test/server). Three checks, each failing the step: +# +# 1. No test skips except the ones the allow-list of that label names, each +# with the reason it may skip at the base tier. A SKIP line is matched by +# the test's full name, a subtest included (`--- SKIP: Test/sub (`), and +# an entry covers its own subtests. A skip is not coverage; a test that +# needs the high tier says so here. +# 2. Every NAME is a top-level test that ran once and PASSED, with no SKIP or +# FAIL line (named-tally.sh). +# 3. For "iouring": the cap took effect. TestReportTier844 printed the capped +# probe once as io_uring_tier=base with the variable set to "base", and no +# engine arm of the celeris#844 tests reported tier=high. A value the +# probe does not know counts as "none", which would skip every io_uring +# test and leave check 1 as the only witness; this one names the cause. +# +# Prints one summary line on stdout. Exit status 0 all hold, 1 some do not, +# 2 usage or an unreadable log. +set -euo pipefail + +[ "$#" -ge 2 ] || { echo "usage: base-tier-tally.sh LABEL LOG [NAME...]" >&2; exit 2; } +label=$1 +log=$2 +shift 2 +[ -r "$log" ] || { echo "base-tier-tally: cannot read $log" >&2; exit 2; } +here=$(cd "$(dirname "$0")" && pwd) + +# Allowed skips: an alternation of full test names, one entry per line with the +# reason after the name. Measured on the runs cited in the pull request. +allowed_iouring() { + cat <<'LIST' +LIST +} +allowed_root() { + cat <<'LIST' +LIST +} + +case "$label" in + iouring) list=$(allowed_iouring) ;; + root) list=$(allowed_root) ;; + *) echo "base-tier-tally: label '$label' is neither iouring nor root" >&2; exit 2 ;; +esac +names=$(printf '%s\n' "$list" | awk 'NF { print $1 }' | paste -sd'|' -) + +fail=0 +skipped=$("$here"/skip-count.sh "$log") || fail=1 +if [ -n "$names" ]; then + unlisted=$(grep -aE '^[[:space:]]*--- SKIP: ' "$log" | grep -avE "^[[:space:]]*--- SKIP: (${names})( |/)" || true) +else + unlisted=$(grep -aE '^[[:space:]]*--- SKIP: ' "$log" || true) +fi +n_unlisted=0 +[ -n "$unlisted" ] && n_unlisted=$(printf '%s\n' "$unlisted" | wc -l | tr -d '[:space:]') + +tiers= +if [ "$label" = iouring ]; then + probe=$(grep -acE 'celeris844 PROBE .* io_uring_tier=base CELERIS_MAX_IOURING_TIER="base"$' "$log" || true) + high=$(grep -acE 'celeris844 RESULT engine=io_uring tier=high ' "$log" || true) + base=$(grep -acE 'celeris844 RESULT engine=io_uring tier=base ' "$log" || true) + tiers=", capped probe lines $probe (want 1), engine arms at tier=base $base (want > 0) and tier=high $high (want 0)" + if [ "$probe" -ne 1 ] || [ "$high" -ne 0 ] || [ "$base" -eq 0 ]; then + fail=1 + echo "base-tier-tally: the cap did not take effect: CELERIS_MAX_IOURING_TIER=base is not the tier the io_uring arms ran on" >&2 + fi +fi + +echo "base-tier-tally $label: SKIP lines $skipped, not in the allow-list $n_unlisted$tiers" +if [ "$n_unlisted" -ne 0 ]; then + fail=1 + printf '%s\n' "$unlisted" >&2 + echo "base-tier-tally: tests skipped at the base tier that the allow-list of '$label' does not name;" >&2 + echo "add a name only with the reason it must skip at this tier (a skip is not coverage)" >&2 +fi +if [ "$#" -gt 0 ]; then + "$here"/named-tally.sh "base tier $label" "$log" "$@" || fail=1 +fi +exit "$fail" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 86a447d3d..c08682e33 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2336,6 +2336,115 @@ jobs: if [ "$skipped" -ne 0 ]; then echo "a SKIP is not a result ($skipped SKIP lines)"; bad=1; fi exit "$bad" + # NOT a required check (see the note above `iouring`). celeris#890 item 87: + # nothing ran the io_uring base tier. CELERIS_MAX_IOURING_TIER=base caps the + # probe's tier at "base" (no provided buffer ring, single-shot accept, no + # multishot recv), the tier a kernel without those features gets and the one + # PR #1026 made non-blocking on accept (item 53, celeris#884): a partial + # inline write of a detached stream leaves a remainder in writeBuf there + # (TestShutdownDetachedPartialWriteIsFlushed806), a state the other tiers do + # not reach the same way. Until this job only local runs covered it. + # + # Two steps, each with memlock raised (the tests that ask for several workers + # need it) and the cap in the environment: + # * ./internal/engine/iouring under -race. The cap must have taken effect: + # TestReportTier844 must print io_uring_tier=base for the capped probe + # and no engine arm may report tier=high (a misspelled value counts as + # "none" and would skip everything, which the tally also refuses); + # * the root package and ./test/server, the tests whose names say Shutdown, + # Detach, SSE, Stream, WebSocket, Hijack or Drain, under -race. + # Tests that need the high tier skip at this tier on purpose; each skip that + # is allowed is listed by name below with its reason, and any other SKIP line + # fails the step. + iouring-base-tier: + name: io_uring base tier (CELERIS_MAX_IOURING_TIER=base, celeris#890) + runs-on: ubuntu-latest + timeout-minutes: 45 + env: + CELERIS_MAX_IOURING_TIER: base + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version: "1.27.3" + - name: internal/engine/iouring at the base tier (race, tier asserted, skips accounted for) + shell: bash + run: | + set -o pipefail + sudo prlimit --pid "$$" --memlock=unlimited:unlimited + echo "memlock (KiB): $(ulimit -l), CELERIS_MAX_IOURING_TIER=$CELERIS_MAX_IOURING_TIER" + start=$SECONDS + CELERIS_REQUIRE_DUALSTACK=1 go test -race -count=1 -timeout=1500s -v ./internal/engine/iouring/ 2>&1 | tee /tmp/base-iouring.log + echo "internal/engine/iouring at the base tier took $((SECONDS - start)) s" + .github/scripts/base-tier-tally.sh iouring /tmp/base-iouring.log TestShutdownDetachedPartialWriteIsFlushed806 TestReportTier844 + - name: Root package and test/server, Shutdown/Detach/SSE/Stream/WebSocket/Hijack/Drain tests at the base tier (race) + if: ${{ !cancelled() }} + shell: bash + run: | + set -o pipefail + sudo prlimit --pid "$$" --memlock=unlimited:unlimited + echo "memlock (KiB): $(ulimit -l), CELERIS_MAX_IOURING_TIER=$CELERIS_MAX_IOURING_TIER" + start=$SECONDS + # -p 1: one test process at a time, so the runner's one memlock budget never funds two. + CELERIS_REQUIRE_DUALSTACK=1 go test -p 1 -race -count=1 -timeout=1500s -v -run 'Shutdown|Detach|SSE|Stream|WebSocket|Hijack|Drain' . ./test/server 2>&1 | tee /tmp/base-root.log + echo "root and test/server at the base tier took $((SECONDS - start)) s" + .github/scripts/base-tier-tally.sh root /tmp/base-root.log + + # NOT a required check. celeris#890 item 92: the large-response tests of + # ./test/server (celeris#761, #817, #836) have a lean shape and a full-size + # one. lean761() (large_response_linux_test.go) is true under -race and + # under coverage, which are the only ways the other jobs run ./test/server, + # so their 16 MiB / 64 MiB bodies, their request counts and the h2c + # AsyncHandlers arm of TestH2CUnmatchedThroughTimeoutSingleflight836 + # (AsyncHandlers: !raceOn761) never ran in CI. This job builds the test binary + # without -race and without coverage (race761_off_test.go makes raceOn761 + # false) and runs the eight tests that read lean761(), n836() or raceOn761, + # by name, with memlock raised (they start io_uring engines), -v, and a tally + # by name: each must run once and PASS, with no SKIP line. The script prints + # the wall time and the peak resident set of the test process, so a log says + # what the full-size shape costs on a runner. + large-response: + name: Large responses at full size, no race (./test/server, celeris#890) + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version: "1.27.3" + - name: test/server large-response tests at full size (no race, no coverage) + shell: bash + run: | + set -o pipefail + sudo prlimit --pid "$$" --memlock=unlimited:unlimited + echo "memlock (KiB): $(ulimit -l), CPUs: $(nproc), memory: $(free -m | awk '/^Mem:/ {print $2}') MiB" + tests=( + TestLargeResponseIsDelivered + TestLargeFileResponseIsDelivered + TestLargeResponseIsDeliveredH2 + TestConcurrentResponsesOwnTheirBodies + TestH2DerivedContextDoesNotPanic836 + TestH2DetachedContextIsItsRequests836 + TestH2InlineCleanupDoesNotReleaseAReusedStream947 + TestH2CUnmatchedThroughTimeoutSingleflight836 + ) + regex=$(IFS='|'; echo "${tests[*]}") + go test -c -o /tmp/server.test ./test/server + python3 - /tmp/server.test "^(${regex})\$" <<'PY' 2>&1 | tee /tmp/large.log + import resource, subprocess, sys, time + binary, run = sys.argv[1], sys.argv[2] + t0 = time.monotonic() + rc = subprocess.run([binary, "-test.run", run, "-test.v", "-test.count=1", "-test.timeout=1500s"]).returncode + rss = resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss # KiB on Linux + print("full-size shape: wall %.1f s, peak RSS of the test process %.0f MiB, exit %d" % (time.monotonic() - t0, rss / 1024, rc)) + sys.exit(rc) + PY + .github/scripts/named-tally.sh 'test/server full-size shape (celeris#890 item 92)' /tmp/large.log "${tests[@]}" + conformance: name: Conformance runs-on: ubuntu-latest diff --git a/internal/engine/epoll/conns_slot_bench_linux_test.go b/internal/engine/epoll/conns_slot_bench_linux_test.go index 1163fe6b9..d73b9ac6f 100644 --- a/internal/engine/epoll/conns_slot_bench_linux_test.go +++ b/internal/engine/epoll/conns_slot_bench_linux_test.go @@ -38,6 +38,13 @@ func BenchmarkCloseConnSlotClear(b *testing.B) { // has counted every close. The kernel's connect and close dominate each op, so // the figure is read against an A/A floor; BenchmarkCloseConnSlotClear // isolates the lock itself. +// +// Warning: this instrument has extreme outlier samples. One A/A pair (two runs +// of the same code) differed by up to about 9x in the review of the change that +// added it (celeris#960, tracked in celeris#885), a spread the median floor of +// 0.032 does not show. Do not read one pair, or one run: compare medians over +// many interleaved rounds (.github/scripts/bench-ab.sh) and report the spread +// beside the median. The 9x figure is that review's, not measured here. func BenchmarkCloseChurn(b *testing.B) { eng, stop := newTestEngine(b) defer stop() diff --git a/middleware/adapters/retained_strings_linux_test.go b/middleware/adapters/retained_strings_linux_test.go index 0cde78f5f..a375fc77b 100644 --- a/middleware/adapters/retained_strings_linux_test.go +++ b/middleware/adapters/retained_strings_linux_test.go @@ -132,7 +132,7 @@ func keptArms(t *testing.T) []keptArm { {"epoll", celeris.Epoll, false}, {"epoll-async", celeris.Epoll, true}, } - if ok, p := keptProbeIOUring(); ok { + if ok, p := keptProbeIOUring(t); ok { arms = append(arms, keptArm{"io_uring", celeris.IOUring, false}, keptArm{"io_uring-async", celeris.IOUring, true}) } else if os.Getenv("CELERIS_REQUIRE_IOURING_WORKERS") == "1" { t.Fatalf("io_uring tier=%s kernel=%s, and CELERIS_REQUIRE_IOURING_WORKERS=1 forbids dropping the io_uring arms", p.IOUringTier, p.KernelVersion) @@ -144,17 +144,21 @@ func keptArms(t *testing.T) []keptArm { // keptProbeIOUring probes the kernel's io_uring support. With // CELERIS_REQUIRE_IOURING_WORKERS=1 a probe that finds no usable ring is -// retried for up to 10 s: the probe's ring can fail with ENOMEM against +// retried for up to 10 s, logging each round through tb: the probe's ring can fail with ENOMEM against // RLIMIT_MEMLOCK while the rings of engines stopped moments ago, or of // another test binary of the same user, are still charged // (internal/engine/iouring/ring_budget_linux_test.go). -func keptProbeIOUring() (usable bool, p celerisengine.CapabilityProfile) { +func keptProbeIOUring(tb testing.TB) (usable bool, p celerisengine.CapabilityProfile) { p = probe.Probe() usable = p.IOUringTier >= celerisengine.High && p.ProvidedBuffers if os.Getenv("CELERIS_REQUIRE_IOURING_WORKERS") != "1" { return usable, p } - for deadline := time.Now().Add(10 * time.Second); !usable && time.Now().Before(deadline); { + deadline := time.Now().Add(10 * time.Second) + for round := 1; !usable && time.Now().Before(deadline); round++ { + // One line per round, so a CI log shows that the probe fired and how long it + // kept failing. Rounds are 10 ms apart, so this is at most about 1000 lines. + tb.Logf("keptProbeIOUring: retry round %d, no usable io_uring ring yet (tier %v, provided buffers %t)", round, p.IOUringTier, p.ProvidedBuffers) time.Sleep(10 * time.Millisecond) p = probe.Probe() usable = p.IOUringTier >= celerisengine.High && p.ProvidedBuffers diff --git a/middleware/logger/retained_attrs_linux_test.go b/middleware/logger/retained_attrs_linux_test.go index c346d997c..cbf9d8d93 100644 --- a/middleware/logger/retained_attrs_linux_test.go +++ b/middleware/logger/retained_attrs_linux_test.go @@ -171,7 +171,7 @@ func keptArms(t *testing.T) []keptArm { {"epoll", celeris.Epoll, false}, {"epoll-async", celeris.Epoll, true}, } - if ok, p := keptProbeIOUring(); ok { + if ok, p := keptProbeIOUring(t); ok { arms = append(arms, keptArm{"io_uring", celeris.IOUring, false}, keptArm{"io_uring-async", celeris.IOUring, true}) } else if os.Getenv("CELERIS_REQUIRE_IOURING_WORKERS") == "1" { t.Fatalf("io_uring tier=%s kernel=%s, and CELERIS_REQUIRE_IOURING_WORKERS=1 forbids dropping the io_uring arms", p.IOUringTier, p.KernelVersion) @@ -183,17 +183,21 @@ func keptArms(t *testing.T) []keptArm { // keptProbeIOUring probes the kernel's io_uring support. With // CELERIS_REQUIRE_IOURING_WORKERS=1 a probe that finds no usable ring is -// retried for up to 10 s: the probe's ring can fail with ENOMEM against +// retried for up to 10 s, logging each round through tb: the probe's ring can fail with ENOMEM against // RLIMIT_MEMLOCK while the rings of engines stopped moments ago, or of // another test binary of the same user, are still charged // (internal/engine/iouring/ring_budget_linux_test.go). -func keptProbeIOUring() (usable bool, p celerisengine.CapabilityProfile) { +func keptProbeIOUring(tb testing.TB) (usable bool, p celerisengine.CapabilityProfile) { p = probe.Probe() usable = p.IOUringTier >= celerisengine.High && p.ProvidedBuffers if os.Getenv("CELERIS_REQUIRE_IOURING_WORKERS") != "1" { return usable, p } - for deadline := time.Now().Add(10 * time.Second); !usable && time.Now().Before(deadline); { + deadline := time.Now().Add(10 * time.Second) + for round := 1; !usable && time.Now().Before(deadline); round++ { + // One line per round, so a CI log shows that the probe fired and how long it + // kept failing. Rounds are 10 ms apart, so this is at most about 1000 lines. + tb.Logf("keptProbeIOUring: retry round %d, no usable io_uring ring yet (tier %v, provided buffers %t)", round, p.IOUringTier, p.ProvidedBuffers) time.Sleep(10 * time.Millisecond) p = probe.Probe() usable = p.IOUringTier >= celerisengine.High && p.ProvidedBuffers diff --git a/middleware/metrics/retained_labels_linux_test.go b/middleware/metrics/retained_labels_linux_test.go index 381ed37d4..3e4811f0b 100644 --- a/middleware/metrics/retained_labels_linux_test.go +++ b/middleware/metrics/retained_labels_linux_test.go @@ -182,7 +182,7 @@ func keptArms(t *testing.T) []keptArm { {"epoll", celeris.Epoll, false}, {"epoll-async", celeris.Epoll, true}, } - if ok, p := keptProbeIOUring(); ok { + if ok, p := keptProbeIOUring(t); ok { arms = append(arms, keptArm{"io_uring", celeris.IOUring, false}, keptArm{"io_uring-async", celeris.IOUring, true}) } else if os.Getenv("CELERIS_REQUIRE_IOURING_WORKERS") == "1" { t.Fatalf("io_uring tier=%s kernel=%s, and CELERIS_REQUIRE_IOURING_WORKERS=1 forbids dropping the io_uring arms", p.IOUringTier, p.KernelVersion) @@ -194,17 +194,21 @@ func keptArms(t *testing.T) []keptArm { // keptProbeIOUring probes the kernel's io_uring support. With // CELERIS_REQUIRE_IOURING_WORKERS=1 a probe that finds no usable ring is -// retried for up to 10 s: the probe's ring can fail with ENOMEM against +// retried for up to 10 s, logging each round through tb: the probe's ring can fail with ENOMEM against // RLIMIT_MEMLOCK while the rings of engines stopped moments ago, or of // another test binary of the same user, are still charged // (internal/engine/iouring/ring_budget_linux_test.go). -func keptProbeIOUring() (usable bool, p celerisengine.CapabilityProfile) { +func keptProbeIOUring(tb testing.TB) (usable bool, p celerisengine.CapabilityProfile) { p = probe.Probe() usable = p.IOUringTier >= celerisengine.High && p.ProvidedBuffers if os.Getenv("CELERIS_REQUIRE_IOURING_WORKERS") != "1" { return usable, p } - for deadline := time.Now().Add(10 * time.Second); !usable && time.Now().Before(deadline); { + deadline := time.Now().Add(10 * time.Second) + for round := 1; !usable && time.Now().Before(deadline); round++ { + // One line per round, so a CI log shows that the probe fired and how long it + // kept failing. Rounds are 10 ms apart, so this is at most about 1000 lines. + tb.Logf("keptProbeIOUring: retry round %d, no usable io_uring ring yet (tier %v, provided buffers %t)", round, p.IOUringTier, p.ProvidedBuffers) time.Sleep(10 * time.Millisecond) p = probe.Probe() usable = p.IOUringTier >= celerisengine.High && p.ProvidedBuffers diff --git a/middleware/otel/retained_attrs_linux_test.go b/middleware/otel/retained_attrs_linux_test.go index d8bab5409..a8875df5f 100644 --- a/middleware/otel/retained_attrs_linux_test.go +++ b/middleware/otel/retained_attrs_linux_test.go @@ -187,7 +187,7 @@ func keptArms(t *testing.T) []keptArm { {"epoll", celeris.Epoll, false}, {"epoll-async", celeris.Epoll, true}, } - if ok, p := keptProbeIOUring(); ok { + if ok, p := keptProbeIOUring(t); ok { arms = append(arms, keptArm{"io_uring", celeris.IOUring, false}, keptArm{"io_uring-async", celeris.IOUring, true}) } else if os.Getenv("CELERIS_REQUIRE_IOURING_WORKERS") == "1" { t.Fatalf("io_uring tier=%s kernel=%s, and CELERIS_REQUIRE_IOURING_WORKERS=1 forbids dropping the io_uring arms", p.IOUringTier, p.KernelVersion) @@ -199,17 +199,21 @@ func keptArms(t *testing.T) []keptArm { // keptProbeIOUring probes the kernel's io_uring support. With // CELERIS_REQUIRE_IOURING_WORKERS=1 a probe that finds no usable ring is -// retried for up to 10 s: the probe's ring can fail with ENOMEM against +// retried for up to 10 s, logging each round through tb: the probe's ring can fail with ENOMEM against // RLIMIT_MEMLOCK while the rings of engines stopped moments ago, or of // another test binary of the same user, are still charged // (internal/engine/iouring/ring_budget_linux_test.go). -func keptProbeIOUring() (usable bool, p celerisengine.CapabilityProfile) { +func keptProbeIOUring(tb testing.TB) (usable bool, p celerisengine.CapabilityProfile) { p = probe.Probe() usable = p.IOUringTier >= celerisengine.High && p.ProvidedBuffers if os.Getenv("CELERIS_REQUIRE_IOURING_WORKERS") != "1" { return usable, p } - for deadline := time.Now().Add(10 * time.Second); !usable && time.Now().Before(deadline); { + deadline := time.Now().Add(10 * time.Second) + for round := 1; !usable && time.Now().Before(deadline); round++ { + // One line per round, so a CI log shows that the probe fired and how long it + // kept failing. Rounds are 10 ms apart, so this is at most about 1000 lines. + tb.Logf("keptProbeIOUring: retry round %d, no usable io_uring ring yet (tier %v, provided buffers %t)", round, p.IOUringTier, p.ProvidedBuffers) time.Sleep(10 * time.Millisecond) p = probe.Probe() usable = p.IOUringTier >= celerisengine.High && p.ProvidedBuffers diff --git a/middleware/recovery/retained_attrs_linux_test.go b/middleware/recovery/retained_attrs_linux_test.go index 1a487da70..188e41023 100644 --- a/middleware/recovery/retained_attrs_linux_test.go +++ b/middleware/recovery/retained_attrs_linux_test.go @@ -207,7 +207,7 @@ func mwArms(t *testing.T) []keptArm { {"epoll", celeris.Epoll, false}, {"epoll-async", celeris.Epoll, true}, } - if ok, p := keptProbeIOUring(); ok { + if ok, p := keptProbeIOUring(t); ok { arms = append(arms, keptArm{"io_uring", celeris.IOUring, false}, keptArm{"io_uring-async", celeris.IOUring, true}) } else if os.Getenv("CELERIS_REQUIRE_IOURING_WORKERS") == "1" { t.Fatalf("io_uring tier=%s kernel=%s, and CELERIS_REQUIRE_IOURING_WORKERS=1 forbids dropping the io_uring arms", p.IOUringTier, p.KernelVersion) @@ -219,17 +219,21 @@ func mwArms(t *testing.T) []keptArm { // keptProbeIOUring probes the kernel's io_uring support. With // CELERIS_REQUIRE_IOURING_WORKERS=1 a probe that finds no usable ring is -// retried for up to 10 s: the probe's ring can fail with ENOMEM against +// retried for up to 10 s, logging each round through tb: the probe's ring can fail with ENOMEM against // RLIMIT_MEMLOCK while the rings of engines stopped moments ago, or of // another test binary of the same user, are still charged // (internal/engine/iouring/ring_budget_linux_test.go). -func keptProbeIOUring() (usable bool, p celerisengine.CapabilityProfile) { +func keptProbeIOUring(tb testing.TB) (usable bool, p celerisengine.CapabilityProfile) { p = probe.Probe() usable = p.IOUringTier >= celerisengine.High && p.ProvidedBuffers if os.Getenv("CELERIS_REQUIRE_IOURING_WORKERS") != "1" { return usable, p } - for deadline := time.Now().Add(10 * time.Second); !usable && time.Now().Before(deadline); { + deadline := time.Now().Add(10 * time.Second) + for round := 1; !usable && time.Now().Before(deadline); round++ { + // One line per round, so a CI log shows that the probe fired and how long it + // kept failing. Rounds are 10 ms apart, so this is at most about 1000 lines. + tb.Logf("keptProbeIOUring: retry round %d, no usable io_uring ring yet (tier %v, provided buffers %t)", round, p.IOUringTier, p.ProvidedBuffers) time.Sleep(10 * time.Millisecond) p = probe.Probe() usable = p.IOUringTier >= celerisengine.High && p.ProvidedBuffers diff --git a/test/integration/pipeline_test.go b/test/integration/pipeline_test.go index 05e50037b..23a088adf 100644 --- a/test/integration/pipeline_test.go +++ b/test/integration/pipeline_test.go @@ -37,7 +37,9 @@ func TestHTTP1PipeliningAsync(t *testing.T) { cfg.AsyncHandlers = tc.async eng, err := epoll.New(cfg, &echoHandler{}) if err != nil { - t.Skipf("epoll engine unavailable: %v", err) + // epoll.New fails on config validation only, so a failure here is a + // defect, not a host without the engine (celeris#890 item 95). + t.Fatalf("epoll.New: %v", err) } startEngine(t, eng) diff --git a/test/integration/slowloris_synthetic_test.go b/test/integration/slowloris_synthetic_test.go index b8e6a2ce1..80e75bd9c 100644 --- a/test/integration/slowloris_synthetic_test.go +++ b/test/integration/slowloris_synthetic_test.go @@ -14,6 +14,18 @@ import ( "github.com/goceleris/celeris/internal/resource" ) +// engineNewFailed890 handles a failed engine constructor in a table of engines +// (celeris#890 item 95). epoll.New fails on config validation only, so on the +// epoll arm a failure is a defect and the test fails; the io_uring arm skips, +// as before (io_uring skips are celeris#888's). +func engineNewFailed890(t *testing.T, name string, err error) { + t.Helper() + if name == "epoll" { + t.Fatalf("epoll.New: %v", err) + } + t.Skipf("%s engine unavailable on this host: %v", name, err) +} + // TestSlowlorisSynthetic is the focused per-engine reproducer for the // v1.4.11 slowloris-defence work. Replaces the 1h+ probatorium nightly // cycle with an in-process test that logs every relevant step. @@ -39,9 +51,8 @@ import ( // (FIN-vs-RST issue — RST would force immediate fail). func TestSlowlorisSynthetic(t *testing.T) { for _, tc := range []struct { - name string - newEng func(cfg resource.Config) (engine.Engine, error) - minSkip string // skip message if engine unsupported + name string + newEng func(cfg resource.Config) (engine.Engine, error) }{ { name: "iouring", @@ -68,10 +79,7 @@ func TestSlowlorisSynthetic(t *testing.T) { } eng, err := tc.newEng(cfg) if err != nil { - if tc.minSkip != "" { - t.Skip(tc.minSkip) - } - t.Skipf("%s engine unavailable on this host: %v", tc.name, err) + engineNewFailed890(t, tc.name, err) } startEngine(t, eng) @@ -210,7 +218,7 @@ func TestSlowlorisSyntheticConcurrent(t *testing.T) { } eng, err := tc.newEng(cfg) if err != nil { - t.Skipf("%s engine unavailable: %v", tc.name, err) + engineNewFailed890(t, tc.name, err) } startEngine(t, eng) addr := eng.(addrGetter).Addr().String() diff --git a/test/server/adapt_request_linux_test.go b/test/server/adapt_request_linux_test.go index e0baacf5d..7a313fe3c 100644 --- a/test/server/adapt_request_linux_test.go +++ b/test/server/adapt_request_linux_test.go @@ -174,7 +174,7 @@ func keptArms(t *testing.T) []keptArm { {"epoll", celeris.Epoll, false}, {"epoll-async", celeris.Epoll, true}, } - if ok, p := keptProbeIOUring(); ok { + if ok, p := keptProbeIOUring(t); ok { arms = append(arms, keptArm{"io_uring", celeris.IOUring, false}, keptArm{"io_uring-async", celeris.IOUring, true}) } else if os.Getenv("CELERIS_REQUIRE_IOURING_WORKERS") == "1" { t.Fatalf("io_uring tier=%s kernel=%s, and CELERIS_REQUIRE_IOURING_WORKERS=1 forbids dropping the io_uring arms", p.IOUringTier, p.KernelVersion) @@ -186,17 +186,21 @@ func keptArms(t *testing.T) []keptArm { // keptProbeIOUring probes the kernel's io_uring support. With // CELERIS_REQUIRE_IOURING_WORKERS=1 a probe that finds no usable ring is -// retried for up to 10 s: the probe's ring can fail with ENOMEM against +// retried for up to 10 s, logging each round through tb: the probe's ring can fail with ENOMEM against // RLIMIT_MEMLOCK while the rings of engines stopped moments ago, or of // another test binary of the same user, are still charged // (internal/engine/iouring/ring_budget_linux_test.go). -func keptProbeIOUring() (usable bool, p celerisengine.CapabilityProfile) { +func keptProbeIOUring(tb testing.TB) (usable bool, p celerisengine.CapabilityProfile) { p = probe.Probe() usable = p.IOUringTier >= celerisengine.High && p.ProvidedBuffers if os.Getenv("CELERIS_REQUIRE_IOURING_WORKERS") != "1" { return usable, p } - for deadline := time.Now().Add(10 * time.Second); !usable && time.Now().Before(deadline); { + deadline := time.Now().Add(10 * time.Second) + for round := 1; !usable && time.Now().Before(deadline); round++ { + // One line per round, so a CI log shows that the probe fired and how long it + // kept failing. Rounds are 10 ms apart, so this is at most about 1000 lines. + tb.Logf("keptProbeIOUring: retry round %d, no usable io_uring ring yet (tier %v, provided buffers %t)", round, p.IOUringTier, p.ProvidedBuffers) time.Sleep(10 * time.Millisecond) p = probe.Probe() usable = p.IOUringTier >= celerisengine.High && p.ProvidedBuffers diff --git a/test/server/server_async_goexit_791_linux_test.go b/test/server/server_async_goexit_791_linux_test.go index ff3918fdb..d945ffae3 100644 --- a/test/server/server_async_goexit_791_linux_test.go +++ b/test/server/server_async_goexit_791_linux_test.go @@ -97,7 +97,7 @@ const ( func runServerAsyncAbort791(t *testing.T, eng celeris.EngineType, mode string) { if eng == celeris.IOUring { - if ok, p := keptProbeIOUring(); !ok { + if ok, p := keptProbeIOUring(t); !ok { if os.Getenv("CELERIS_REQUIRE_IOURING_WORKERS") == "1" { t.Fatalf("io_uring tier=%s kernel=%s, and CELERIS_REQUIRE_IOURING_WORKERS=1 forbids skipping", p.IOUringTier, p.KernelVersion) } diff --git a/test/server/server_sync_goexit_848_linux_test.go b/test/server/server_sync_goexit_848_linux_test.go index fbf9d3110..7588349dc 100644 --- a/test/server/server_sync_goexit_848_linux_test.go +++ b/test/server/server_sync_goexit_848_linux_test.go @@ -141,7 +141,7 @@ func need848(t *testing.T, eng celeris.EngineType) { if eng != celeris.IOUring && eng != celeris.Adaptive { return } - if ok, p := keptProbeIOUring(); !ok { + if ok, p := keptProbeIOUring(t); !ok { if os.Getenv("CELERIS_REQUIRE_IOURING_WORKERS") == "1" { t.Fatalf("io_uring tier=%s kernel=%s, and CELERIS_REQUIRE_IOURING_WORKERS=1 forbids skipping", p.IOUringTier, p.KernelVersion) } From 234e18b8f06fa069e5e137eba36b642f9a962c62 Mon Sep 17 00:00:00 2001 From: Albert Bausili Date: Mon, 26 Oct 2026 23:50:07 +0100 Subject: [PATCH 04/14] ci, test: the fixes for the gates of the first commit: postgres budget 14, zizmor pinned in a requirements file, the integration skip registry (Refs #890) - item 91: driver/postgres Query_1col_1row has the budget the driver costs, 14 allocs/op (measured 14 on linux/arm64 in 20 of 20 runs and on the GitHub amd64 runner), and the allocation-guard step stays strict. - item 93: zizmor==1.30.0 moves to .github/tools/requirements.txt, installed by `pip install -r` in a venv; dependabot.yml gets a pip entry for /.github/tools. The Dependabot CLI (v1.96.0, pip updater) proposes zizmor 1.30.0 to 1.31.1 from that file. Check 4 of tool-pins-check.sh passes and so do its fixtures. - item 96: skipOrFailUpswitch890 is the one definition of the gate in test/integration (the standby gate and the io_uring skip of adaptive_standby_886 use it); the ten raw skips of adaptive_hybrid_test.go are registered with their count and why, which item 95 leaves to the adaptive lane. - item 92: the large-response job also asserts that the 64 MiB and 16 MiB bodies ran and the lean ones did not. - item 87: base-tier-tally.sh has a self-test in the Lint job, and its allow-lists are data files. --- .github/base-tier-skips-iouring.txt | 1 + .github/base-tier-skips-root.txt | 1 + .github/dependabot.yml | 20 ++++++ .github/scripts/base-tier-tally-test.sh | 69 +++++++++++++++++++ .github/scripts/base-tier-tally.sh | 22 +++--- .github/tools/requirements.txt | 5 ++ .github/workflows/ci.yml | 50 ++++++++++---- driver/postgres/alloc_guard_test.go | 7 +- .../adaptive_standby_886_linux_test.go | 2 +- .../adaptive_standby_guard_886_linux_test.go | 13 +++- test/integration/skip_routing_890_test.go | 56 +++++++++++---- 11 files changed, 202 insertions(+), 44 deletions(-) create mode 100644 .github/base-tier-skips-iouring.txt create mode 100644 .github/base-tier-skips-root.txt create mode 100755 .github/scripts/base-tier-tally-test.sh create mode 100644 .github/tools/requirements.txt diff --git a/.github/base-tier-skips-iouring.txt b/.github/base-tier-skips-iouring.txt new file mode 100644 index 000000000..0ca8ba3a5 --- /dev/null +++ b/.github/base-tier-skips-iouring.txt @@ -0,0 +1 @@ +# Tests that skip at CELERIS_MAX_IOURING_TIER=base on purpose: the full test name, then why. diff --git a/.github/base-tier-skips-root.txt b/.github/base-tier-skips-root.txt new file mode 100644 index 000000000..0ca8ba3a5 --- /dev/null +++ b/.github/base-tier-skips-root.txt @@ -0,0 +1 @@ +# Tests that skip at CELERIS_MAX_IOURING_TIER=base on purpose: the full test name, then why. diff --git a/.github/dependabot.yml b/.github/dependabot.yml index d5efb6158..03e7b1a2a 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -92,6 +92,26 @@ updates: commit-message: prefix: "deps" + # ----- CI Python tools ----- + # zizmor, the workflow security audit of the Lint job, is the one CI tool + # that is not a `tool` directive of .github/tools/go.mod. Its pin is + # .github/tools/requirements.txt, installed by `pip install -r`. The Lint step + # "Tool pins agree" (.github/scripts/tool-pins-check.sh, check 4) fails when a + # workflow names a `name==version` of its own, and when a requirements file + # under .github/ has no entry like this one. No `allow` rule: the file lists + # direct dependencies only. A bump of zizmor can add or change findings of the + # Lint job; the pull request shows them. + - package-ecosystem: pip + directory: /.github/tools + schedule: + interval: daily + groups: + ci-python-tools: + patterns: + - "*" + commit-message: + prefix: "deps" + # ----- GitHub Actions ----- # Watches every workflow under .github/workflows. Group all action # version bumps into ONE PR per scheduled run. diff --git a/.github/scripts/base-tier-tally-test.sh b/.github/scripts/base-tier-tally-test.sh new file mode 100755 index 000000000..fc1f3fc8a --- /dev/null +++ b/.github/scripts/base-tier-tally-test.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# Self-test of base-tier-tally.sh on fixture logs (celeris#890 item 87). Run from +# the repository root; the Lint job runs it, so a change to the tally that stops +# it failing on a skip nobody allowed, on a cap that did not take effect or on a +# test that is missing is a red Lint. +# +# BASE_TIER_TALLY=path the tally to test (default .github/scripts/base-tier-tally.sh) +set -uo pipefail + +tally=${BASE_TIER_TALLY:-.github/scripts/base-tier-tally.sh} +dir=$(mktemp -d) +trap 'rm -rf "$dir"' EXIT +fail=0 +n=0 +export BASE_TIER_SKIPS_DIR=$dir +printf '# header\nTestNeedsHighTier needs a provided buffer ring\n' >"$dir/base-tier-skips-iouring.txt" +printf '# header\n' >"$dir/base-tier-skips-root.txt" + +# expect NAME WANT_STATUS WANT_STDERR_SUBSTRING LABEL LOG [TESTS...] +expect() { + local name=$1 want=$2 sub=$3 status=0 + shift 3 + "$tally" "$@" >"$dir/out" 2>"$dir/err" || status=$? + n=$((n + 1)) + if [ "$status" -ne "$want" ]; then + echo "FAIL $name: exit $status, want $want"; cat "$dir/out" "$dir/err"; fail=1 + elif [ -n "$sub" ] && ! grep -qF -e "$sub" "$dir/err" "$dir/out"; then + echo "FAIL $name: output does not say '$sub':"; cat "$dir/out" "$dir/err"; fail=1 + else + echo "ok $name" + fi +} + +# good.log: the shape of an iouring log at the base tier. 806 and the tier test passed, +# one allowed test skipped (with a subtest). +good() { + printf '%s\n' \ + '=== RUN TestShutdownDetachedPartialWriteIsFlushed806' '--- PASS: TestShutdownDetachedPartialWriteIsFlushed806 (0.10s)' \ + '=== RUN TestReportTier844' \ + ' async_abort_844_engine_linux_test.go:160: celeris844 PROBE kernel=6.11.0 io_uring_tier=base CELERIS_MAX_IOURING_TIER="base"' \ + '--- PASS: TestReportTier844 (0.00s)' \ + '=== RUN TestNeedsHighTier' '=== RUN TestNeedsHighTier/sub' ' --- SKIP: TestNeedsHighTier/sub (0.00s)' '--- SKIP: TestNeedsHighTier (0.00s)' \ + ' async_abort_844_linux_test.go:518: celeris844 RESULT engine=io_uring tier=base arm=park inject=true' \ + 'PASS' +} +good >"$dir/good.log" +expect "good iouring log" 0 "" iouring "$dir/good.log" TestShutdownDetachedPartialWriteIsFlushed806 TestReportTier844 +sed 's/NeedsHighTier/NeedsSomething/' "$dir/good.log" >"$dir/unlisted.log" +expect "a skip nobody allowed fails" 1 "TestNeedsSomething" iouring "$dir/unlisted.log" TestShutdownDetachedPartialWriteIsFlushed806 +sed 's/tier=base arm=park/tier=high arm=park/' "$dir/good.log" >"$dir/high.log" +expect "an arm at tier=high fails" 1 "the cap did not take effect" iouring "$dir/high.log" TestShutdownDetachedPartialWriteIsFlushed806 +grep -v 'celeris844 PROBE' "$dir/good.log" >"$dir/noprobe.log" +expect "a missing capped probe line fails" 1 "the cap did not take effect" iouring "$dir/noprobe.log" TestShutdownDetachedPartialWriteIsFlushed806 +sed 's/CELERIS_MAX_IOURING_TIER="base"/CELERIS_MAX_IOURING_TIER=""/' "$dir/good.log" >"$dir/nocap.log" +expect "a probe line of an uncapped run fails" 1 "the cap did not take effect" iouring "$dir/nocap.log" TestShutdownDetachedPartialWriteIsFlushed806 +grep -v 'TestShutdownDetachedPartialWriteIsFlushed806' "$dir/good.log" >"$dir/no806.log" +expect "a named test that never ran fails" 1 "TestShutdownDetachedPartialWriteIsFlushed806 ran 0" iouring "$dir/no806.log" TestShutdownDetachedPartialWriteIsFlushed806 +sed 's/^--- PASS: TestShutdownDetachedPartialWriteIsFlushed806/--- SKIP: TestShutdownDetachedPartialWriteIsFlushed806/' "$dir/good.log" >"$dir/skip806.log" +expect "a named test that skipped fails" 1 "TestShutdownDetachedPartialWriteIsFlushed806" iouring "$dir/skip806.log" TestShutdownDetachedPartialWriteIsFlushed806 +# The root label has no tier lines and allows nothing here. +printf '%s\n' '=== RUN TestA' '--- PASS: TestA (0.00s)' 'PASS' >"$dir/root.log" +expect "root: a clean log" 0 "" root "$dir/root.log" +printf '%s\n' '=== RUN TestA' '--- SKIP: TestA (0.00s)' 'PASS' >"$dir/rootskip.log" +expect "root: any skip fails" 1 "TestA" root "$dir/rootskip.log" +expect "a label that is neither fails with usage" 2 "neither iouring nor root" other "$dir/root.log" +expect "an unreadable log is a usage error" 2 "cannot read" root "$dir/absent.log" + +echo "$n cases" +exit "$fail" diff --git a/.github/scripts/base-tier-tally.sh b/.github/scripts/base-tier-tally.sh index cdce8c222..4be62ef4f 100755 --- a/.github/scripts/base-tier-tally.sh +++ b/.github/scripts/base-tier-tally.sh @@ -29,23 +29,17 @@ shift 2 [ -r "$log" ] || { echo "base-tier-tally: cannot read $log" >&2; exit 2; } here=$(cd "$(dirname "$0")" && pwd) -# Allowed skips: an alternation of full test names, one entry per line with the -# reason after the name. Measured on the runs cited in the pull request. -allowed_iouring() { - cat <<'LIST' -LIST -} -allowed_root() { - cat <<'LIST' -LIST -} - +# Allowed skips: .github/base-tier-skips-LABEL.txt, one test per line, its full +# name first and the reason it must skip at the base tier after it (blank lines +# and # lines are ignored). BASE_TIER_SKIPS_DIR points elsewhere for the +# self-test. case "$label" in - iouring) list=$(allowed_iouring) ;; - root) list=$(allowed_root) ;; + iouring | root) ;; *) echo "base-tier-tally: label '$label' is neither iouring nor root" >&2; exit 2 ;; esac -names=$(printf '%s\n' "$list" | awk 'NF { print $1 }' | paste -sd'|' -) +list_file=${BASE_TIER_SKIPS_DIR:-$here/..}/base-tier-skips-$label.txt +[ -r "$list_file" ] || { echo "base-tier-tally: cannot read $list_file" >&2; exit 2; } +names=$(awk 'NF && $1 !~ /^#/ { print $1 }' "$list_file" | paste -sd'|' -) fail=0 skipped=$("$here"/skip-count.sh "$log") || fail=1 diff --git a/.github/tools/requirements.txt b/.github/tools/requirements.txt new file mode 100644 index 000000000..ac452646c --- /dev/null +++ b/.github/tools/requirements.txt @@ -0,0 +1,5 @@ +# The Python tools CI runs, pinned here so that Dependabot's pip entry for +# /.github/tools (.github/dependabot.yml) proposes their bumps (celeris#890 +# item 93). The Lint job installs this file with `pip install -r`; no workflow +# names a version of its own (.github/scripts/tool-pins-check.sh, check 4). +zizmor==1.30.0 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c08682e33..b8cc162bf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -84,11 +84,19 @@ jobs: # and with a result line a test's own output joined (celeris#890). - name: named-tally.sh self-test run: .github/scripts/named-tally-test.sh - # The tools CI runs are pinned in one place, .github/tools/go.mod, and - # three other places name them: the allow list of its Dependabot entry + # The tally of the job `iouring-base-tier` (base-tier-tally.sh) on fixture + # logs: an allowed skip, a skip nobody allowed, an arm that ran at tier=high, + # a probe line of an uncapped run, and a named test missing or skipped + # (celeris#890 item 87). + - name: base-tier-tally.sh self-test + run: .github/scripts/base-tier-tally-test.sh + # The tools CI runs are pinned in one place, .github/tools/go.mod (the Go + # ones) or .github/tools/requirements.txt (zizmor), and three other places + # name them: the allow list of its Dependabot entry # (a tool without an entry, or one that names a package path instead # of the module path, silently stops getting bump PRs), CONTRIBUTING's - # golangci-lint version, and the workflows (no `go install ...@v`). + # golangci-lint version, and the workflows (no `go install ...@v`, no + # `name==version` of a Python tool). # tool-pins-check.sh compares them; its self-test runs it on broken # fixtures, so a check that stops failing is a red Lint (celeris#890). - name: Tool pins agree @@ -138,10 +146,15 @@ jobs: # template injection, excessive permissions). Complements # actionlint, which checks syntax rather than security posture. # Needs only the ephemeral job token (read) for the online audits; - # no repository secrets. Pinned to an exact version. + # no repository secrets. Pinned to an exact version in + # .github/tools/requirements.txt, which Dependabot's pip entry bumps + # (celeris#890 item 93); the step names no version of its own. env: GH_TOKEN: ${{ github.token }} - run: pipx run --spec zizmor==1.30.0 zizmor --color always .github/workflows + run: | + python3 -m venv "$RUNNER_TEMP/zizmor-venv" + "$RUNNER_TEMP/zizmor-venv/bin/pip" install --quiet --disable-pip-version-check -r .github/tools/requirements.txt + "$RUNNER_TEMP/zizmor-venv/bin/zizmor" --color always .github/workflows # The required check "Unit (root + middleware sub-modules)" is the `unit-gate` # job below: it passes only when this job AND every root-package shard passed. @@ -1252,13 +1265,15 @@ jobs: # with -v, and tallies by name per package: every listed test must run and # PASS, and no SKIP line is allowed. # - # Which of them can fail a pull request: celeristest, middleware/timeout and - # middleware/ratelimit fail on any count over their budget. The rest (the - # drivers, middleware/{cache,idempotency,overload}) only log an over-budget - # count unless TESTING_STRICT_ALLOC_BUDGETS=1, which this step does not set, - # so for them it proves the guard ran (PASS, no skip) and not that the count - # held. Setting it is the stricter choice; see the pull request for the - # numbers behind it. The sse and websocket tests fail on their own bounds. + # Every one of them fails a pull request on a count over its budget. + # celeristest, middleware/timeout and middleware/ratelimit always did; the + # rest (the drivers, middleware/{cache,idempotency,overload}) only logged an + # over-budget count unless TESTING_STRICT_ALLOC_BUDGETS=1, which this step + # now sets (celeris#890 item 91), so for them a PASS says the count held and + # not only that the guard ran. The one guard that was over budget when the + # variable was first set, driver/postgres Query_1col_1row (14 allocs/op + # against 4), has the budget the driver costs. The sse and websocket tests + # fail on their own bounds. - name: celeris#890 allocation guards and alloc-count tests, without -race (skipping forbidden) if: ${{ !cancelled() }} shell: bash @@ -2444,6 +2459,17 @@ jobs: sys.exit(rc) PY .github/scripts/named-tally.sh 'test/server full-size shape (celeris#890 item 92)' /tmp/large.log "${tests[@]}" + # The shape itself: the 64 MiB HTTP/1 bodies (16 MiB under lean761) and the 16 MiB + # HTTP/2 ones (8 MiB under lean761) ran, on all four engines. + full1=$(grep -cE '^=== RUN TestLargeResponseIsDelivered/[a-z_]+/[a-z-]+/67108864/' /tmp/large.log || true) + full2=$(grep -cE '^=== RUN TestLargeResponseIsDeliveredH2/[a-z_]+/[a-z-]+/16777216$' /tmp/large.log || true) + lean1=$(grep -cE '^=== RUN TestLargeResponseIsDelivered/.*/16777216/' /tmp/large.log || true) + lean2=$(grep -cE '^=== RUN TestLargeResponseIsDeliveredH2/.*/8388608$' /tmp/large.log || true) + echo "full-size bodies: HTTP/1 64 MiB runs $full1 (want 12), HTTP/2 16 MiB runs $full2 (want 8), lean-size runs $lean1 and $lean2 (want 0)" + if [ "$full1" -ne 12 ] || [ "$full2" -ne 8 ] || [ "$lean1" -ne 0 ] || [ "$lean2" -ne 0 ]; then + echo "expected the full-size bodies, not the lean ones -- did lean761() become true here (race or coverage), or did a size change?" + exit 1 + fi conformance: name: Conformance diff --git a/driver/postgres/alloc_guard_test.go b/driver/postgres/alloc_guard_test.go index d991f3efb..083770369 100644 --- a/driver/postgres/alloc_guard_test.go +++ b/driver/postgres/alloc_guard_test.go @@ -43,7 +43,12 @@ func TestAllocBudgets(t *testing.T) { } guards := []guard{ - {name: "Query_1col_1row", budget: 4, bench: benchQuery1col1row}, + // 14, what the driver costs: celeris#890 item 91 measured 14 allocs/op on a + // linux/arm64 container (20 of 20 runs) and on the GitHub amd64 runner (run + // 39074958353) against the aspirational 4 this row held until then, which + // never failed because no CI step set TESTING_STRICT_ALLOC_BUDGETS=1. The + // allocation-guard step of ci.yml sets it now, so 14 is the bound. + {name: "Query_1col_1row", budget: 14, bench: benchQuery1col1row}, {name: "Pool_AcquireRelease", budget: 0, bench: benchPoolAcquireRelease}, // Prepared exec is intentionally omitted: it requires the extended // protocol (Parse/Bind/Execute/Sync) handshake which the existing diff --git a/test/integration/adaptive_standby_886_linux_test.go b/test/integration/adaptive_standby_886_linux_test.go index b8aaafd2c..105875fab 100644 --- a/test/integration/adaptive_standby_886_linux_test.go +++ b/test/integration/adaptive_standby_886_linux_test.go @@ -38,7 +38,7 @@ func TestAdaptiveSwitchVerdictOnAFailedStandby886(t *testing.T) { t.Skip("integration") } if !probe.Probe().IOUringTier.Available() { - t.Skip("io_uring unavailable: the standby refusal is never reached") + skipOrFailUpswitch890(t, "io_uring unavailable: the standby refusal is never reached") } run := func(require bool) (string, int) { t.Helper() diff --git a/test/integration/adaptive_standby_guard_886_linux_test.go b/test/integration/adaptive_standby_guard_886_linux_test.go index dd10f7d4e..82b6cd21e 100644 --- a/test/integration/adaptive_standby_guard_886_linux_test.go +++ b/test/integration/adaptive_standby_guard_886_linux_test.go @@ -56,7 +56,18 @@ func skipOrFailIfStandbyAborted886(t *testing.T, switches uint64, logs *syncBuf8 if switches != 0 || !standbyAborted886(logs) { return } - const msg = "the io_uring standby did not come up here: the engine aborted the forced switch and stayed put" + skipOrFailUpswitch890(t, "the io_uring standby did not come up here: the engine aborted the forced switch and stayed put") +} + +// skipOrFailUpswitch890 is this package's one definition of the gate that +// ./internal/adaptive keeps in skipOrFailUpswitch662 (the two cannot share a +// function: separate packages, and the test files of internal/adaptive cannot be +// imported). A test of the adaptive_*_test.go files that cannot run where the +// io_uring standby is unavailable skips through it, and fails instead under +// CELERIS_REQUIRE_UPSWITCH=1; TestAdaptiveSkipRoutingIntegration890 fails on a +// raw t.Skip in those files that bypasses it (celeris#890 item 96). +func skipOrFailUpswitch890(t *testing.T, msg string) { + t.Helper() if os.Getenv("CELERIS_REQUIRE_UPSWITCH") == "1" { t.Fatal(msg + " -- CELERIS_REQUIRE_UPSWITCH=1 forbids skipping") } diff --git a/test/integration/skip_routing_890_test.go b/test/integration/skip_routing_890_test.go index bafb6886e..350851b75 100644 --- a/test/integration/skip_routing_890_test.go +++ b/test/integration/skip_routing_890_test.go @@ -16,8 +16,9 @@ import ( // celeris#890 item 96: a test of the adaptive files of this package that cannot // run where the io_uring standby is unavailable skips through -// skipOrFailIfStandbyAborted886 (adaptive_standby_guard_886_linux_test.go), -// which fails instead under CELERIS_REQUIRE_UPSWITCH=1. It repeats, in this +// skipOrFailUpswitch890 (adaptive_standby_guard_886_linux_test.go, which +// skipOrFailIfStandbyAborted886 calls too), which fails instead under +// CELERIS_REQUIRE_UPSWITCH=1. It repeats, in this // package, the gate that internal/adaptive keeps in skipOrFailUpswitch662: the // two cannot share a function (separate packages, and the test files of // internal/adaptive cannot be imported), so what keeps this package's copy @@ -40,12 +41,36 @@ import ( // gateHelpers890 are the functions whose job is to skip or fail; a raw skip // inside one is the gate itself. var gateHelpers890 = map[string]bool{ - "skipOrFailIfStandbyAborted886": true, // the standby gate (CELERIS_REQUIRE_UPSWITCH=1) + "skipOrFailUpswitch890": true, // the gate (CELERIS_REQUIRE_UPSWITCH=1) } // unrouted890 are the raw skips that stay for now, by file and the string -// literal that starts the skip's message, each with why. -var unrouted890 = map[string]string{} +// literal that starts the skip's message, each with how many sites carry that +// message and why they stay. The count is exact: a further site with a +// registered message fails the guard as a stale entry would. +// +// All ten are in adaptive_hybrid_test.go. #890 item 95 leaves the adaptive +// skips of this package to the lane that owns that file; the table is the count +// of what is left (the guard listed these ten and one more, in +// adaptive_standby_886_linux_test.go, which now goes through the gate), and a +// new one cannot join them unseen. +type unrouted890Entry struct { + sites int + why string +} + +const ( + newFails890 = "adaptive.New fails on config validation or a port hold (internal/adaptive/engine.go New), not on a host without io_uring: a failure there is a defect, as for epoll.New in item 95" + startFail890 = "an engine that does not start, bind or answer: this skip cannot tell a host without io_uring from a defect" +) + +var unrouted890 = map[string]unrouted890Entry{ + "adaptive_hybrid_test.go|adaptive engine not available: %v": {5, newFails890}, + "adaptive_hybrid_test.go|engine failed to start: %v": {2, startFail890}, + "adaptive_hybrid_test.go|engine Listen returned nil without addr": {1, startFail890}, + "adaptive_hybrid_test.go|engine did not start in time": {1, startFail890}, + "adaptive_hybrid_test.go|engine not functional: %v": {1, startFail890}, +} // rawSkipViolations890 lists the t.Skip* calls of one Go source that the rules // above do not admit, as "file:line: call(message)". It returns an error for a @@ -184,7 +209,7 @@ func TestAdaptiveSkipRoutingIntegration890(t *testing.T) { } sort.Strings(files) var bad []string - seen := map[string]bool{} + seen := map[string]int{} for _, f := range files { src, err := os.ReadFile(f) if err != nil { @@ -203,7 +228,7 @@ func TestAdaptiveSkipRoutingIntegration890(t *testing.T) { ast.Inspect(af, func(n ast.Node) bool { if c, ok := n.(*ast.CallExpr); ok { if s, ok := c.Fun.(*ast.SelectorExpr); ok && isSkipSelector890(s) { - seen[f+"|"+skipMessage890(c)] = true + seen[f+"|"+skipMessage890(c)]++ } } return true @@ -211,14 +236,14 @@ func TestAdaptiveSkipRoutingIntegration890(t *testing.T) { } t.Logf("%d adaptive test files, %d raw skips not admitted, %d registered in unrouted890", len(files), len(bad), len(unrouted890)) if len(bad) > 0 { - t.Errorf("t.Skip calls that bypass skipOrFailIfStandbyAborted886 (adaptive_standby_guard_886_linux_test.go) and are not registered in unrouted890:\n %s", + t.Errorf("t.Skip calls that bypass skipOrFailUpswitch890 (adaptive_standby_guard_886_linux_test.go) and are not registered in unrouted890:\n %s", strings.Join(bad, "\n ")) } - // Every registered skip must still exist, or the table keeps a reason for - // nothing. - for k := range unrouted890 { - if !seen[k] { - t.Errorf("unrouted890 registers %q, which no t.Skip in the adaptive files has any more: delete the entry", k) + // Every registered skip must still exist, as many times as registered, or + // the table keeps a reason for nothing or admits a site unseen. + for k, e := range unrouted890 { + if seen[k] != e.sites { + t.Errorf("unrouted890 registers %q at %d sites, but the adaptive files have %d t.Skip calls with it: fix the entry", k, e.sites, seen[k]) } } } @@ -236,8 +261,9 @@ func TestAdaptiveSkipRoutingGuardSelfTest890(t *testing.T) { {"raw Skipf fails", head + `func TestX(t *testing.T) { t.Skipf("adaptive engine not available: %v", 1) }`, 1}, {"raw SkipNow fails", head + `func TestX(t *testing.T) { t.SkipNow() }`, 1}, {"a raw skip in a subtest closure fails", head + `func TestX(t *testing.T) { t.Run("a", func(t *testing.T) { t.Skip("io_uring unavailable") }) }`, 1}, - {"the gate passes", head + `func TestX(t *testing.T) { skipOrFailIfStandbyAborted886(t, 0, nil) }`, 0}, - {"a skip inside the gate helper passes", head + `func skipOrFailIfStandbyAborted886(t *testing.T) { t.Skip("x") }`, 0}, + {"the gate passes", head + `func TestX(t *testing.T) { skipOrFailUpswitch890(t, "io_uring unavailable") }`, 0}, + {"a skip inside the gate helper passes", head + `func skipOrFailUpswitch890(t *testing.T, m string) { t.Skip(m) }`, 0}, + {"a registered message is not exempt in an unregistered file", head + `func TestX(t *testing.T) { t.Skipf("engine failed to start: %v", 1) }`, 1}, {"a -short gate passes", head + `func TestX(t *testing.T) { if testing.Short() { t.Skip("integration") } }`, 0}, {"a -short gate with another clause passes", head + `func TestX(t *testing.T) { if x && testing.Short() { t.Skip("integration") } }`, 0}, {"a raw skip after a -short gate fails", head + `func TestX(t *testing.T) { if testing.Short() { t.Skip("integration") }; t.Skip("io_uring unavailable") }`, 1}, From b56c8fd97c7030f9bd418b273b1436f78b8224d3 Mon Sep 17 00:00:00 2001 From: Albert Bausili Date: Mon, 26 Oct 2026 23:55:49 +0100 Subject: [PATCH 05/14] ci: the base-tier allow-lists, measured on the first run of the job (Refs #890 item 87) Run 40414278698 on b90001a: ./internal/engine/iouring took 473 s under -race with 554 top-level tests passed and 12 SKIP lines, the root package and ./test/server 4.6 s and 239 s with 107 passed and 19 SKIP lines, and TestShutdownDetachedPartialWriteIsFlushed806 passed on the capped probe. Each skip is named in .github/base-tier-skips-{iouring,root}.txt with its reason, the tally has a min-pass floor, and its self-test covers it. --- .github/base-tier-skips-iouring.txt | 23 +++++++++++++++++- .github/base-tier-skips-root.txt | 31 ++++++++++++++++++++++++- .github/scripts/base-tier-tally-test.sh | 5 +++- .github/scripts/base-tier-tally.sh | 15 ++++++++++-- .github/workflows/ci.yml | 14 +++++++++-- 5 files changed, 81 insertions(+), 7 deletions(-) diff --git a/.github/base-tier-skips-iouring.txt b/.github/base-tier-skips-iouring.txt index 0ca8ba3a5..febc6f8bc 100644 --- a/.github/base-tier-skips-iouring.txt +++ b/.github/base-tier-skips-iouring.txt @@ -1 +1,22 @@ -# Tests that skip at CELERIS_MAX_IOURING_TIER=base on purpose: the full test name, then why. +# Tests that skip at CELERIS_MAX_IOURING_TIER=base on purpose, for .github/scripts/base-tier-tally.sh +# (celeris#890 item 87): the full test or subtest name, then why. A name covers its own +# subtests. Measured on the run of the job iouring-base-tier cited in the pull request: +# 554 top-level tests passed, 4 skipped at the top level, 12 SKIP lines in all. +min-pass 500 + +# Tier base has no multishot recv with provided buffers (the tests say so in their skip). +TestSpentDetectBudgetReturnsItsRingBuffer/check no provided buffer ring at the base tier +TestSpentDetectBudgetReturnsItsRingBuffer/recheck no provided buffer ring at the base tier +TestSpentDetectBudgetReturnsItsRingBuffer/check-async no provided buffer ring at the base tier +TestSpentDetectBudgetReturnsItsRingBuffer/recheck-async no provided buffer ring at the base tier +TestStartupLineReportsTheMultishotRecvThatIsInUse/unset tier base has no multishot recv +TestStartupLineReportsTheMultishotRecvThatIsInUse/zero tier base has no multishot recv +TestStartupLineReportsTheMultishotRecvThatIsInUse/true-is-not-1 tier base has no multishot recv +TestStartupLineReportsTheMultishotRecvThatIsInUse/one tier base has no multishot recv +TestMultishotRecvDecidedOnceAtNew/on_at_New,_off_after tier base has no multishot recv +TestMultishotRecvDecidedOnceAtNew/off_at_New,_on_after tier base has no multishot recv + +# They need net.ipv4.tcp_synack_retries=0; the runner has 5. The step "celeris#662 +# tcp_synack_retries=0 guard" of the job adaptive sets it and forbids the skip. +TestPauseAcceptServesDeferredIdleConnectionsSynackRetriesZero needs net.ipv4.tcp_synack_retries=0 +TestPauseAcceptNoGuardLosesAtSynackRetriesZero needs net.ipv4.tcp_synack_retries=0 diff --git a/.github/base-tier-skips-root.txt b/.github/base-tier-skips-root.txt index 0ca8ba3a5..6072a605d 100644 --- a/.github/base-tier-skips-root.txt +++ b/.github/base-tier-skips-root.txt @@ -1 +1,30 @@ -# Tests that skip at CELERIS_MAX_IOURING_TIER=base on purpose: the full test name, then why. +# Tests that skip at CELERIS_MAX_IOURING_TIER=base on purpose, for .github/scripts/base-tier-tally.sh +# (celeris#890 item 87): the full test or subtest name, then why. A name covers its own +# subtests. Measured on the run of the job iouring-base-tier cited in the pull request: +# 107 top-level tests passed, 3 skipped at the top level, 19 SKIP lines in all. +min-pass 100 + +# The probes of these tests (c714ProbeIOUring, keptProbeIOUring and the one of +# h2_handler_return_889) call an io_uring "usable" only at the high tier with provided +# buffers, so with the cap their io_uring and adaptive arms skip with "no usable +# io_uring". The std and epoll arms of the same tests run. +TestH2HandlerReturnWithoutClosingEndsTheStream889/io_uring no usable io_uring at the base tier +TestH2HandlerReturnWithoutClosingEndsTheStream889/adaptive no usable io_uring at the base tier +TestH2HandlerReturnWithoutClosingEndsTheStream889/adaptive-iouring no usable io_uring at the base tier +TestH2HandlerShortOfItsContentLengthResetsTheStream889/io_uring no usable io_uring at the base tier +TestH2HandlerShortOfItsContentLengthResetsTheStream889/adaptive no usable io_uring at the base tier +TestH2HandlerShortOfItsContentLengthResetsTheStream889/adaptive-iouring no usable io_uring at the base tier +TestH2AsyncHandlerGoexitResetsTheStreamAndIsLogged889/io_uring no usable io_uring at the base tier +TestH2AsyncHandlerGoexitResetsTheStreamAndIsLogged889/adaptive no usable io_uring at the base tier +TestH2AsyncHandlerGoexitResetsTheStreamAndIsLogged889/adaptive-iouring no usable io_uring at the base tier +TestH2StreamedResponseBrokenOffIsReset889/io_uring no usable io_uring at the base tier +TestH2StreamedResponseBrokenOffIsReset889/adaptive no usable io_uring at the base tier +TestH2StreamedResponseBrokenOffIsReset889/adaptive-iouring no usable io_uring at the base tier +TestServerAsyncRouteDetachThenGoexitOnIOUring no usable io_uring at the base tier +TestH2InlineStreamGoexitIsLoud848/io_uring no usable io_uring at the base tier +TestH2AsyncStreamGoexitKeepsTheLoop848/io_uring no usable io_uring at the base tier +TestH2OverBudgetStreamGoexitKeepsTheLoop848/io_uring no usable io_uring at the base tier + +# Tier base has no multishot recv, and these two tests are about it. +TestHijackCopiesRequestValuesUnderMultishotRecv tier base has no multishot recv +TestMultishotRingSurvivesHijacks tier base has no multishot recv diff --git a/.github/scripts/base-tier-tally-test.sh b/.github/scripts/base-tier-tally-test.sh index fc1f3fc8a..f9e1b2275 100755 --- a/.github/scripts/base-tier-tally-test.sh +++ b/.github/scripts/base-tier-tally-test.sh @@ -13,7 +13,7 @@ trap 'rm -rf "$dir"' EXIT fail=0 n=0 export BASE_TIER_SKIPS_DIR=$dir -printf '# header\nTestNeedsHighTier needs a provided buffer ring\n' >"$dir/base-tier-skips-iouring.txt" +printf '# header\nmin-pass 2\nTestNeedsHighTier needs a provided buffer ring\n' >"$dir/base-tier-skips-iouring.txt" printf '# header\n' >"$dir/base-tier-skips-root.txt" # expect NAME WANT_STATUS WANT_STDERR_SUBSTRING LABEL LOG [TESTS...] @@ -57,6 +57,9 @@ grep -v 'TestShutdownDetachedPartialWriteIsFlushed806' "$dir/good.log" >"$dir/no expect "a named test that never ran fails" 1 "TestShutdownDetachedPartialWriteIsFlushed806 ran 0" iouring "$dir/no806.log" TestShutdownDetachedPartialWriteIsFlushed806 sed 's/^--- PASS: TestShutdownDetachedPartialWriteIsFlushed806/--- SKIP: TestShutdownDetachedPartialWriteIsFlushed806/' "$dir/good.log" >"$dir/skip806.log" expect "a named test that skipped fails" 1 "TestShutdownDetachedPartialWriteIsFlushed806" iouring "$dir/skip806.log" TestShutdownDetachedPartialWriteIsFlushed806 +# min-pass: a run that lost its tests (a -run pattern that stopped matching). +grep -v 'TestShutdownDetachedPartialWriteIsFlushed806' "$dir/good.log" | sed 's/^--- PASS: TestReportTier844/--- XXX: TestReportTier844/' >"$dir/shrunk.log" +expect "fewer top-level PASS lines than min-pass fails" 1 "fewer than the 2" iouring "$dir/shrunk.log" # The root label has no tier lines and allows nothing here. printf '%s\n' '=== RUN TestA' '--- PASS: TestA (0.00s)' 'PASS' >"$dir/root.log" expect "root: a clean log" 0 "" root "$dir/root.log" diff --git a/.github/scripts/base-tier-tally.sh b/.github/scripts/base-tier-tally.sh index 4be62ef4f..8d5b2a374 100755 --- a/.github/scripts/base-tier-tally.sh +++ b/.github/scripts/base-tier-tally.sh @@ -18,6 +18,8 @@ # probe does not know counts as "none", which would skip every io_uring # test and leave check 1 as the only witness; this one names the cause. # +# 4. At least `min-pass` top-level tests passed (the data file of the label). +# # Prints one summary line on stdout. Exit status 0 all hold, 1 some do not, # 2 usage or an unreadable log. set -euo pipefail @@ -39,7 +41,11 @@ case "$label" in esac list_file=${BASE_TIER_SKIPS_DIR:-$here/..}/base-tier-skips-$label.txt [ -r "$list_file" ] || { echo "base-tier-tally: cannot read $list_file" >&2; exit 2; } -names=$(awk 'NF && $1 !~ /^#/ { print $1 }' "$list_file" | paste -sd'|' -) +names=$(awk 'NF && $1 !~ /^#/ && $1 != "min-pass" { print $1 }' "$list_file" | paste -sd'|' -) +# `min-pass N` in the file: the top-level tests that must PASS at least, so that a -run +# pattern that stops matching (a rename) or a package that stops compiling its tests +# in cannot shrink the run unseen. The figure is what the first run measured. +minpass=$(awk '$1 == "min-pass" { print $2 }' "$list_file") fail=0 skipped=$("$here"/skip-count.sh "$log") || fail=1 @@ -63,7 +69,12 @@ if [ "$label" = iouring ]; then fi fi -echo "base-tier-tally $label: SKIP lines $skipped, not in the allow-list $n_unlisted$tiers" +passed=$(grep -acE '^--- PASS: Test' "$log" || true) +if [ -n "$minpass" ] && [ "$passed" -lt "$minpass" ]; then + fail=1 + echo "base-tier-tally: $passed top-level tests passed, fewer than the $minpass of $list_file: did the run shrink (a renamed test, a -run pattern that no longer matches)?" >&2 +fi +echo "base-tier-tally $label: top-level PASS $passed (min ${minpass:-none}), SKIP lines $skipped, not in the allow-list $n_unlisted$tiers" if [ "$n_unlisted" -ne 0 ]; then fail=1 printf '%s\n' "$unlisted" >&2 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b8cc162bf..0db3a04b3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2369,8 +2369,18 @@ jobs: # * the root package and ./test/server, the tests whose names say Shutdown, # Detach, SSE, Stream, WebSocket, Hijack or Drain, under -race. # Tests that need the high tier skip at this tier on purpose; each skip that - # is allowed is listed by name below with its reason, and any other SKIP line - # fails the step. + # is allowed is listed by name with its reason in .github/base-tier-skips-iouring.txt + # and -root.txt, and any other SKIP line fails the step (base-tier-tally.sh, + # whose self-test is a Lint step). First run on a hosted runner (kernel + # 6.17.0-1022-azure, run 40414278698): the package took 473 s under -race (504 s + # for the step), 554 top-level tests passed and 12 SKIP lines are the allowed + # ones; the root package and ./test/server took 4.6 s and 239 s (258 s for the + # step), 107 top-level tests passed and 19 SKIP lines are allowed. Those 19 are + # the io_uring and adaptive arms of tests whose probe (c714ProbeIOUring, + # keptProbeIOUring) calls io_uring usable only at the high tier: with the cap + # they skip with "no usable io_uring", so what this step adds for the root tests + # is the tests that start an io_uring engine without such a probe: they run at + # the base tier here and at the high tier in every other job. iouring-base-tier: name: io_uring base tier (CELERIS_MAX_IOURING_TIER=base, celeris#890) runs-on: ubuntu-latest From e7a797ce163cd688812519a4c3c14f8257bbb4ae Mon Sep 17 00:00:00 2001 From: Albert Bausili Date: Mon, 26 Oct 2026 23:56:28 +0100 Subject: [PATCH 06/14] DO NOT MERGE: bad inputs for the gates of items 87 and 92 (a misspelled tier cap, a -race build of the full-size job); reverted by the next commit (Refs #890) --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0db3a04b3..e3b02246e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2386,7 +2386,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 45 env: - CELERIS_MAX_IOURING_TIER: base + CELERIS_MAX_IOURING_TIER: bse steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -2458,7 +2458,7 @@ jobs: TestH2CUnmatchedThroughTimeoutSingleflight836 ) regex=$(IFS='|'; echo "${tests[*]}") - go test -c -o /tmp/server.test ./test/server + go test -race -c -o /tmp/server.test ./test/server python3 - /tmp/server.test "^(${regex})\$" <<'PY' 2>&1 | tee /tmp/large.log import resource, subprocess, sys, time binary, run = sys.argv[1], sys.argv[2] From 6793859364d1eff03f6a8ca119773431f90e66b7 Mon Sep 17 00:00:00 2001 From: Albert Bausili Date: Tue, 27 Oct 2026 00:24:16 +0100 Subject: [PATCH 07/14] docs(contributing): the performance label also starts the CodSpeed run (Refs #890 item 7) --- CONTRIBUTING.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 940810040..7f6ada74a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -171,6 +171,11 @@ category it belongs to so it lands in the right section: Anything else lands under "Other". If a change is breaking, label it `breaking` *and* say so in the PR description. +The `performance` label has a second job: a pull request that touches the +benchmarked paths starts the CodSpeed run (`.github/workflows/codspeed.yml`) +only while it carries that label, because the run spends macro-runner minutes +from a monthly budget shared with loadgen. Main is measured once a day. + ### Commit sign-off We recommend signing off your commits with `git commit -s`, which adds a From d8a006ec996da79023cb6c9317bba72571204d7f Mon Sep 17 00:00:00 2001 From: Albert Bausili Date: Tue, 27 Oct 2026 00:24:16 +0100 Subject: [PATCH 08/14] Revert "DO NOT MERGE: bad inputs for the gates of items 87 and 92 (a misspelled tier cap, a -race build of the full-size job); reverted by the next commit (Refs #890)" This reverts commit e7a797ce163cd688812519a4c3c14f8257bbb4ae. --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e3b02246e..0db3a04b3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2386,7 +2386,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 45 env: - CELERIS_MAX_IOURING_TIER: bse + CELERIS_MAX_IOURING_TIER: base steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -2458,7 +2458,7 @@ jobs: TestH2CUnmatchedThroughTimeoutSingleflight836 ) regex=$(IFS='|'; echo "${tests[*]}") - go test -race -c -o /tmp/server.test ./test/server + go test -c -o /tmp/server.test ./test/server python3 - /tmp/server.test "^(${regex})\$" <<'PY' 2>&1 | tee /tmp/large.log import resource, subprocess, sys, time binary, run = sys.argv[1], sys.argv[2] From 08464be6979f720f18a5d053d93e15f82945325d Mon Sep 17 00:00:00 2001 From: Albert Bausili Date: Tue, 27 Oct 2026 00:24:16 +0100 Subject: [PATCH 09/14] ci: remove the temporary hosted twin of the CodSpeed gate (Refs #890 item 7) Its runs are cited in the pull request: 40414920862 (label spike, ran, not cancelled by the run another label started, 40414979324 skipped), 40415458668 (a push with the label ran) and 40416207215 (cancelled by the run of the next push, 40416230527). --- .github/workflows/l18-gate-proof.yml | 41 ---------------------------- 1 file changed, 41 deletions(-) delete mode 100644 .github/workflows/l18-gate-proof.yml diff --git a/.github/workflows/l18-gate-proof.yml b/.github/workflows/l18-gate-proof.yml deleted file mode 100644 index 5366331c5..000000000 --- a/.github/workflows/l18-gate-proof.yml +++ /dev/null @@ -1,41 +0,0 @@ -# TEMPORARY (celeris#890 item 7, lane L18). A hosted twin of codspeed.yml's -# trigger, concurrency group and job condition, with the label `spike` in place -# of `performance` and no draft clause, so that the label gate can be shown on -# GitHub without a macro-runner minute. Deleted before the pull request is -# marked ready. -name: L18 gate proof - -on: - pull_request: - branches: [main] - types: [opened, synchronize, reopened, ready_for_review, labeled] - paths: - - ".github/workflows/l18-gate-proof.yml" - -permissions: - contents: read - -concurrency: - group: l18proof-${{ (github.event.action == 'labeled' && github.event.label.name != 'spike' && github.run_id) || github.event.pull_request.number || github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - -jobs: - benchmarks: - name: Gate twin - if: >- - github.event_name != 'pull_request' || - (github.event.pull_request.head.repo.full_name == github.repository && - contains(github.event.pull_request.labels.*.name, 'spike') && - (github.event.action != 'labeled' || github.event.label.name == 'spike')) - runs-on: ubuntu-latest - timeout-minutes: 5 - steps: - - name: Hold the slot - env: - ACTION: ${{ github.event.action }} - LABEL: ${{ github.event.label.name }} - SHA: ${{ github.event.pull_request.head.sha }} - run: | - echo "twin ran: action=$ACTION label=$LABEL head=$SHA" - sleep 90 - echo "twin finished" From 5556e7f19f0bceb0a11f166e7d419e8e6d66384e Mon Sep 17 00:00:00 2001 From: Albert Bausili Date: Tue, 27 Oct 2026 00:47:00 +0100 Subject: [PATCH 10/14] ci: the base-tier root allow-list names the probe-gated arms by pattern (Refs #890 item 87) Run 40419578127 (the head after a merge with main) failed the job: main had brought in the TestH1Unclosed...889 family, nine tests whose io_uring, adaptive and adaptive-iouring arms skip with 'io_uring tier=base ...: no usable io_uring' (117 top-level PASS, 46 SKIP lines, 27 not named). An exact-name list would turn the job red for every test of that kind that a later pull request adds. The root list now has a pattern entry for the three arm names, honoured only where the log says tier=base under the skipped arm; the self-test covers an arm that logged it, one that skipped for another reason, one the pattern does not name and a top-level test named like an arm. --- .github/base-tier-skips-root.txt | 35 +++++++++-------------- .github/scripts/base-tier-tally-test.sh | 22 +++++++++++--- .github/scripts/base-tier-tally.sh | 38 ++++++++++++++++--------- .github/workflows/ci.yml | 5 +++- 4 files changed, 61 insertions(+), 39 deletions(-) diff --git a/.github/base-tier-skips-root.txt b/.github/base-tier-skips-root.txt index 6072a605d..cd7f9ae11 100644 --- a/.github/base-tier-skips-root.txt +++ b/.github/base-tier-skips-root.txt @@ -1,30 +1,23 @@ # Tests that skip at CELERIS_MAX_IOURING_TIER=base on purpose, for .github/scripts/base-tier-tally.sh # (celeris#890 item 87): the full test or subtest name, then why. A name covers its own # subtests. Measured on the run of the job iouring-base-tier cited in the pull request: -# 107 top-level tests passed, 3 skipped at the top level, 19 SKIP lines in all. +# 107 top-level tests passed, 3 skipped at the top level, 19 SKIP lines in all (and +# 117 passed with 46 SKIP lines once main had brought in nine more tests of the kind below). min-pass 100 -# The probes of these tests (c714ProbeIOUring, keptProbeIOUring and the one of -# h2_handler_return_889) call an io_uring "usable" only at the high tier with provided -# buffers, so with the cap their io_uring and adaptive arms skip with "no usable -# io_uring". The std and epoll arms of the same tests run. -TestH2HandlerReturnWithoutClosingEndsTheStream889/io_uring no usable io_uring at the base tier -TestH2HandlerReturnWithoutClosingEndsTheStream889/adaptive no usable io_uring at the base tier -TestH2HandlerReturnWithoutClosingEndsTheStream889/adaptive-iouring no usable io_uring at the base tier -TestH2HandlerShortOfItsContentLengthResetsTheStream889/io_uring no usable io_uring at the base tier -TestH2HandlerShortOfItsContentLengthResetsTheStream889/adaptive no usable io_uring at the base tier -TestH2HandlerShortOfItsContentLengthResetsTheStream889/adaptive-iouring no usable io_uring at the base tier -TestH2AsyncHandlerGoexitResetsTheStreamAndIsLogged889/io_uring no usable io_uring at the base tier -TestH2AsyncHandlerGoexitResetsTheStreamAndIsLogged889/adaptive no usable io_uring at the base tier -TestH2AsyncHandlerGoexitResetsTheStreamAndIsLogged889/adaptive-iouring no usable io_uring at the base tier -TestH2StreamedResponseBrokenOffIsReset889/io_uring no usable io_uring at the base tier -TestH2StreamedResponseBrokenOffIsReset889/adaptive no usable io_uring at the base tier -TestH2StreamedResponseBrokenOffIsReset889/adaptive-iouring no usable io_uring at the base tier -TestServerAsyncRouteDetachThenGoexitOnIOUring no usable io_uring at the base tier -TestH2InlineStreamGoexitIsLoud848/io_uring no usable io_uring at the base tier -TestH2AsyncStreamGoexitKeepsTheLoop848/io_uring no usable io_uring at the base tier -TestH2OverBudgetStreamGoexitKeepsTheLoop848/io_uring no usable io_uring at the base tier +# The probes of the tests that gate their io_uring arms (c714ProbeIOUring, +# keptProbeIOUring, h2_handler_return_889's) call io_uring "usable" only at the high +# tier with provided buffers, so with the cap their io_uring and adaptive arms skip with +# "io_uring tier=base kernel=...: no usable io_uring". The std and epoll arms of the +# same tests run. A new test of this kind joins them without an edit here, because the +# entry names the arms, not the tests (run 40419578127 met nine new ones that a merge +# with main brought in: the TestH1Unclosed...889 family); the log must say tier=base +# under the skipped arm, so an arm that skips for another reason still fails the step. +pattern:^.+/(io_uring|adaptive|adaptive-iouring)$ the io_uring, adaptive and adaptive-iouring arms of probe-gated tests # Tier base has no multishot recv, and these two tests are about it. TestHijackCopiesRequestValuesUnderMultishotRecv tier base has no multishot recv TestMultishotRingSurvivesHijacks tier base has no multishot recv + +# Its probe (keptProbeIOUring) calls io_uring usable only at the high tier: "no usable io_uring". +TestServerAsyncRouteDetachThenGoexitOnIOUring no usable io_uring at the base tier diff --git a/.github/scripts/base-tier-tally-test.sh b/.github/scripts/base-tier-tally-test.sh index f9e1b2275..47802d373 100755 --- a/.github/scripts/base-tier-tally-test.sh +++ b/.github/scripts/base-tier-tally-test.sh @@ -14,7 +14,7 @@ fail=0 n=0 export BASE_TIER_SKIPS_DIR=$dir printf '# header\nmin-pass 2\nTestNeedsHighTier needs a provided buffer ring\n' >"$dir/base-tier-skips-iouring.txt" -printf '# header\n' >"$dir/base-tier-skips-root.txt" +printf '# header\nmin-pass 1\npattern:^.+/(io_uring|adaptive)$ the arms of probe-gated tests\nTestNeedsHighTier needs a provided buffer ring\n' >"$dir/base-tier-skips-root.txt" # expect NAME WANT_STATUS WANT_STDERR_SUBSTRING LABEL LOG [TESTS...] expect() { @@ -60,11 +60,25 @@ expect "a named test that skipped fails" 1 "TestShutdownDetachedPartialWriteIsFl # min-pass: a run that lost its tests (a -run pattern that stopped matching). grep -v 'TestShutdownDetachedPartialWriteIsFlushed806' "$dir/good.log" | sed 's/^--- PASS: TestReportTier844/--- XXX: TestReportTier844/' >"$dir/shrunk.log" expect "fewer top-level PASS lines than min-pass fails" 1 "fewer than the 2" iouring "$dir/shrunk.log" -# The root label has no tier lines and allows nothing here. +# The root label has no tier lines. A pattern entry allows an arm only while the log +# shows tier=base under that arm's RUN; a top-level test, or an arm that skips for +# another reason, is not allowed by it. printf '%s\n' '=== RUN TestA' '--- PASS: TestA (0.00s)' 'PASS' >"$dir/root.log" expect "root: a clean log" 0 "" root "$dir/root.log" -printf '%s\n' '=== RUN TestA' '--- SKIP: TestA (0.00s)' 'PASS' >"$dir/rootskip.log" -expect "root: any skip fails" 1 "TestA" root "$dir/rootskip.log" +printf '%s\n' '=== RUN TestA' '--- SKIP: TestA (0.00s)' '--- PASS: TestB (0.00s)' 'PASS' >"$dir/rootskip.log" +expect "root: a skip no entry names fails" 1 "TestA" root "$dir/rootskip.log" +arm() { # arm NAME MESSAGE: a probe-gated test whose NAME arm skipped with MESSAGE + printf '%s\n' '=== RUN TestP' "=== RUN TestP/$1" " p_test.go:9: $2" '=== RUN TestP/std' '--- PASS: TestP (0.00s)' \ + " --- SKIP: TestP/$1 (0.00s)" ' --- PASS: TestP/std (0.00s)' 'PASS' +} +arm io_uring 'io_uring tier=base kernel=6.17: no usable io_uring' >"$dir/arm.log" +expect "root: an arm that logged tier=base is allowed by the pattern" 0 "" root "$dir/arm.log" +arm io_uring 'the kernel has no io_uring' >"$dir/arm-other.log" +expect "root: an arm that skipped for another reason fails" 1 "TestP/io_uring" root "$dir/arm-other.log" +arm epoll 'io_uring tier=base kernel=6.17: no usable io_uring' >"$dir/arm-epoll.log" +expect "root: an arm the pattern does not name fails" 1 "TestP/epoll" root "$dir/arm-epoll.log" +printf '%s\n' '=== RUN io_uring' ' p_test.go:9: io_uring tier=base: no usable io_uring' '--- SKIP: io_uring (0.00s)' '--- PASS: TestB (0.00s)' >"$dir/arm-top.log" +expect "root: a top-level test named like an arm fails" 1 "SKIP: io_uring" root "$dir/arm-top.log" expect "a label that is neither fails with usage" 2 "neither iouring nor root" other "$dir/root.log" expect "an unreadable log is a usage error" 2 "cannot read" root "$dir/absent.log" diff --git a/.github/scripts/base-tier-tally.sh b/.github/scripts/base-tier-tally.sh index 8d5b2a374..671aadfe5 100755 --- a/.github/scripts/base-tier-tally.sh +++ b/.github/scripts/base-tier-tally.sh @@ -31,29 +31,41 @@ shift 2 [ -r "$log" ] || { echo "base-tier-tally: cannot read $log" >&2; exit 2; } here=$(cd "$(dirname "$0")" && pwd) -# Allowed skips: .github/base-tier-skips-LABEL.txt, one test per line, its full -# name first and the reason it must skip at the base tier after it (blank lines -# and # lines are ignored). BASE_TIER_SKIPS_DIR points elsewhere for the -# self-test. +# Allowed skips: .github/base-tier-skips-LABEL.txt, one entry per line, then the reason +# (blank lines and # lines are ignored; BASE_TIER_SKIPS_DIR points elsewhere for the +# self-test). An entry is +# NAME a test or subtest by its full name; it covers its own subtests, or +# pattern:REGEX a skipped test or subtest whose full name matches REGEX, and only +# when the log shows "tier=base" in a line the test logged (the +# skip message of a probe-gated test says "io_uring tier=base ..."). +# min-pass N the top-level tests that must PASS at least, so that a -run pattern +# that stops matching (a rename) or a package that stops compiling its +# tests in cannot shrink the run unseen; the figure is measured. case "$label" in iouring | root) ;; *) echo "base-tier-tally: label '$label' is neither iouring nor root" >&2; exit 2 ;; esac list_file=${BASE_TIER_SKIPS_DIR:-$here/..}/base-tier-skips-$label.txt [ -r "$list_file" ] || { echo "base-tier-tally: cannot read $list_file" >&2; exit 2; } -names=$(awk 'NF && $1 !~ /^#/ && $1 != "min-pass" { print $1 }' "$list_file" | paste -sd'|' -) -# `min-pass N` in the file: the top-level tests that must PASS at least, so that a -run -# pattern that stops matching (a rename) or a package that stops compiling its tests -# in cannot shrink the run unseen. The figure is what the first run measured. +names=$(awk 'NF && $1 !~ /^#/ && $1 != "min-pass" && $1 !~ /^pattern:/ { print $1 }' "$list_file" | paste -sd'|' -) +patterns=$(awk 'NF && $1 ~ /^pattern:/ { sub(/^pattern:/, "", $1); print $1 }' "$list_file" | paste -sd'|' -) minpass=$(awk '$1 == "min-pass" { print $2 }' "$list_file") fail=0 skipped=$("$here"/skip-count.sh "$log") || fail=1 -if [ -n "$names" ]; then - unlisted=$(grep -aE '^[[:space:]]*--- SKIP: ' "$log" | grep -avE "^[[:space:]]*--- SKIP: (${names})( |/)" || true) -else - unlisted=$(grep -aE '^[[:space:]]*--- SKIP: ' "$log" || true) -fi +# Each SKIP line is judged by the name of the test: allowed by an exact entry (or as a +# subtest of one), or by a pattern while the log shows tier=base under that test's RUN. +unlisted=$(awk -v names="$names" -v pats="$patterns" ' + /^=== RUN / { cur = $3; next } + /tier=base/ && cur != "" { base[cur] = 1 } + /^[[:space:]]*--- SKIP: / { + n = $3 + ok = 0 + if (names != "" && (n ~ ("^(" names ")$") || n ~ ("^(" names ")/"))) ok = 1 + if (!ok && pats != "" && n ~ pats && base[n]) ok = 1 + if (!ok) print + } +' "$log") n_unlisted=0 [ -n "$unlisted" ] && n_unlisted=$(printf '%s\n' "$unlisted" | wc -l | tr -d '[:space:]') diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0db3a04b3..620258c43 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2380,7 +2380,10 @@ jobs: # keptProbeIOUring) calls io_uring usable only at the high tier: with the cap # they skip with "no usable io_uring", so what this step adds for the root tests # is the tests that start an io_uring engine without such a probe: they run at - # the base tier here and at the high tier in every other job. + # the base tier here and at the high tier in every other job. The root list names + # those arms by pattern (a new probe-gated test needs no edit; the log must say + # tier=base under the skipped arm) after run 40419578127, on a merge with main, + # met nine new such tests (117 passed, 46 SKIP lines) and failed a list of names. iouring-base-tier: name: io_uring base tier (CELERIS_MAX_IOURING_TIER=base, celeris#890) runs-on: ubuntu-latest From 34382216b64c78afb1cadebcfdb1001752b68a5b Mon Sep 17 00:00:00 2001 From: Albert Bausili Date: Tue, 27 Oct 2026 00:52:39 +0100 Subject: [PATCH 11/14] ci: base-tier pattern evidence is the probe's own skip message, shorter step timeouts, the CodSpeed schedule caveat (Refs #890 items 7, 87) - The pattern entry no longer takes the engine's startup line (tier=base) as evidence: an arm that started an engine and skipped for another reason fails. Self-test case added (17 cases). - -timeout 900 s and 600 s (about twice the measured 473 s and 239 s), job limit 30 minutes: a run with the cap lost ended after 23 minutes in run 40416230192. - codspeed.yml says that CodSpeed documents push as the baseline and does not mention schedule, and what to do if the first scheduled run is not recorded. --- .github/base-tier-skips-root.txt | 5 +++-- .github/scripts/base-tier-tally-test.sh | 5 +++++ .github/scripts/base-tier-tally.sh | 11 +++++++---- .github/workflows/ci.yml | 15 ++++++++++----- .github/workflows/codspeed.yml | 8 ++++++++ 5 files changed, 33 insertions(+), 11 deletions(-) diff --git a/.github/base-tier-skips-root.txt b/.github/base-tier-skips-root.txt index cd7f9ae11..da1b45c28 100644 --- a/.github/base-tier-skips-root.txt +++ b/.github/base-tier-skips-root.txt @@ -11,8 +11,9 @@ min-pass 100 # "io_uring tier=base kernel=...: no usable io_uring". The std and epoll arms of the # same tests run. A new test of this kind joins them without an edit here, because the # entry names the arms, not the tests (run 40419578127 met nine new ones that a merge -# with main brought in: the TestH1Unclosed...889 family); the log must say tier=base -# under the skipped arm, so an arm that skips for another reason still fails the step. +# with main brought in: the TestH1Unclosed...889 family); the arm must have logged the +# probe's own skip message ("io_uring tier=base kernel=...") under its RUN line, so an +# arm that starts an engine and skips for another reason still fails the step. pattern:^.+/(io_uring|adaptive|adaptive-iouring)$ the io_uring, adaptive and adaptive-iouring arms of probe-gated tests # Tier base has no multishot recv, and these two tests are about it. diff --git a/.github/scripts/base-tier-tally-test.sh b/.github/scripts/base-tier-tally-test.sh index 47802d373..8905f98a8 100755 --- a/.github/scripts/base-tier-tally-test.sh +++ b/.github/scripts/base-tier-tally-test.sh @@ -73,6 +73,11 @@ arm() { # arm NAME MESSAGE: a probe-gated test whose NAME arm skipped with MESSA } arm io_uring 'io_uring tier=base kernel=6.17: no usable io_uring' >"$dir/arm.log" expect "root: an arm that logged tier=base is allowed by the pattern" 0 "" root "$dir/arm.log" +# An arm that started an engine (the engine's INFO line says tier=base) and then skipped +# for another reason is not excused by the pattern. +printf '%s\n' '=== RUN TestP' '=== RUN TestP/io_uring' '2026/10/26 23:34:23 INFO io_uring engine selected tier=base multishot_accept=false' \ + ' p_test.go:9: the peer never connected' '--- PASS: TestP (0.00s)' ' --- SKIP: TestP/io_uring (0.00s)' >"$dir/arm-engine.log" +expect "root: an arm that started an engine and skipped for another reason fails" 1 "TestP/io_uring" root "$dir/arm-engine.log" arm io_uring 'the kernel has no io_uring' >"$dir/arm-other.log" expect "root: an arm that skipped for another reason fails" 1 "TestP/io_uring" root "$dir/arm-other.log" arm epoll 'io_uring tier=base kernel=6.17: no usable io_uring' >"$dir/arm-epoll.log" diff --git a/.github/scripts/base-tier-tally.sh b/.github/scripts/base-tier-tally.sh index 671aadfe5..f3993acbf 100755 --- a/.github/scripts/base-tier-tally.sh +++ b/.github/scripts/base-tier-tally.sh @@ -36,8 +36,11 @@ here=$(cd "$(dirname "$0")" && pwd) # self-test). An entry is # NAME a test or subtest by its full name; it covers its own subtests, or # pattern:REGEX a skipped test or subtest whose full name matches REGEX, and only -# when the log shows "tier=base" in a line the test logged (the -# skip message of a probe-gated test says "io_uring tier=base ..."). +# when the test itself logged the probe's skip message under its RUN +# line ("file_test.go:N: io_uring tier=base kernel=...: no usable +# io_uring"). The engine's own startup line ("INFO io_uring engine +# selected tier=base ...") does not count: an arm that started an +# engine and then skipped for another reason must still fail. # min-pass N the top-level tests that must PASS at least, so that a -run pattern # that stops matching (a rename) or a package that stops compiling its # tests in cannot shrink the run unseen; the figure is measured. @@ -54,10 +57,10 @@ minpass=$(awk '$1 == "min-pass" { print $2 }' "$list_file") fail=0 skipped=$("$here"/skip-count.sh "$log") || fail=1 # Each SKIP line is judged by the name of the test: allowed by an exact entry (or as a -# subtest of one), or by a pattern while the log shows tier=base under that test's RUN. +# subtest of one), or by a pattern while the test logged the probe's tier=base skip message. unlisted=$(awk -v names="$names" -v pats="$patterns" ' /^=== RUN / { cur = $3; next } - /tier=base/ && cur != "" { base[cur] = 1 } + /^[[:space:]]+[A-Za-z0-9_.]+\.go:[0-9]+: .*io_uring tier=base kernel=/ && cur != "" { base[cur] = 1 } /^[[:space:]]*--- SKIP: / { n = $3 ok = 0 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 620258c43..99a462573 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2368,6 +2368,10 @@ jobs: # "none" and would skip everything, which the tally also refuses); # * the root package and ./test/server, the tests whose names say Shutdown, # Detach, SSE, Stream, WebSocket, Hijack or Drain, under -race. + # The -timeout of each step is about twice its measured time (473 s and 239 s), so a + # run with the cap lost (a misspelled value made 47 tests fail slowly and kept the + # second step busy for 23 minutes before it was cancelled, run 40416230192) ends + # in minutes. # Tests that need the high tier skip at this tier on purpose; each skip that # is allowed is listed by name with its reason in .github/base-tier-skips-iouring.txt # and -root.txt, and any other SKIP line fails the step (base-tier-tally.sh, @@ -2381,13 +2385,14 @@ jobs: # they skip with "no usable io_uring", so what this step adds for the root tests # is the tests that start an io_uring engine without such a probe: they run at # the base tier here and at the high tier in every other job. The root list names - # those arms by pattern (a new probe-gated test needs no edit; the log must say - # tier=base under the skipped arm) after run 40419578127, on a merge with main, + # those arms by pattern (a new probe-gated test needs no edit; the arm must have + # logged the probe's own "io_uring tier=base kernel=..." skip message) after run + # 40419578127, on a merge with main, # met nine new such tests (117 passed, 46 SKIP lines) and failed a list of names. iouring-base-tier: name: io_uring base tier (CELERIS_MAX_IOURING_TIER=base, celeris#890) runs-on: ubuntu-latest - timeout-minutes: 45 + timeout-minutes: 30 env: CELERIS_MAX_IOURING_TIER: base steps: @@ -2404,7 +2409,7 @@ jobs: sudo prlimit --pid "$$" --memlock=unlimited:unlimited echo "memlock (KiB): $(ulimit -l), CELERIS_MAX_IOURING_TIER=$CELERIS_MAX_IOURING_TIER" start=$SECONDS - CELERIS_REQUIRE_DUALSTACK=1 go test -race -count=1 -timeout=1500s -v ./internal/engine/iouring/ 2>&1 | tee /tmp/base-iouring.log + CELERIS_REQUIRE_DUALSTACK=1 go test -race -count=1 -timeout=900s -v ./internal/engine/iouring/ 2>&1 | tee /tmp/base-iouring.log echo "internal/engine/iouring at the base tier took $((SECONDS - start)) s" .github/scripts/base-tier-tally.sh iouring /tmp/base-iouring.log TestShutdownDetachedPartialWriteIsFlushed806 TestReportTier844 - name: Root package and test/server, Shutdown/Detach/SSE/Stream/WebSocket/Hijack/Drain tests at the base tier (race) @@ -2416,7 +2421,7 @@ jobs: echo "memlock (KiB): $(ulimit -l), CELERIS_MAX_IOURING_TIER=$CELERIS_MAX_IOURING_TIER" start=$SECONDS # -p 1: one test process at a time, so the runner's one memlock budget never funds two. - CELERIS_REQUIRE_DUALSTACK=1 go test -p 1 -race -count=1 -timeout=1500s -v -run 'Shutdown|Detach|SSE|Stream|WebSocket|Hijack|Drain' . ./test/server 2>&1 | tee /tmp/base-root.log + CELERIS_REQUIRE_DUALSTACK=1 go test -p 1 -race -count=1 -timeout=600s -v -run 'Shutdown|Detach|SSE|Stream|WebSocket|Hijack|Drain' . ./test/server 2>&1 | tee /tmp/base-root.log echo "root and test/server at the base tier took $((SECONDS - start)) s" .github/scripts/base-tier-tally.sh root /tmp/base-root.log diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 87d48e2a1..25959bfb6 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -183,6 +183,14 @@ # The label gate in loadgen's workflow is the other half of the decision and # is not in this repository. # +# Unproven, and to be checked after the merge: CodSpeed's GitHub Actions page +# (https://codspeed.io/docs/integrations/ci/github-actions) says that pushes to +# the default branch record the baseline and names workflow_dispatch for +# backtests; it does not mention `schedule`. Look at the first scheduled run in +# the CodSpeed dashboard; if it is not recorded as the main baseline, put +# `push: branches: [main]` (with the paths below) back beside the schedule, or +# have the schedule dispatch this workflow. +# # The trigger paths below are the non-test code of every package the # benchmark set executes (measured: the set run once with -coverpkg over the # whole module, on Linux and macOS), the benchmark files, the root From a136253a5c469ec08ff7220f310035a1f2499872 Mon Sep 17 00:00:00 2001 From: Albert Bausili Date: Tue, 27 Oct 2026 01:48:33 +0100 Subject: [PATCH 12/14] ci: self-test rows that fail on the scripts of 34382216 (review r1 of #1124, m1 m2 m4 n1) base-tier-tally-test.sh: a probe message of one parallel arm must not excuse another arm's skip (=== NAME / === CONT lines), parallel arms that each logged it pass, another tier=base message is no evidence, and a regular expression character in an allowed name is literal. tool-pins-check-test.sh: uvx name@version, uv tool run name@version and range pins (>=, ~=, <) are flagged, an @ that is no version and a redirection are not. Run against the previous scripts: 3 and 5 rows fail (Refs #890). --- .github/scripts/base-tier-tally-test.sh | 24 +++++++++++++++++++++++- .github/scripts/tool-pins-check-test.sh | 24 ++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/.github/scripts/base-tier-tally-test.sh b/.github/scripts/base-tier-tally-test.sh index 8905f98a8..62bc9e041 100755 --- a/.github/scripts/base-tier-tally-test.sh +++ b/.github/scripts/base-tier-tally-test.sh @@ -13,7 +13,7 @@ trap 'rm -rf "$dir"' EXIT fail=0 n=0 export BASE_TIER_SKIPS_DIR=$dir -printf '# header\nmin-pass 2\nTestNeedsHighTier needs a provided buffer ring\n' >"$dir/base-tier-skips-iouring.txt" +printf '# header\nmin-pass 2\nTestNeedsHighTier needs a provided buffer ring\nTestDot.Name a name with a regular expression character\n' >"$dir/base-tier-skips-iouring.txt" printf '# header\nmin-pass 1\npattern:^.+/(io_uring|adaptive)$ the arms of probe-gated tests\nTestNeedsHighTier needs a provided buffer ring\n' >"$dir/base-tier-skips-root.txt" # expect NAME WANT_STATUS WANT_STDERR_SUBSTRING LABEL LOG [TESTS...] @@ -49,6 +49,11 @@ sed 's/NeedsHighTier/NeedsSomething/' "$dir/good.log" >"$dir/unlisted.log" expect "a skip nobody allowed fails" 1 "TestNeedsSomething" iouring "$dir/unlisted.log" TestShutdownDetachedPartialWriteIsFlushed806 sed 's/tier=base arm=park/tier=high arm=park/' "$dir/good.log" >"$dir/high.log" expect "an arm at tier=high fails" 1 "the cap did not take effect" iouring "$dir/high.log" TestShutdownDetachedPartialWriteIsFlushed806 +# A name is literal: the dot of an allowed name is no wildcard. +{ cat "$dir/good.log"; printf '%s\n' '--- SKIP: TestDot.Name (0.00s)' '--- SKIP: TestDotXName (0.00s)'; } >"$dir/dot.log" +expect "a regular expression character in an allowed name is literal" 1 "TestDotXName" iouring "$dir/dot.log" +grep -v 'TestDotXName' "$dir/dot.log" >"$dir/dot-ok.log" +expect "the allowed name itself passes" 0 "" iouring "$dir/dot-ok.log" grep -v 'celeris844 PROBE' "$dir/good.log" >"$dir/noprobe.log" expect "a missing capped probe line fails" 1 "the cap did not take effect" iouring "$dir/noprobe.log" TestShutdownDetachedPartialWriteIsFlushed806 sed 's/CELERIS_MAX_IOURING_TIER="base"/CELERIS_MAX_IOURING_TIER=""/' "$dir/good.log" >"$dir/nocap.log" @@ -78,6 +83,23 @@ expect "root: an arm that logged tier=base is allowed by the pattern" 0 "" root printf '%s\n' '=== RUN TestP' '=== RUN TestP/io_uring' '2026/10/26 23:34:23 INFO io_uring engine selected tier=base multishot_accept=false' \ ' p_test.go:9: the peer never connected' '--- PASS: TestP (0.00s)' ' --- SKIP: TestP/io_uring (0.00s)' >"$dir/arm-engine.log" expect "root: an arm that started an engine and skipped for another reason fails" 1 "TestP/io_uring" root "$dir/arm-engine.log" +# Parallel subtests interleave their output and go test marks the switch with `=== NAME` +# (and `=== CONT` after a pause): a probe message printed under one arm must not be +# credited to another arm that skipped for a different reason (review r1 of #1124, m1). +printf '%s\n' '=== RUN TestP' '=== RUN TestP/io_uring' '=== PAUSE TestP/io_uring' '=== RUN TestQ' '=== RUN TestQ/io_uring' '=== PAUSE TestQ/io_uring' \ + '=== CONT TestP/io_uring' ' p_test.go:9: io_uring tier=base kernel=6.17: no usable io_uring' \ + '=== NAME TestQ/io_uring' ' q_test.go:12: listener never became ready' \ + '--- PASS: TestP (0.00s)' ' --- SKIP: TestP/io_uring (0.00s)' '--- PASS: TestQ (0.00s)' ' --- SKIP: TestQ/io_uring (0.00s)' >"$dir/arm-interleaved.log" +expect "root: a probe message of one parallel arm does not excuse another's skip" 1 "TestQ/io_uring" root "$dir/arm-interleaved.log" +printf '%s\n' '=== RUN TestP' '=== RUN TestP/io_uring' '=== PAUSE TestP/io_uring' '=== RUN TestP/adaptive' '=== PAUSE TestP/adaptive' \ + '=== CONT TestP/io_uring' ' p_test.go:9: io_uring tier=base kernel=6.17: no usable io_uring' \ + '=== CONT TestP/adaptive' ' p_test.go:9: io_uring tier=base kernel=6.17: no usable io_uring' \ + '--- PASS: TestP (0.00s)' ' --- SKIP: TestP/io_uring (0.00s)' ' --- SKIP: TestP/adaptive (0.00s)' >"$dir/arm-interleaved-ok.log" +expect "root: parallel arms that each logged the probe message are allowed" 0 "" root "$dir/arm-interleaved-ok.log" +# The pattern's evidence is the message the comments name, not any tier=base line of a probe: +# a Logf that merely says so, or another tier=base message, is no excuse (m2). +arm io_uring 'io_uring tier=base kernel=6.17: io_uring arms not run' >"$dir/arm-notrun.log" +expect "root: another tier=base message is not the probe's skip message" 1 "TestP/io_uring" root "$dir/arm-notrun.log" arm io_uring 'the kernel has no io_uring' >"$dir/arm-other.log" expect "root: an arm that skipped for another reason fails" 1 "TestP/io_uring" root "$dir/arm-other.log" arm epoll 'io_uring tier=base kernel=6.17: no usable io_uring' >"$dir/arm-epoll.log" diff --git a/.github/scripts/tool-pins-check-test.sh b/.github/scripts/tool-pins-check-test.sh index 5c80f216a..bc12418b4 100755 --- a/.github/scripts/tool-pins-check-test.sh +++ b/.github/scripts/tool-pins-check-test.sh @@ -167,6 +167,30 @@ expect pip-install-continuation 1 'ci.yml' fixture pip-install-composite printf ' - run: pipx install zizmor==1.30.0\n shell: bash\n' >>"$dir/pip-install-composite/.github/actions/probe/action.yml" expect pip-install-composite 1 'actions/probe/action.yml' +# Pin forms other than ==: uv's name@version, and range pins (review r1 of #1124, m4). +fixture uvx-at-pin +printf ' - run: uvx zizmor@1.30.0 .github/workflows\n' >>"$dir/uvx-at-pin/.github/workflows/ci.yml" +expect uvx-at-pin 1 'ci.yml' +fixture uv-tool-run-at-pin +printf ' - run: uv tool run zizmor@v1.30.0 .\n' >>"$dir/uv-tool-run-at-pin/.github/workflows/ci.yml" +expect uv-tool-run-at-pin 1 'ci.yml' +fixture pip-range-pin +printf ' - run: pip install "zizmor>=1.30,<1.31"\n' >>"$dir/pip-range-pin/.github/workflows/ci.yml" +expect pip-range-pin 1 'ci.yml' +fixture pip-compatible-pin +printf ' - run: pipx run --spec zizmor~=1.30 zizmor .\n' >>"$dir/pip-compatible-pin/.github/workflows/ci.yml" +expect pip-compatible-pin 1 'ci.yml' +fixture pip-upper-pin +printf ' - run: python3 -m pip install "zizmor<2"\n' >>"$dir/pip-upper-pin/.github/workflows/ci.yml" +expect pip-upper-pin 1 'ci.yml' +# An @ that is no version (a path, a mail address) and a plain redirection are fine. +fixture uvx-at-path +printf ' - run: uvx zizmor .github/workflows 2>&1 | tee log@host.txt\n' >>"$dir/uvx-at-path/.github/workflows/ci.yml" +expect uvx-at-path 0 +fixture pip-redirect +printf ' - run: pip install -r .github/tools/requirements.txt >/dev/null\n' >>"$dir/pip-redirect/.github/workflows/ci.yml" +expect pip-redirect 0 + # Reading the pins from a requirements file, and a comment, are fine. fixture pip-install-requirements printf ' - run: pip install -r .github/tools/requirements.txt\n' >>"$dir/pip-install-requirements/.github/workflows/ci.yml" From a979806d3f40a19fae89797c904dec5468e94a8f Mon Sep 17 00:00:00 2001 From: Albert Bausili Date: Tue, 27 Oct 2026 01:48:33 +0100 Subject: [PATCH 13/14] ci, test: the tally follows NAME/CONT and wants the probe's message, check 4 sees range and uv pins, the integration guard scans helpers_test.go, closed PRs do not start CodSpeed, discovery for the full-size job (review r1 of #1124) - m1, m2, n1: base-tier-tally.sh credits a probe message to the test that is current on RUN, NAME and CONT lines, the evidence is the 'no usable io_uring' message the comments name, allowed names are compared as strings. The head's real root log (117 PASS, 46 SKIP, 100 NAME/CONT/PAUSE lines) still passes. - m4: tool-pins-check.sh check 4 flags >=, ~=, <, > pins and uvx/uv tool name@version. - m3, n2: the skip guard scans helpers_test.go (startEngine, three raw skips, registered as debt) and does not take a method for the gate function. - m6: the CodSpeed job runs only for an open pull request. - m5: the comments of the two new jobs say they gate releases. - m7: the full-size job fails when a Test function that reads the lean-shape helpers is not in its list. - M2: codspeed.yml's header names the fallbacks and the date of the quota reset. (Refs #890) --- .github/base-tier-skips-root.txt | 2 +- .github/scripts/base-tier-tally.sh | 17 ++++++++++------- .github/scripts/tool-pins-check.sh | 16 +++++++++++----- .github/workflows/ci.yml | 19 +++++++++++++++++-- .github/workflows/codspeed.yml | 15 +++++++++++---- test/integration/skip_routing_890_test.go | 23 +++++++++++++++++------ 6 files changed, 67 insertions(+), 25 deletions(-) diff --git a/.github/base-tier-skips-root.txt b/.github/base-tier-skips-root.txt index da1b45c28..c064634b4 100644 --- a/.github/base-tier-skips-root.txt +++ b/.github/base-tier-skips-root.txt @@ -12,7 +12,7 @@ min-pass 100 # same tests run. A new test of this kind joins them without an edit here, because the # entry names the arms, not the tests (run 40419578127 met nine new ones that a merge # with main brought in: the TestH1Unclosed...889 family); the arm must have logged the -# probe's own skip message ("io_uring tier=base kernel=...") under its RUN line, so an +# probe's own skip message ("io_uring tier=base kernel=...") while it was the current test, so an # arm that starts an engine and skips for another reason still fails the step. pattern:^.+/(io_uring|adaptive|adaptive-iouring)$ the io_uring, adaptive and adaptive-iouring arms of probe-gated tests diff --git a/.github/scripts/base-tier-tally.sh b/.github/scripts/base-tier-tally.sh index f3993acbf..82940b4a3 100755 --- a/.github/scripts/base-tier-tally.sh +++ b/.github/scripts/base-tier-tally.sh @@ -36,9 +36,9 @@ here=$(cd "$(dirname "$0")" && pwd) # self-test). An entry is # NAME a test or subtest by its full name; it covers its own subtests, or # pattern:REGEX a skipped test or subtest whose full name matches REGEX, and only -# when the test itself logged the probe's skip message under its RUN -# line ("file_test.go:N: io_uring tier=base kernel=...: no usable -# io_uring"). The engine's own startup line ("INFO io_uring engine +# when the test itself logged the probe's skip message while it was the +# current test ("file_test.go:N: io_uring tier=base kernel=...: no +# usable io_uring"). The engine's own startup line ("INFO io_uring engine # selected tier=base ...") does not count: an arm that started an # engine and then skipped for another reason must still fail. # min-pass N the top-level tests that must PASS at least, so that a -run pattern @@ -50,6 +50,7 @@ case "$label" in esac list_file=${BASE_TIER_SKIPS_DIR:-$here/..}/base-tier-skips-$label.txt [ -r "$list_file" ] || { echo "base-tier-tally: cannot read $list_file" >&2; exit 2; } +# Names are literal (compared as strings below, not as regular expressions). names=$(awk 'NF && $1 !~ /^#/ && $1 != "min-pass" && $1 !~ /^pattern:/ { print $1 }' "$list_file" | paste -sd'|' -) patterns=$(awk 'NF && $1 ~ /^pattern:/ { sub(/^pattern:/, "", $1); print $1 }' "$list_file" | paste -sd'|' -) minpass=$(awk '$1 == "min-pass" { print $2 }' "$list_file") @@ -58,13 +59,15 @@ fail=0 skipped=$("$here"/skip-count.sh "$log") || fail=1 # Each SKIP line is judged by the name of the test: allowed by an exact entry (or as a # subtest of one), or by a pattern while the test logged the probe's tier=base skip message. -unlisted=$(awk -v names="$names" -v pats="$patterns" ' - /^=== RUN / { cur = $3; next } - /^[[:space:]]+[A-Za-z0-9_.]+\.go:[0-9]+: .*io_uring tier=base kernel=/ && cur != "" { base[cur] = 1 } +unlisted=$(awk -v names="$names" -v pats="$patterns" 'BEGIN { nn = (names == "") ? 0 : split(names, nm, "|") } + # Parallel subtests interleave their output; go test names the owner of the lines that + # follow with RUN, and with NAME or CONT when it switches back to a test. + /^=== (RUN|NAME|CONT) / { cur = $3; next } + /^[[:space:]]+[A-Za-z0-9_.]+\.go:[0-9]+: .*io_uring tier=base kernel=.*: no usable io_uring/ && cur != "" { base[cur] = 1 } /^[[:space:]]*--- SKIP: / { n = $3 ok = 0 - if (names != "" && (n ~ ("^(" names ")$") || n ~ ("^(" names ")/"))) ok = 1 + for (i = 1; i <= nn; i++) if (n == nm[i] || index(n, nm[i] "/") == 1) ok = 1 if (!ok && pats != "" && n ~ pats && base[n]) ok = 1 if (!ok) print } diff --git a/.github/scripts/tool-pins-check.sh b/.github/scripts/tool-pins-check.sh index fe5f04fe4..ec756ea17 100755 --- a/.github/scripts/tool-pins-check.sh +++ b/.github/scripts/tool-pins-check.sh @@ -25,8 +25,8 @@ # 4. The Python tools of CI (zizmor) are pinned in a requirements file under # .github/, which Dependabot's pip ecosystem reads. (a) No workflow or # composite action installs or runs a Python tool with a `name==version` -# of its own (`pipx run --spec zizmor==1.30.0`, `pip install x==1`, `uvx -# x==1`): that pin is read by no Dependabot entry, and stayed on its +# (or `>=`, `~=`, `<`, `uvx name@1`) of its own (`pipx run --spec zizmor==1.30.0`, +# `pip install x==1`, `uvx x==1`): that pin is read by no Dependabot entry, and stayed on its # version until somebody remembered (celeris#890 item 93). The same line # rules as check 3 apply. (b) Every directory under .github/ that holds a # requirements*.txt has a `package-ecosystem: pip` entry in dependabot.yml @@ -127,9 +127,15 @@ import glob, os, re, sys root = sys.argv[1] pats = ['.github/workflows/*.yml', '.github/workflows/*.yaml', '.github/actions/**/*.yml', '.github/actions/**/*.yaml'] -# A Python tool's installer or runner, then (flags) name==version. pip's -r -# FILE form has no ==, so reading the pins from the file is not flagged. -pat = re.compile(r'(?:\bpipx\s+(?:run|install)\b|\bpip3?\s+install\b|\bpython3?\s+-m\s+pip\s+install\b|\buvx\b|\buv\s+tool\s+(?:run|install)\b).*?[A-Za-z0-9_.-]+(?:\[[^]]*\])?==\S') +# A Python tool's installer or runner, then (flags) name==version or another +# specifier with a version (>=1.30,<1.31), or uv's name@version. pip's -r FILE +# form has none of them, so reading the pins from the file is not flagged. +installer = r'(?:\bpipx\s+(?:run|install)\b|\bpip3?\s+install\b|\bpython3?\s+-m\s+pip\s+install\b|\buvx\b|\buv\s+tool\s+(?:run|install)\b)' +# name==version, and the other specifiers of PEP 440 (~=, >=, <=, !=, <, >) with a version. +spec = r'[A-Za-z0-9_.-]+(?:\[[^]]*\])?(?:==|~=|>=|<=|!=|<|>)\s*[0-9]' +# uv's own pin, name@version, after uvx or uv tool (a path or a mail address has no digit or v after the @). +at = r'\b(?:uvx|uv\s+tool\s+(?:run|install))\b.*?\s[A-Za-z0-9_.-]+@v?[0-9]' +pat = re.compile(installer + r'.*?' + spec + '|' + at) files = sorted({f for p in pats for f in glob.glob(os.path.join(root, p), recursive=True)}) for f in files: lines = open(f).read().splitlines() diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 99a462573..492a7cbee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2351,7 +2351,9 @@ jobs: if [ "$skipped" -ne 0 ]; then echo "a SKIP is not a result ($skipped SKIP lines)"; bad=1; fi exit "$bad" - # NOT a required check (see the note above `iouring`). celeris#890 item 87: + # NOT a required check for merging (see the note above `iouring`), but a gate of + # releases: release.yml calls this workflow and tag-submodules needs its result to + # be success, so a red run of this job blocks the tag. celeris#890 item 87: # nothing ran the io_uring base tier. CELERIS_MAX_IOURING_TIER=base caps the # probe's tier at "base" (no provided buffer ring, single-shot accept, no # multishot recv), the tier a kernel without those features gets and the one @@ -2425,7 +2427,8 @@ jobs: echo "root and test/server at the base tier took $((SECONDS - start)) s" .github/scripts/base-tier-tally.sh root /tmp/base-root.log - # NOT a required check. celeris#890 item 92: the large-response tests of + # NOT a required check for merging, but a gate of releases (release.yml calls this + # workflow and tag-submodules needs its result to be success). celeris#890 item 92: the large-response tests of # ./test/server (celeris#761, #817, #836) have a lean shape and a full-size # one. lean761() (large_response_linux_test.go) is true under -race and # under coverage, which are the only ways the other jobs run ./test/server, @@ -2466,6 +2469,18 @@ jobs: TestH2CUnmatchedThroughTimeoutSingleflight836 ) regex=$(IFS='|'; echo "${tests[*]}") + # Discovery: a Test function of ./test/server that reads lean761(), raceOn761, n836() or + # dur836() runs a lean shape everywhere else, so it must be in the list above or its + # full-size shape runs nowhere (the failure this job exists for). A helper that is not a + # Test function and reads them is not seen: its callers are not found this way. + used=$(awk 'FNR == 1 { fn = "" } /^func / { fn = $2; sub(/\(.*/, "", fn) } /lean761\(\)|raceOn761|n836\(|dur836\(/ && fn ~ /^Test/ { print fn }' test/server/*_test.go | sort -u) + listed=$(printf '%s\n' "${tests[@]}" | sort -u) + if [ "$used" != "$listed" ]; then + echo "tests of ./test/server that read the lean-shape helpers (left) against the list of this step (right):" + diff <(echo "$used") <(echo "$listed") || true + echo "add the test to the list (or take it out when it no longer reads a helper)" + exit 1 + fi go test -c -o /tmp/server.test ./test/server python3 - /tmp/server.test "^(${regex})\$" <<'PY' 2>&1 | tee /tmp/large.log import resource, subprocess, sys, time diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 25959bfb6..20a95ca3e 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -187,9 +187,13 @@ # (https://codspeed.io/docs/integrations/ci/github-actions) says that pushes to # the default branch record the baseline and names workflow_dispatch for # backtests; it does not mention `schedule`. Look at the first scheduled run in -# the CodSpeed dashboard; if it is not recorded as the main baseline, put -# `push: branches: [main]` (with the paths below) back beside the schedule, or -# have the schedule dispatch this workflow. +# the CodSpeed dashboard (not before the quota resets on 2026-10-31; until then +# no run gets a runner); if it is not recorded as the main baseline, either +# bring `push: branches: [main]` (with the paths below) back beside the schedule, +# or keep the cron but have it start `gh workflow run codspeed.yml --ref main` +# (a `workflow_dispatch` run, which CodSpeed documents), for example from a small +# hosted workflow with `actions: write`. Either way a pull request then compares +# with a main run of the day, not with an older report. # # The trigger paths below are the non-test code of every package the # benchmark set executes (measured: the set run once with -coverpkg over the @@ -280,7 +284,9 @@ jobs: name: Benchmarks (arm64 macro runner) # A pull request runs only with the `performance` label, and the label's # own event (celeris#890 item 7): the run that adding another label starts - # skips the job. Fork pull requests are skipped as this file is written + # skips the job, and so does a label added to a closed or merged pull request + # (a release-notes clean-up after the merge must not spend the quota). Fork pull + # requests are skipped as this file is written # (the approval policy in the header is what holds against a fork that # edits it), and so are drafts, which wait until they are marked ready # (ready_for_review above), to save minutes. Every other event (the @@ -288,6 +294,7 @@ jobs: if: >- github.event_name != 'pull_request' || (github.event.pull_request.head.repo.full_name == github.repository && + github.event.pull_request.state == 'open' && !github.event.pull_request.draft && contains(github.event.pull_request.labels.*.name, 'performance') && (github.event.action != 'labeled' || github.event.label.name == 'performance')) diff --git a/test/integration/skip_routing_890_test.go b/test/integration/skip_routing_890_test.go index 350851b75..483b3aa80 100644 --- a/test/integration/skip_routing_890_test.go +++ b/test/integration/skip_routing_890_test.go @@ -25,7 +25,7 @@ import ( // honest is this guard, the counterpart of internal/adaptive's // TestAdaptiveSkipRouting684. // -// It parses every adaptive_*_test.go file of the package and lists each t.Skip, +// It parses every adaptive_*_test.go file of the package, and helpers_test.go, and lists each t.Skip, // t.Skipf and t.SkipNow call that is none of: // // - inside a gate helper (gateHelpers890); @@ -49,9 +49,9 @@ var gateHelpers890 = map[string]bool{ // message and why they stay. The count is exact: a further site with a // registered message fails the guard as a stale entry would. // -// All ten are in adaptive_hybrid_test.go. #890 item 95 leaves the adaptive +// Ten are in adaptive_hybrid_test.go and three in helpers_test.go (startEngine). #890 item 95 leaves the adaptive // skips of this package to the lane that owns that file; the table is the count -// of what is left (the guard listed these ten and one more, in +// of what is left (the guard listed these ten, startEngine's three and one more, in // adaptive_standby_886_linux_test.go, which now goes through the gate), and a // new one cannot join them unseen. type unrouted890Entry struct { @@ -70,6 +70,9 @@ var unrouted890 = map[string]unrouted890Entry{ "adaptive_hybrid_test.go|engine Listen returned nil without addr": {1, startFail890}, "adaptive_hybrid_test.go|engine did not start in time": {1, startFail890}, "adaptive_hybrid_test.go|engine not functional: %v": {1, startFail890}, + // startEngine, which the adaptive tests and the epoll arms of the slowloris and pipeline tests share. + "helpers_test.go|engine failed to start: %v": {2, startFail890}, + "helpers_test.go|engine not functional (skipping): %v": {1, startFail890}, } // rawSkipViolations890 lists the t.Skip* calls of one Go source that the rules @@ -94,7 +97,11 @@ func rawSkipViolations890(name string, src []byte) ([]string, error) { case *ast.FuncDecl: if x != n { if x.Body != nil { // a bodyless declaration holds no call - walk(x.Body, x.Name.Name, short) + name := x.Name.Name + if x.Recv != nil { // a method of that name is not the gate function + name = "(method)" + name + } + walk(x.Body, name, short) } return false } @@ -196,13 +203,16 @@ func TestAdaptiveSkipRoutingIntegration890(t *testing.T) { if err != nil { t.Fatal(err) } + // helpers_test.go: startEngine skips when Listen fails, and the adaptive tests call it + // (review r1 of #1124, m3). + files = append(files, "helpers_test.go") // A glob that finds nothing, or not the files this guard exists for, passes // vacuously. have := map[string]bool{} for _, f := range files { have[f] = true } - for _, must := range []string{"adaptive_standby_guard_886_linux_test.go", "adaptive_hybrid_test.go", "adaptive_standby_886_linux_test.go"} { + for _, must := range []string{"adaptive_standby_guard_886_linux_test.go", "adaptive_hybrid_test.go", "adaptive_standby_886_linux_test.go", "helpers_test.go"} { if !have[must] { t.Fatalf("the guard did not see %s among %d adaptive test files: it would pass without checking it", must, len(files)) } @@ -234,7 +244,7 @@ func TestAdaptiveSkipRoutingIntegration890(t *testing.T) { return true }) } - t.Logf("%d adaptive test files, %d raw skips not admitted, %d registered in unrouted890", len(files), len(bad), len(unrouted890)) + t.Logf("%d scanned test files, %d raw skips not admitted, %d registered in unrouted890", len(files), len(bad), len(unrouted890)) if len(bad) > 0 { t.Errorf("t.Skip calls that bypass skipOrFailUpswitch890 (adaptive_standby_guard_886_linux_test.go) and are not registered in unrouted890:\n %s", strings.Join(bad, "\n ")) @@ -274,6 +284,7 @@ func TestAdaptiveSkipRoutingGuardSelfTest890(t *testing.T) { {"a Skip method value fails", head + `func TestX(t *testing.T) { s := t.Skip; s("io_uring unavailable") }`, 1}, {"a SkipNow method value passed as an argument fails", head + `func TestX(t *testing.T) { run(t.SkipNow) }`, 1}, {"a Skip method value under a -short gate passes", head + `func TestX(t *testing.T) { if testing.Short() { s := t.Skip; s("integration") } }`, 0}, + {"a method named like the gate is not the gate", head + `func (s *S) skipOrFailUpswitch890(t *testing.T, m string) { t.Skip(m) }`, 1}, {"a bodyless function declaration is not a violation or a panic", head + `func foo()`, 0}, {"input that does not parse is an error, not a pass", head + `func TestX(t *testing.T) { t.Skip(`, -1}, {"empty input is an error, not a pass", ``, -1}, From bc707c6283543cec8f6176153e5114729ccab6f2 Mon Sep 17 00:00:00 2001 From: Albert Bausili Date: Tue, 27 Oct 2026 03:40:28 +0100 Subject: [PATCH 14/14] ci: the base-tier job sets CELERIS_REQUIRE_ZONED_PEER=1, floors from the merged tree (Refs #890 item 87) Main 8936e89 (with #1123 and #1127) run through the job's logic in docker at CELERIS_MAX_IOURING_TIER=base: the iouring package gives 577 top-level PASS and 13 SKIP lines, the new one being TestListenOnAZonedLinkLocalServes885 (the container has no fe80:: address); root and test/server give 118 PASS and 46 SKIP lines, all allowed. The zoned test fails instead of skipping under CELERIS_REQUIRE_ZONED_PEER=1, as in the unit job, so both steps set it and the list stays as it was. min-pass floors 540 and 105. --- .github/base-tier-skips-iouring.txt | 8 ++++++-- .github/base-tier-skips-root.txt | 3 ++- .github/workflows/ci.yml | 7 +++++-- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/.github/base-tier-skips-iouring.txt b/.github/base-tier-skips-iouring.txt index febc6f8bc..63a6af39d 100644 --- a/.github/base-tier-skips-iouring.txt +++ b/.github/base-tier-skips-iouring.txt @@ -1,8 +1,12 @@ # Tests that skip at CELERIS_MAX_IOURING_TIER=base on purpose, for .github/scripts/base-tier-tally.sh # (celeris#890 item 87): the full test or subtest name, then why. A name covers its own # subtests. Measured on the run of the job iouring-base-tier cited in the pull request: -# 554 top-level tests passed, 4 skipped at the top level, 12 SKIP lines in all. -min-pass 500 +# 554 top-level tests passed, 4 skipped at the top level, 12 SKIP lines in all. On main +# 8936e89 (with #1123 and #1127) a docker run (kernel without an fe80:: address) gave 577 +# passed and 13 SKIP lines, the 13th being TestListenOnAZonedLinkLocalServes885, which +# the job does not allow: it sets CELERIS_REQUIRE_ZONED_PEER=1, so the test fails rather +# than skips on a runner without one. +min-pass 540 # Tier base has no multishot recv with provided buffers (the tests say so in their skip). TestSpentDetectBudgetReturnsItsRingBuffer/check no provided buffer ring at the base tier diff --git a/.github/base-tier-skips-root.txt b/.github/base-tier-skips-root.txt index c064634b4..93e24be4e 100644 --- a/.github/base-tier-skips-root.txt +++ b/.github/base-tier-skips-root.txt @@ -3,7 +3,8 @@ # subtests. Measured on the run of the job iouring-base-tier cited in the pull request: # 107 top-level tests passed, 3 skipped at the top level, 19 SKIP lines in all (and # 117 passed with 46 SKIP lines once main had brought in nine more tests of the kind below). -min-pass 100 +# On main 8936e89 (docker, merged tree): 118 passed, 46 SKIP lines, all allowed. +min-pass 105 # The probes of the tests that gate their io_uring arms (c714ProbeIOUring, # keptProbeIOUring, h2_handler_return_889's) call io_uring "usable" only at the high diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b44ba22d9..3b595e8e0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2377,6 +2377,9 @@ jobs: # run with the cap lost (a misspelled value made 47 tests fail slowly and kept the # second step busy for 23 minutes before it was cancelled, run 40416230192) ends # in minutes. + # CELERIS_REQUIRE_ZONED_PEER=1, as in the unit job: the zoned link-local tests (celeris#885, + # #1127) fail rather than skip on a runner without an fe80:: address, so they need no + # allow-list entry. # Tests that need the high tier skip at this tier on purpose; each skip that # is allowed is listed by name with its reason in .github/base-tier-skips-iouring.txt # and -root.txt, and any other SKIP line fails the step (base-tier-tally.sh, @@ -2414,7 +2417,7 @@ jobs: sudo prlimit --pid "$$" --memlock=unlimited:unlimited echo "memlock (KiB): $(ulimit -l), CELERIS_MAX_IOURING_TIER=$CELERIS_MAX_IOURING_TIER" start=$SECONDS - CELERIS_REQUIRE_DUALSTACK=1 go test -race -count=1 -timeout=900s -v ./internal/engine/iouring/ 2>&1 | tee /tmp/base-iouring.log + CELERIS_REQUIRE_DUALSTACK=1 CELERIS_REQUIRE_ZONED_PEER=1 go test -race -count=1 -timeout=900s -v ./internal/engine/iouring/ 2>&1 | tee /tmp/base-iouring.log echo "internal/engine/iouring at the base tier took $((SECONDS - start)) s" .github/scripts/base-tier-tally.sh iouring /tmp/base-iouring.log TestShutdownDetachedPartialWriteIsFlushed806 TestReportTier844 - name: Root package and test/server, Shutdown/Detach/SSE/Stream/WebSocket/Hijack/Drain tests at the base tier (race) @@ -2426,7 +2429,7 @@ jobs: echo "memlock (KiB): $(ulimit -l), CELERIS_MAX_IOURING_TIER=$CELERIS_MAX_IOURING_TIER" start=$SECONDS # -p 1: one test process at a time, so the runner's one memlock budget never funds two. - CELERIS_REQUIRE_DUALSTACK=1 go test -p 1 -race -count=1 -timeout=600s -v -run 'Shutdown|Detach|SSE|Stream|WebSocket|Hijack|Drain' . ./test/server 2>&1 | tee /tmp/base-root.log + CELERIS_REQUIRE_DUALSTACK=1 CELERIS_REQUIRE_ZONED_PEER=1 go test -p 1 -race -count=1 -timeout=600s -v -run 'Shutdown|Detach|SSE|Stream|WebSocket|Hijack|Drain' . ./test/server 2>&1 | tee /tmp/base-root.log echo "root and test/server at the base tier took $((SECONDS - start)) s" .github/scripts/base-tier-tally.sh root /tmp/base-root.log