From 1f9c7c007c027886aaa8df2fef6fc0d1f82ce5e7 Mon Sep 17 00:00:00 2001 From: lmoresi Date: Tue, 6 Oct 2026 14:12:10 +1100 Subject: [PATCH 1/2] scripts/triage.py: what is actually blocking the issue and PR queues A 133-issue backlog and a 32-PR queue turned out to be two different failures wearing the same clothes, and neither was visible without cross-referencing by hand. This does the cross-reference in one command, read-only. Three things it reports, each because it was expensive to learn the first time. Issues referenced by a merged PR, as candidates to PROBE and never as a verdict. Fifteen open issues were already fixed on development; the ones that closed cleanly all had a regression test naming the issue, and the ones that did not were references to a NEIGHBOURING defect. So the tool says "probe these" and refuses to rank them any further. PRs bucketed by mergeable x CI, with what each bucket closes, and the count that carry no Closes line at all -- 25 of 32 today, which is the whole reason a merge stopped resolving anything. And the one that cost ten days: a red PR whose failing tests no longer exist on the base. Four PRs sat red on two tests that development had already renamed, with byte-identical numbers across all four. The tool pulls the failing names out of the CI log and classifies each: phantom (the file is on the base, the test is not -- merge the base and it clears), new (the file came with the PR, so the failure is its own and real), or live. That distinction is not cosmetic -- the first version reported #800's own new test_1103_navier_stokes_velocity_transport.py as a phantom, which would have sent someone to merge the base and find nothing fixed. gh's GraphQL endpoint returns a bare 502 often enough that the calls retry; a tool meant to be run several times a session cannot treat one as fatal. Underworld development team with AI support from Claude Code --- scripts/triage.py | 249 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 249 insertions(+) create mode 100755 scripts/triage.py diff --git a/scripts/triage.py b/scripts/triage.py new file mode 100755 index 000000000..d0efbeae3 --- /dev/null +++ b/scripts/triage.py @@ -0,0 +1,249 @@ +#!/usr/bin/env python3 +"""What is actually blocking the issue and PR queues, in one command. + +Written because a 133-issue backlog and a 30-PR queue turned out to be two +different failures wearing the same clothes, and neither was visible without +cross-referencing by hand: + +* Fifteen issues were already fixed on ``development`` and nobody had closed + them. The signal is cheap -- an issue referenced by a merged PR -- but the + reference is often to a NEIGHBOUR, so this reports candidates to probe and + never a verdict. +* Four PRs sat red for ten days on two tests that ``development`` had already + fixed and renamed. Their failures named tests that no longer existed. That + is the ``phantom`` column, and it is the one worth looking at first. + +Read-only: it calls ``gh`` and ``git`` and writes nothing. Run it often; the +cost of the sweep is why it was not being done. + + python scripts/triage.py # the summary + python scripts/triage.py --verbose # every row behind the counts +""" + +import argparse +import collections +import json +import re +import subprocess +import sys +import time + +CLOSES = re.compile(r"(?:fix(?:e[sd])?|close[sd]?|resolve[sd]?)\s+#(\d{2,4})", re.I) +REF = re.compile(r"#(\d{2,4})") +#: A failing test pytest named, e.g. "tests/test_1060_x.py::TestC::test_y". +FAILED = re.compile(r"^FAILED\s+(\S+?)::(\S+?)(?:\[|\s|$)") + +BASE = "development" + + +def sh(*args, check=True, tries=1): + """Run a command, optionally retrying. Returns stdout. + + ``tries`` exists for the ``gh`` calls: the GraphQL API returns a bare 502 + often enough that a tool meant to be run several times a session cannot + treat one as fatal. + """ + for attempt in range(1, tries + 1): + out = subprocess.run(args, capture_output=True, text=True) + if out.returncode == 0: + return out.stdout + transient = any(code in out.stderr for code in ("502", "503", "504", + "timeout", "rate limit")) + if attempt < tries and transient: + wait = 2 ** attempt + print(f"triage: {out.stderr.strip().splitlines()[-1][:70]} " + f"-- retrying in {wait}s ({attempt}/{tries - 1})", file=sys.stderr) + time.sleep(wait) + continue + if check: + print(f"triage: `{' '.join(args[:3])}…` failed:\n{out.stderr}", file=sys.stderr) + sys.exit(1) + return out.stdout + return "" + + +def gh_json(*args): + return json.loads(sh("gh", *args, tries=4) or "[]") + + +def issues(): + return gh_json("issue", "list", "--limit", "400", "--json", + "number,title,labels,author,createdAt,body") + + +def prs(state="open"): + fields = ("number,title,mergeable,additions,deletions,changedFiles," + "statusCheckRollup,body,headRefName,author,updatedAt") + args = ["pr", "list", "--limit", "400", "--json", fields] + if state != "open": + args += ["--state", state] + return gh_json(*args) + + +def ci_state(pr): + outcomes = [c.get("conclusion") for c in (pr.get("statusCheckRollup") or [])] + if "FAILURE" in outcomes: + return "red" + if "SUCCESS" in outcomes: + return "green" + return "none" + + +def failing_test_names(pr): + """The (file, test) pairs this PR's failing check reported, if we can get them. + + Best-effort: the log is fetched only for red PRs, and a PR whose log has + aged out of GitHub's retention simply yields nothing. + """ + for check in pr.get("statusCheckRollup") or []: + if check.get("conclusion") != "FAILURE": + continue + url = check.get("detailsUrl") or "" + job = url.rsplit("/job/", 1)[-1] + if not job.isdigit(): + continue + log = sh("gh", "run", "view", "--log-failed", "--job", job, check=False) + found = set() + for line in log.splitlines(): + # The CI log prefixes each line with the step name and a timestamp. + hit = FAILED.search(line[line.find("FAILED"):]) if "FAILED" in line else None + if hit: + found.add((hit.group(1), hit.group(2))) + if found: + return found + return set() + + +def classify_on_base(path, name): + """Why a failing test might not reproduce on the base branch. + + Three outcomes, and only one of them is a phantom: + + ``"phantom"`` + The file is on the base but the test is not. It was renamed or + rewritten there, so the PR is red for a reason that no longer exists + and a merge clears it. This is the case that cost ten days. + ``"new"`` + The file is not on the base at all, so it arrived with the PR. The + failure is the PR's OWN new test and is real -- reporting it as a + phantom would send someone to merge the base and find nothing fixed. + (Caught exactly this way on #800's + ``test_1103_navier_stokes_velocity_transport.py``.) + ``"live"`` + Both file and test are on the base. The failure stands. + """ + body = sh("git", "show", f"origin/{BASE}:{path}", check=False) + if not body: + return "new" + leaf = name.split("::")[-1] + return "live" if f"def {leaf}" in body else "phantom" + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--verbose", "-v", action="store_true", + help="list every row behind the counts") + ap.add_argument("--no-logs", action="store_true", + help="skip CI log fetches (fast, drops the phantom column)") + args = ap.parse_args() + + sh("git", "fetch", "-q", "origin", BASE, check=False) + base_sha = sh("git", "rev-parse", "--short", f"origin/{BASE}").strip() + + iss = issues() + open_numbers = {i["number"] for i in iss} + titles = {i["number"]: i["title"] for i in iss} + opn = prs() + merged = prs("merged") + + print(f"=== triage against origin/{BASE} ({base_sha}) ===\n") + + # ---- issues ----------------------------------------------------------- + by_author = collections.Counter(i["author"]["login"] for i in iss) + unlabelled = sum(1 for i in iss if not i["labels"]) + print(f"ISSUES {len(iss)} open {unlabelled} unlabelled") + print(" by author: " + ", ".join(f"{a} {n}" for a, n in by_author.most_common())) + + # An issue a MERGED PR referenced is a candidate to probe, not a verdict: + # the reference is often to a neighbouring defect. + probe = collections.defaultdict(set) + for p in merged: + text = (p["body"] or "") + " " + p["title"] + for n in {int(m) for m in CLOSES.findall(text)}: + if n in open_numbers: + probe[n].add(("closes", p["number"])) + for n in {int(m) for m in REF.findall(text)}: + if n in open_numbers and n != p["number"]: + probe[n].add(("mentions", p["number"])) + strong = {n for n, src in probe.items() if any(k == "closes" for k, _ in src)} + print(f" {len(probe)} referenced by a merged PR -- PROBE these, do not close on the reference") + print(f" of which {len(strong)} have a merged PR claiming to close them") + if args.verbose: + for n in sorted(probe, key=lambda n: (n not in strong, n)): + kind = "closes " if n in strong else "mention" + src = ",".join(f"#{p}" for _, p in sorted(probe[n])) + print(f" #{n:<5} [{kind}] {src:<18} {titles[n][:60]}") + + # ---- PRs -------------------------------------------------------------- + print(f"\nPULL REQUESTS {len(opn)} open") + buckets = collections.defaultdict(list) + for p in opn: + buckets[(p["mergeable"], ci_state(p))].append(p) + for key in sorted(buckets, key=lambda k: (k[0] != "MERGEABLE", k[1] != "green")): + mergeable, ci = key + group = buckets[key] + closes = sorted({n for p in group + for n in {int(m) for m in CLOSES.findall(p["body"] or "")} + if n in open_numbers}) + tag = ", ".join(f"#{n}" for n in closes) or "nothing" + print(f" {mergeable:<12} CI {ci:<6} {len(group):>3} closes {tag}") + if args.verbose: + for p in sorted(group, key=lambda p: p["additions"] + p["deletions"]): + behind = sh("git", "rev-list", "--count", + f"origin/{p['headRefName']}..origin/{BASE}", check=False).strip() + size = p["additions"] + p["deletions"] + print(f" #{p['number']:<5} {size:>7}L {p['changedFiles']:>3}f " + f"behind {behind or '?':>4} {p['title'][:52]}") + + no_closes = [p for p in opn if not CLOSES.search(p["body"] or "")] + print(f" {len(no_closes)} of {len(opn)} carry no Closes line") + + # ---- the one that cost ten days --------------------------------------- + if args.no_logs: + return + red = [p for p in opn if ci_state(p) == "red"] + if not red: + return + print(f"\nRED PR FAILURES (checked against {BASE})") + any_phantom = False + for p in red: + groups = collections.defaultdict(list) + for path, test in failing_test_names(p): + groups[classify_on_base(path, test)].append((path, test)) + if not groups: + print(f" #{p['number']} no test names in the log (aged out, or the " + f"failure is not a test)") + continue + behind = sh("git", "rev-list", "--count", + f"origin/{p['headRefName']}..origin/{BASE}", check=False).strip() + if groups["phantom"]: + any_phantom = True + print(f" #{p['number']} behind {behind} -- MERGE {BASE} IN, " + f"{len(groups['phantom'])} failure(s) renamed away there:") + for path, test in sorted(groups["phantom"]): + print(f" phantom {path}::{test}") + for kind, note in (("new", "the PR's own new file -- a real failure"), + ("live", "still on the base -- a real failure")): + if groups[kind]: + if not groups["phantom"]: + print(f" #{p['number']} behind {behind}") + for path, test in sorted(groups[kind]): + print(f" {kind:<8} {path}::{test}") + print(f" ^ {note}") + if not any_phantom: + print(f" no phantoms -- every red PR fails on something that is really there") + + +if __name__ == "__main__": + main() From 41f609131e4c55b391072d299b1872fafc5f1219 Mon Sep 17 00:00:00 2001 From: lmoresi Date: Tue, 6 Oct 2026 22:00:05 +1100 Subject: [PATCH 2/2] Do not read a pasted sample as a closing keyword This script reports which issues a PR closes by scanning its body. Its own PR pastes its own output, which contains the line "closes #640, #747, #793" as sample text -- so it reported itself as closing seven issues it has nothing to do with. Fenced code blocks are now stripped before the scan. Whether GitHub's own linked-issue parser makes the same mistake is unresolved: `closingIssuesReferences` comes back empty through `gh` even for a PR whose body plainly says "Closes #793", so that field does not answer it either way. The sample output in the PR description has had its hashes removed so the question cannot arise. Underworld development team with AI support from Claude Code --- scripts/triage.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/scripts/triage.py b/scripts/triage.py index d0efbeae3..3f091de61 100755 --- a/scripts/triage.py +++ b/scripts/triage.py @@ -30,6 +30,16 @@ CLOSES = re.compile(r"(?:fix(?:e[sd])?|close[sd]?|resolve[sd]?)\s+#(\d{2,4})", re.I) REF = re.compile(r"#(\d{2,4})") +#: A fenced code block. Stripped before scanning for closing keywords, because +#: a PR that PASTES this tool's own output has "closes #640, #747" in its body +#: as sample text. Scanned naively, this script reported its own PR as closing +#: seven issues it has nothing to do with. +FENCE = re.compile(r"^```.*?^```", re.M | re.S) + + +def prose(body): + """A PR body with its fenced code blocks removed.""" + return FENCE.sub("", body or "") #: A failing test pytest named, e.g. "tests/test_1060_x.py::TestC::test_y". FAILED = re.compile(r"^FAILED\s+(\S+?)::(\S+?)(?:\[|\s|$)") @@ -169,7 +179,7 @@ def main(): # the reference is often to a neighbouring defect. probe = collections.defaultdict(set) for p in merged: - text = (p["body"] or "") + " " + p["title"] + text = prose(p["body"]) + " " + p["title"] for n in {int(m) for m in CLOSES.findall(text)}: if n in open_numbers: probe[n].add(("closes", p["number"])) @@ -194,7 +204,7 @@ def main(): mergeable, ci = key group = buckets[key] closes = sorted({n for p in group - for n in {int(m) for m in CLOSES.findall(p["body"] or "")} + for n in {int(m) for m in CLOSES.findall(prose(p["body"]))} if n in open_numbers}) tag = ", ".join(f"#{n}" for n in closes) or "nothing" print(f" {mergeable:<12} CI {ci:<6} {len(group):>3} closes {tag}") @@ -206,7 +216,7 @@ def main(): print(f" #{p['number']:<5} {size:>7}L {p['changedFiles']:>3}f " f"behind {behind or '?':>4} {p['title'][:52]}") - no_closes = [p for p in opn if not CLOSES.search(p["body"] or "")] + no_closes = [p for p in opn if not CLOSES.search(prose(p["body"]))] print(f" {len(no_closes)} of {len(opn)} carry no Closes line") # ---- the one that cost ten days ---------------------------------------