From 3ac81856eb0b9d5abaac6fde1b957ea857fcf191 Mon Sep 17 00:00:00 2001 From: avallete Date: Tue, 20 Oct 2026 16:46:14 +0200 Subject: [PATCH 1/5] ci: benchmark and plan-guard lint changes against base Every PR touching lints/, bin/compile.py or bench/ runs each lint and the full union on base and head, on a synthetic schema at three sizes and on five open-source Supabase projects at pinned commits. The check fails when a lint's rows change, it errors, its plan gains a per-row catalog scan, it reads more buffers, or it is consistently slower. Co-Authored-By: Claude Opus 5.5 --- .claude/skills/new-lint/SKILL.md | 1 + .github/workflows/bench.yml | 135 ++++++++++ README.md | 4 + bench/README.md | 55 ++++ bench/bench.py | 449 +++++++++++++++++++++++++++++++ bench/fixture/chunk.sql | 44 +++ bench/fixture/head.sql | 47 ++++ bench/fixture/tail.sql | 2 + bench/projects/projects.json | 7 + bench/projects/services-stub.sql | 108 ++++++++ 10 files changed, 852 insertions(+) create mode 100644 .github/workflows/bench.yml create mode 100644 bench/README.md create mode 100644 bench/bench.py create mode 100644 bench/fixture/chunk.sql create mode 100644 bench/fixture/head.sql create mode 100644 bench/fixture/tail.sql create mode 100644 bench/projects/projects.json create mode 100644 bench/projects/services-stub.sql diff --git a/.claude/skills/new-lint/SKILL.md b/.claude/skills/new-lint/SKILL.md index 5d05c5b4..c07cd511 100644 --- a/.claude/skills/new-lint/SKILL.md +++ b/.claude/skills/new-lint/SKILL.md @@ -283,6 +283,7 @@ Then check: - [ ] `docs/XXXX_.md` exists - [ ] `test/sql/XXXX_.sql` exists - [ ] `test/expected/XXXX_.out` exists +- [ ] `bench/fixture/` creates objects the new lint reports, so the benchmark measures it on a non-empty result --- diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml new file mode 100644 index 00000000..e65d55c0 --- /dev/null +++ b/.github/workflows/bench.yml @@ -0,0 +1,135 @@ +name: benchmark +on: + pull_request: + types: [opened, synchronize, reopened, labeled, unlabeled] + paths: + - "lints/**" + - "bin/compile.py" + - "bench/**" + - ".github/workflows/bench.yml" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: bench-${{ github.ref }} + cancel-in-progress: true + +jobs: + measure: + name: Measure (${{ matrix.target }}) + # Label events other than the two waivers would only repeat the last run. + if: >- + github.event_name != 'pull_request' + || !contains(fromJSON('["labeled","unlabeled"]'), github.event.action) + || contains(fromJSON('["perf-accepted","lint-results-changed"]'), github.event.label.name) + strategy: + fail-fast: false + matrix: + target: [synthetic, basejump, atomic-crm, wacrm, grida, carbon] + runs-on: ubuntu-latest + timeout-minutes: 20 + services: + postgres: + image: supabase/postgres:17.6.1.169 + env: + POSTGRES_PASSWORD: postgres + ports: ["5432:5432"] + options: >- + --health-cmd "pg_isready -U postgres -h localhost" + --health-interval 5s --health-timeout 5s --health-retries 20 + env: + PGHOST: localhost + PGPORT: "5432" + PGPASSWORD: postgres + steps: + - name: Checkout head + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + path: head + persist-credentials: false + ref: ${{ github.event.pull_request.head.sha || github.sha }} + + - name: Checkout base + if: github.event_name == 'pull_request' + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + path: base + persist-credentials: false + ref: ${{ github.event.pull_request.base.sha }} + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Measure + env: + TARGET: ${{ matrix.target }} + BASE: ${{ github.event_name == 'pull_request' && 'base' || '' }} + run: | + args=(--head head --out "results-$TARGET.json") + if [ -n "$BASE" ]; then args+=(--base "$BASE"); fi + if [ "$TARGET" != synthetic ]; then + migrations=$(python head/bench/bench.py fetch "$TARGET" --dest "project-$TARGET") + args+=(--project "$TARGET=$migrations") + fi + python head/bench/bench.py measure "${args[@]}" + + - uses: actions/upload-artifact@v4 + with: + name: bench-${{ matrix.target }} + path: results-${{ matrix.target }}.json + + verdict: + name: Verdict + needs: measure + if: always() && needs.measure.result != 'skipped' + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + steps: + - name: Checkout head + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + sparse-checkout: bench + persist-credentials: false + + - uses: actions/download-artifact@v4 + with: + pattern: bench-* + merge-multiple: true + + - name: Compare + env: + ACCEPT_PERF: ${{ contains(github.event.pull_request.labels.*.name, 'perf-accepted') && '--accept-perf' || '' }} + ACCEPT_RESULTS: ${{ contains(github.event.pull_request.labels.*.name, 'lint-results-changed') && '--accept-results' || '' }} + MEASURED: ${{ needs.measure.result }} + run: | + set +e + # shellcheck disable=SC2086 # the waiver flags are empty or one word + python3 bench/bench.py compare results-*.json --markdown report.md $ACCEPT_PERF $ACCEPT_RESULTS + status=$? + if [ "$MEASURED" != success ]; then + printf '\n**A measure job failed**, so some targets are missing above.\n' >> report.md + status=1 + fi + cat report.md >> "$GITHUB_STEP_SUMMARY" + exit $status + + - name: Comment on PR + # Fork PRs get a read-only token; they still have the job summary. + if: always() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && hashFiles('report.md') != '' + env: + GH_TOKEN: ${{ github.token }} + PR: ${{ github.event.pull_request.number }} + run: | + id=$(gh api "repos/$GITHUB_REPOSITORY/issues/$PR/comments" --paginate \ + --jq '.[] | select(.body | startswith("")) | .id' | head -n1) + if [ -n "$id" ]; then + gh api -X PATCH "repos/$GITHUB_REPOSITORY/issues/comments/$id" -F body=@report.md >/dev/null + else + gh api "repos/$GITHUB_REPOSITORY/issues/$PR/comments" -F body=@report.md >/dev/null + fi diff --git a/README.md b/README.md index 023ed88e..11e88f66 100644 --- a/README.md +++ b/README.md @@ -95,3 +95,7 @@ To run the test suite: ```sh docker rmi -f dockerfiles-test && SUPABASE_VERSION=15.1.1.13 docker-compose -f dockerfiles/docker-compose.yml run --rm test ``` + +## Benchmark + +PRs that change a lint are benchmarked against their base on a synthetic schema at three sizes and on real open-source Supabase projects. The check fails when findings change, a lint errors, or its plan, buffer reads or run time regress. See [bench/README.md](bench/README.md). diff --git a/bench/README.md b/bench/README.md new file mode 100644 index 00000000..ea93ebd9 --- /dev/null +++ b/bench/README.md @@ -0,0 +1,55 @@ +# Lint benchmark + +Times every lint, and the full `splinter.sql` union, on Supabase schemas, comparing a base checkout with a head checkout. CI runs it on every PR that touches `lints/`, `bin/compile.py` or `bench/`. Each target runs as its own job on `supabase/postgres`, and one PR comment collects the results and is updated on every run. + +## Targets + +- **synthetic**: `fixture/` built at 20, 300 and 2,000 tables. The sizes are cumulative, so one database serves all three. Every lint has findings at every size. +- **Real projects**: the open-source apps from the dogfooding corpus, fetched from GitHub at the commits pinned in `projects/projects.json`. Their `supabase/migrations` are applied in order on the bare image, after `projects/services-stub.sql` stands in for the tables storage-api, GoTrue and realtime would create. Each migration runs in one transaction; one that still fails is rolled back, skipped and listed in the report. + +| target | tables | policies | `pg_depend` rows | migrations applied | +| -- | --: | --: | --: | --: | +| synthetic / large | 2,197 | 2,736 | 64k | – | +| carbon | 520 | 1,501 | 41k | 1,230 / 1,230 | +| grida | 174 | 89 | 7k | 76 / 87 (timescaledb is not in the image) | +| wacrm | 113 | 114 | 5k | 54 / 54 | +| atomic-crm | 88 | 36 | 3k | 28 / 28 | +| basejump | 82 | 13 | 2k | 4 / 4 | + +Table counts include the image's own schemas. + +## What fails the check + +| Guard | Rule | Waiver label | +| -- | -- | -- | +| Lint errors on head | Any error | none | +| Results | Row count and md5 of the sorted rows (as `jsonb`, so column names count) differ from base | `lint-results-changed` | +| Plan | Head's plan gains a catalog scan that runs once per outer row and reads more than 50 blocks per loop, i.e. a whole catalog or index instead of an index probe | `perf-accepted` | +| Buffers | `EXPLAIN (ANALYZE, BUFFERS)` reads at least 10% and 100 blocks more than base | `perf-accepted` | +| Wall time | The median is at least 15% and 3 ms slower, and every head round is slower than every base round | `perf-accepted` | +| Base errors | The lint errors on base, so there is nothing to compare | `perf-accepted` | + +Buffer counts are deterministic, so they catch plan regressions that are too small to see in wall time on a noisy runner. The plan pass runs twice and keeps the second, so catalog-cache warm-up does not count. Wall time runs base and head in interleaved rounds on the same database, alternating which goes first, and is measured server-side around the query. + +## When you change the benchmark + +- When you add a lint, extend `fixture/` so it has findings. +- When you add a project, pin a commit in `projects/projects.json` and add it to the matrix in `.github/workflows/bench.yml`. + +## Run locally + +```bash +docker run -d --name splinter-bench -e POSTGRES_PASSWORD=postgres -p 54329:5432 supabase/postgres:17.6.1.169 +git worktree add /tmp/splinter-base origin/main +export PGHOST=localhost PGPORT=54329 PGPASSWORD=postgres +python3 bench/bench.py measure --base /tmp/splinter-base --out synthetic.json +python3 bench/bench.py compare synthetic.json +``` + +For a project, use a fresh container and: + +```bash +python3 bench/bench.py measure --base /tmp/splinter-base --project "carbon=$(python3 bench/bench.py fetch carbon --dest /tmp/carbon)" --out carbon.json +``` + +Without `--base` you get absolute numbers: only lint errors fail, and repeated catalog scans are listed. `compare` takes several result files. The schema load is not idempotent, so every `measure` needs a fresh container. diff --git a/bench/bench.py b/bench/bench.py new file mode 100644 index 00000000..a6a5f456 --- /dev/null +++ b/bench/bench.py @@ -0,0 +1,449 @@ +"""Benchmark splinter's lints on Supabase schemas, base vs head. + +measure: loads a schema into a supabase/postgres database, either the synthetic +fixture at increasing sizes or a real project's migrations, then times every lint +(and the full union) from both checkouts in interleaved rounds, so runner drift +hits both sides equally. It also records a digest of each lint's sorted rows and +an EXPLAIN (ANALYZE, BUFFERS) of each lint. + +fetch: downloads a project listed in projects/projects.json. + +compare: turns one or more measure outputs into a markdown report and an exit code. + +See bench/README.md. +""" + +import argparse +import importlib.util +import json +import os +import statistics +import subprocess +import sys +import time +from pathlib import Path + +HERE = Path(__file__).resolve().parent +FIXTURE = HERE / "fixture" +PROJECTS = HERE / "projects" +CHUNK = 200 +DEFAULT_SIZES = "small=20,medium=300,large=2000" +EXPOSED_SCHEMAS = "public, app, billing, pgmq_public, graphql_public" +QUOTE = "$splinter_bench$" +MARK = "@@splinter_bench@@" + +# Wall time regresses only when the medians differ by both bounds and no head +# round is as fast as any base round, so runner noise does not turn the check red. +REGRESSION_RATIO = 1.15 +REGRESSION_MIN_MS = 3.0 +# Buffer accesses do not depend on runner speed, so their bound is tighter. +BUFFERS_RATIO = 1.10 +BUFFERS_MIN = 100 +# A catalog scan repeated per outer row should be an index probe touching a few +# pages. More than this per loop means it reads a whole catalog or index each time. +REPEATED_SCAN_BLOCKS = 50 + +RUN_FN = """ +create or replace function splinter_bench.run(q text, out ms float8, out n bigint, out digest text) +language plpgsql set search_path = '' as $fn$ +declare t0 timestamptz; +begin + t0 := pg_catalog.clock_timestamp(); + -- to_jsonb keeps column names, so renaming an output column changes the digest. + execute pg_catalog.format( + 'select pg_catalog.count(*), pg_catalog.md5(coalesce(pg_catalog.string_agg(j::text, E''\\n'' order by j::text), '''')) from (select pg_catalog.to_jsonb(r) j from (%s) r) s', + q + ) into n, digest; + ms := extract(epoch from pg_catalog.clock_timestamp() - t0) * 1000; +end +$fn$; +""" + + +def psql(sql, user, check=True, variables=None, single_transaction=False): + args = ["psql", "-X", "-q", "-At", "-F", "|", "-U", user] + (["--single-transaction"] if single_transaction else []) + if check: + args += ["-v", "ON_ERROR_STOP=1"] + for k, v in (variables or {}).items(): + args += ["-v", f"{k}={v}"] + proc = subprocess.run(args, input=sql, capture_output=True, text=True) + if check and proc.returncode != 0: + sys.exit(f"psql failed:\n{proc.stderr}") + return proc + + +def load_lints(checkout): + """Per-lint query text, compiled the way that checkout's bin/compile.py compiles it.""" + spec = importlib.util.spec_from_file_location(f"compile_{abs(hash(checkout))}", checkout / "bin" / "compile.py") + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + lints = dict(mod.load_sql_files(checkout / "lints")) + lints["all"] = "\nunion all\n".join(lints.values()) + return mod.HEADER, lints + + +def build_fixture(lo, hi, admin): + if lo == 1: + psql((PROJECTS / "services-stub.sql").read_text(), admin) + psql((FIXTURE / "head.sql").read_text(), admin) + chunk = (FIXTURE / "chunk.sql").read_text() + for start in range(lo, hi + 1, CHUNK): + psql(chunk, admin, variables={"lo": start, "hi": min(start + CHUNK - 1, hi)}) + psql((FIXTURE / "tail.sql").read_text(), admin) + + +def apply_project(migrations, user, admin): + """Apply a project's migrations in order. A failing file is skipped and counted: real + projects lean on services (auth, storage) that the bare image only partly provides.""" + psql((PROJECTS / "services-stub.sql").read_text(), admin) + failed = [] + files = sorted(Path(migrations).glob("*.sql")) + for f in files: + # One transaction per file, so a failed migration leaves nothing half-applied. + proc = psql(f.read_text(), user, check=False, variables={"ON_ERROR_STOP": 1}, single_transaction=True) + if proc.returncode != 0: + errors = [l for l in proc.stderr.splitlines() if "ERROR" in l] or ["failed"] + failed.append(f"{f.name}: {errors[0].strip()}") + psql("analyze;", admin, check=False) + return {"migrations": len(files), "failed": failed} + + +def catalog_stats(user): + out = psql( + """ + select pg_catalog.json_build_object( + 'tables', (select count(*) from pg_catalog.pg_class where relkind in ('r', 'p')), + 'views', (select count(*) from pg_catalog.pg_class where relkind in ('v', 'm')), + 'indexes', (select count(*) from pg_catalog.pg_class where relkind = 'i'), + 'functions', (select count(*) from pg_catalog.pg_proc), + 'policies', (select count(*) from pg_catalog.pg_policy), + 'constraints', (select count(*) from pg_catalog.pg_constraint), + 'pg_depend', (select count(*) from pg_catalog.pg_depend) + ) + """, + user, + ).stdout + return json.loads(out) + + +def txn(header, body): + return ( + "begin;\n" + "set local search_path = '';\n" + f"set local pgrst.db_schemas = '{EXPOSED_SCHEMAS}';\n" + "set local statement_timeout = '120s';\n" + f"{header}\n{body}\n" + "rollback;\n" + ) + + +def round_script(sides, scenarios, flip): + parts = [] + for scenario in scenarios: + order = list(sides.items()) + if flip: + order.reverse() + for side, (header, lints) in order: + if scenario in lints: + parts.append(txn(header, + f"select '{side}', '{scenario}', ms, n, digest " + f"from splinter_bench.run({QUOTE}{lints[scenario]}{QUOTE});")) + return "".join(parts) + + +def plan_script(sides, scenarios): + parts = [] + for scenario in scenarios: + for side, (header, lints) in sides.items(): + if scenario in lints: + parts.append(f"\\echo {MARK}{side}|{scenario}\n") + parts.append(txn(header, f"explain (analyze, buffers, format json) {lints[scenario]};")) + return "".join(parts) + + +def blocks(node): + return node.get("Shared Hit Blocks", 0) + node.get("Shared Read Blocks", 0) + + +def repeated_scans(node, found): + """Catalog scans that run once per outer row and read far more than an index probe.""" + children = node.get("Plans", []) + own = blocks(node) - sum(blocks(c) for c in children) + loops = node.get("Actual Loops", 1) + rel = node.get("Relation Name", "") + if rel.startswith("pg_") and loops > 1 and own / loops > REPEATED_SCAN_BLOCKS: + found.append(f"{node['Node Type']} on {rel}" + (f" using {node['Index Name']}" if "Index Name" in node else "") + + f": {loops} loops Γ— {own / loops:.0f} blocks") + for c in children: + repeated_scans(c, found) + return found + + +def collect_plans(sides, scenarios, user, runs): + # Buffer counts include catalog cache misses, so the first pass only warms the + # session's caches and the second is the one recorded. + script = plan_script(sides, scenarios) + proc = psql(script.replace(MARK, "warmup") + script, user, check=False) + for chunk in proc.stdout.split(MARK)[1:]: + key, _, body = chunk.partition("\n") + side, scenario = key.split("|") + try: + plan = json.loads(body)[0]["Plan"] + except (json.JSONDecodeError, IndexError, KeyError): + continue + cell = runs[scenario][side] + cell["buffers"] = blocks(plan) + cell["repeated_scans"] = repeated_scans(plan, []) + # A lint without a plan would otherwise skip the plan and buffer guards. + for cells in runs.values(): + for cell in cells.values(): + if "buffers" not in cell and not cell["error"]: + cell["error"] = "EXPLAIN failed: " + (proc.stderr.strip()[-300:] or "no plan") + + +def measure(args): + sides = {"head": load_lints(Path(args.head).resolve())} + if args.base: + sides["base"] = load_lints(Path(args.base).resolve()) + scenarios = sorted(set().union(*(lints for _, lints in sides.values()))) + psql("create schema if not exists splinter_bench;" + RUN_FN, args.admin_user) + + if args.project: + label, migrations = args.project.split("=", 1) + t0 = time.monotonic() + applied = apply_project(migrations, args.user, args.admin_user) + print(f"[{label}] {applied['migrations'] - len(applied['failed'])}/{applied['migrations']} migrations " + f"applied in {time.monotonic() - t0:.0f}s", file=sys.stderr) + stages = [(label, lambda: applied)] + else: + stages = [] + built = 0 + for size in args.sizes.split(","): + label, n = size.split("=") + + def build(lo=built + 1, hi=int(n)): + build_fixture(lo, hi, args.admin_user) + return {"fixture_tables": hi} + + stages.append((label, build)) + built = int(n) + + result = { + "target": args.target or (args.project.split("=")[0] if args.project else "synthetic"), + "pg_version": psql("show server_version", args.user).stdout.strip(), + "rounds": args.rounds, + "sizes": {}, + } + for label, build in stages: + t0 = time.monotonic() + source = build() + print(f"[{label}] schema ready in {time.monotonic() - t0:.0f}s", file=sys.stderr) + + runs = {s: {side: {"ms": [], "n": None, "digest": None, "error": None} for side in sides if s in sides[side][1]} + for s in scenarios} + # Round 0 warms caches and is discarded. + for r in range(args.rounds + 1): + t0 = time.monotonic() + proc = psql(round_script(sides, scenarios, flip=r % 2 == 1), args.user, check=False) + seen = set() + for line in proc.stdout.splitlines(): + side, scenario, ms, count, digest = line.split("|") + seen.add((side, scenario)) + cell = runs[scenario][side] + if r > 0: + cell["ms"].append(float(ms)) + if cell["digest"] not in (None, digest): + cell["error"] = "results differ between rounds" + cell["n"], cell["digest"] = int(count), digest + for scenario, cells in runs.items(): + for side, cell in cells.items(): + if (side, scenario) not in seen: + cell["error"] = proc.stderr.strip()[-500:] or "no result" + print(f"[{label}] round {r}/{args.rounds} in {time.monotonic() - t0:.1f}s", file=sys.stderr) + collect_plans(sides, scenarios, args.user, runs) + result["sizes"][label] = {"source": source, "stats": catalog_stats(args.user), "scenarios": runs} + + Path(args.out).write_text(json.dumps(result, indent=2)) + + +def fetch(args): + project = json.loads((PROJECTS / "projects.json").read_text())[args.name] + dest = Path(args.dest) + dest.mkdir(parents=True, exist_ok=True) + git = lambda *a: subprocess.run(["git", "-C", str(dest), *a], check=True, capture_output=True) + git("init", "-q") + git("remote", "add", "origin", f"https://github.com/{project['repo']}.git") + git("sparse-checkout", "set", "--no-cone", project["migrations"]) + git("fetch", "-q", "--depth", "1", "--filter=blob:none", "origin", project["sha"]) + git("checkout", "-q", "FETCH_HEAD") + print(dest / project["migrations"]) + + +def judge(base, head): + """Every (status, detail) that applies to one scenario at one size; [("same", "")] if none. + Statuses are independent so that waiving one (say, a results change) cannot hide another.""" + if head is None: + return [("removed", "")] + if head["error"]: + return [("error", head["error"])] + base_scans = set() if base is None or base["error"] else {s.split(":")[0] for s in base.get("repeated_scans", [])} + found = [] + new_scans = [s for s in head.get("repeated_scans", []) if s.split(":")[0] not in base_scans] + if new_scans: + found.append(("plan", "; ".join(new_scans))) + if base is None: + return found + [("new", "")] + if base["error"]: + return found + [("base-error", base["error"])] + if (base["n"], base["digest"]) != (head["n"], head["digest"]): + found.append(("results-changed", f"{base['n']} β†’ {head['n']} rows")) + hb, bb = head["buffers"], base["buffers"] + if hb >= bb * BUFFERS_RATIO and hb - bb >= BUFFERS_MIN: + found.append(("buffers", f"{bb:,} β†’ {hb:,} blocks")) + b, h = statistics.median(base["ms"]), statistics.median(head["ms"]) + if h >= b * REGRESSION_RATIO and h - b >= REGRESSION_MIN_MS and min(head["ms"]) > max(base["ms"]): + found.append(("slower", "")) + elif b >= h * REGRESSION_RATIO and b - h >= REGRESSION_MIN_MS and min(base["ms"]) > max(head["ms"]): + found.append(("faster", "")) + return found or [("same", "")] + + +ICONS = { + "same": "", + "faster": "πŸš€ faster", + "slower": "πŸ”΄ slower", + "buffers": "πŸ”΄ more buffers", + "plan": "πŸ”΄ repeated catalog scan", + "results-changed": "πŸ”΄ results changed", + "error": "πŸ”΄ error", + "base-error": "πŸ”΄ base error", + "new": "πŸ†• new", + "removed": "removed", +} +# Which label waives which status. Errors on head are never waived. +WAIVERS = {"slower": "perf", "buffers": "perf", "plan": "perf", "base-error": "perf", "results-changed": "results"} + + +def fmt_ms(cell): + return f"{statistics.median(cell['ms']):,.1f}" if cell and cell["ms"] else "–" + + +def fmt_buf(cell): + return f"{cell['buffers']:,}" if cell and cell.get("buffers") is not None else "–" + + +def describe(size): + st, src = size["stats"], size["source"] + what = (f"{src['fixture_tables']} fixture tables" if "fixture_tables" in src + else f"{src['migrations'] - len(src['failed'])}/{src['migrations']} migrations applied") + return (f"{what}: {st['tables']} tables, {st['views']} views, {st['indexes']} indexes, " + f"{st['functions']} functions, {st['policies']} policies, {st['pg_depend']:,} pg_depend rows") + + +def compare(args): + results = [json.loads(Path(p).read_text()) for p in args.results] + has_base = any("base" in cells for d in results for s in d["sizes"].values() for cells in s["scenarios"].values()) + failures = {"perf": [], "results": [], "error": []} + lines = [ + "\n## splinter lint benchmark\n", + f"supabase/postgres {results[0]['pg_version']}, median of {results[0]['rounds']} interleaved rounds per side. " + f"Fails when a lint's rows change (count + digest of sorted rows), it errors, its plan gains a catalog scan " + f"repeated per row (>{REPEATED_SCAN_BLOCKS} blocks per loop), it reads β‰₯{BUFFERS_RATIO - 1:.0%} and β‰₯{BUFFERS_MIN} more buffers, " + f"or its median is β‰₯{REGRESSION_RATIO - 1:.0%} and β‰₯{REGRESSION_MIN_MS:g} ms slower with every head round slower than every base round.\n", + ] + details = [] + header = ("| lint | rows | base ms | head ms | Ξ” | buffers | |\n|---|--:|--:|--:|--:|--:|---|" if has_base + else "| lint | rows | ms | buffers | |\n|---|--:|--:|--:|---|") + for data in results: + for label, size in data["sizes"].items(): + title = f"{data['target']} / {label}" if data["target"] == "synthetic" else data["target"] + lines.append(f"\n### {title}\n{describe(size)}\n") + rows, flagged = [], [] + scenarios = size["scenarios"] + for name in sorted(scenarios, key=lambda s: (s != "all", s)): + base, head = scenarios[name].get("base"), scenarios[name].get("head") + key = f"{title}/{name}" + if not has_base: + status = "error" if head["error"] else ("plan" if head.get("repeated_scans") else "same") + detail = head["error"] or "; ".join(head.get("repeated_scans", [])) + if head["error"]: + failures["error"].append(key) + row = f"| `{name}` | {head['n'] if head['n'] is not None else '–'} | {fmt_ms(head)} | {fmt_buf(head)} | {ICONS[status]} {detail} |" + rows.append(row) + continue + verdicts = judge(base, head) + delta = "–" + if base and head and base["ms"] and head["ms"] and statistics.median(base["ms"]): + b, h = statistics.median(base["ms"]), statistics.median(head["ms"]) + delta = f"{(h - b) / b:+.0%}" + buf = f"{fmt_buf(base)} β†’ {fmt_buf(head)}" if base and head and base.get("buffers") != head.get("buffers") else fmt_buf(head or base) + row = (f"| `{name}` | {head['n'] if head and head['n'] is not None else '–'} | {fmt_ms(base)} | " + f"{fmt_ms(head)} | {delta} | {buf} | " + + "
".join(f"{ICONS[s]} {d}".strip() for s, d in verdicts) + " |") + rows.append(row) + if verdicts != [("same", "")] or name == "all": + flagged.append(row) + for status, _ in verdicts: + if status == "error": + failures["error"].append(key) + elif status in WAIVERS and key not in failures[WAIVERS[status]]: + failures[WAIVERS[status]].append(key) + if has_base: + lines += [header, *flagged, ""] + details += [f"\n**{title}**\n", header, *rows] + else: + lines += [header, *rows] + if size["source"].get("failed"): + lines.append(f"\n
{len(size['source']['failed'])} migrations failed to apply\n\n" + + "\n".join(f"- `{f}`" for f in size["source"]["failed"]) + "\n
\n") + if details: + lines += ["\n
All lints\n", *details, "\n
"] + + blocking = [] + if failures["error"]: + blocking.append(f"lints failed on head: {', '.join(failures['error'])}") + if failures["results"] and not args.accept_results: + blocking.append(f"results changed: {', '.join(failures['results'])} (add the `lint-results-changed` label if intended)") + if failures["perf"] and not args.accept_perf: + blocking.append(f"performance: {', '.join(failures['perf'])} (add the `perf-accepted` label if intended)") + lines.insert(1, "**Verdict:** " + ("πŸ”΄ " + "; ".join(blocking) if blocking else "βœ… no regression") + "\n") + report = "\n".join(lines) + "\n" + if args.markdown: + Path(args.markdown).write_text(report) + print(report) + return 1 if blocking else 0 + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + sub = parser.add_subparsers(dest="cmd", required=True) + m = sub.add_parser("measure", help="load a schema and time the lints (connects with the PG* env vars)") + m.add_argument("--head", default=".", help="checkout whose lints are measured (default: .)") + m.add_argument("--base", help="checkout to compare against; omit for absolute numbers only") + m.add_argument("--sizes", default=DEFAULT_SIZES, help=f"synthetic fixture, label=n_tables ascending (default: {DEFAULT_SIZES})") + m.add_argument("--project", help="label=path/to/migrations: apply a real project's migrations instead of the fixture") + m.add_argument("--target", help="name of this run in the report (default: synthetic or the project label)") + m.add_argument("--rounds", type=int, default=5) + m.add_argument("--user", default="postgres") + m.add_argument("--admin-user", default="supabase_admin") + m.add_argument("--out", default="bench-results.json") + f = sub.add_parser("fetch", help="download a project from projects/projects.json at its pinned commit") + f.add_argument("name") + f.add_argument("--dest", required=True) + c = sub.add_parser("compare", help="report on measure outputs; exits 1 on a regression") + c.add_argument("results", nargs="+") + c.add_argument("--markdown", help="also write the report here") + c.add_argument("--accept-perf", action="store_true") + c.add_argument("--accept-results", action="store_true") + args = parser.parse_args() + os.environ.setdefault("PGDATABASE", "postgres") + if args.cmd == "measure": + measure(args) + elif args.cmd == "fetch": + fetch(args) + else: + sys.exit(compare(args)) + + +if __name__ == "__main__": + main() diff --git a/bench/fixture/chunk.sql b/bench/fixture/chunk.sql new file mode 100644 index 00000000..9f36402e --- /dev/null +++ b/bench/fixture/chunk.sql @@ -0,0 +1,44 @@ +-- Objects for tables :lo..:hi, one transaction per chunk (stays under max_locks_per_transaction). +-- Every lint gets findings at every size: the modulo constants pick which tables carry which defect. +select set_config('bench.lo', :'lo', false), set_config('bench.hi', :'hi', false); +do $$ +declare i int; s text; t text; +begin + for i in current_setting('bench.lo')::int .. current_setting('bench.hi')::int loop + s := (array['public','app','billing','private'])[1 + i % 4]; + t := format('%I.t_%s', s, i); + execute format( + 'create table %s (id bigint generated always as identity primary key, owner uuid, status app.enum_%s, payload jsonb, created_at timestamptz default now(), parent_id bigint%s%s%s)', + t, 1 + i % 20, + case when i % 23 = 0 then ', ssn text, password text' else '' end, + case when i % 29 = 0 then ', handler regproc' else '' end, + case when i % 31 = 0 then ', user_id uuid references auth.users(id)' else '' end); + if i > 4 then + execute format('alter table %s add constraint fk_%s foreign key (parent_id) references %I.t_%s(id)', t, i, s, i - 4); + if i % 2 = 0 then execute format('create index on %s (parent_id)', t); end if; + end if; + execute format('create index on %s (owner, created_at)', t); + if i % 9 = 0 then execute format('create index on %s (owner, created_at)', t); end if; + execute format('create index on %s using gin (payload)', t); + if i % 7 <> 0 then execute format('alter table %s enable row level security', t); end if; + if i % 3 <> 0 then execute format('create policy sel_%s on %s for select to authenticated using (owner = auth.uid())', i, t); end if; + if i % 5 = 0 then execute format('create policy sel2_%s on %s for select to authenticated using (status = ''a'')', i, t); end if; + if i % 3 = 1 then execute format('create policy ins_%s on %s for insert to authenticated with check (owner = (select auth.uid()))', i, t); end if; + if i % 11 = 0 then execute format('create policy all_%s on %s for all to anon using (true) with check (true)', i, t); end if; + if i % 13 = 0 then execute format('create policy meta_%s on %s for select using ((auth.jwt() -> ''user_metadata'' ->> ''role'') = ''admin'')', i, t); end if; + if i % 19 = 0 then execute format('alter table %s set (autovacuum_enabled = false)', t); end if; + if i % 17 = 0 then execute format('create table %I.nopk_%s (id bigint, note text)', s, i); end if; + execute format('comment on table %s is ''table %s''', t, i); + if i % 4 = 0 then execute format('create view %I.v_%s %s as select id, owner from %s', s, i, case when i % 8 = 0 then '' else 'with (security_invoker = true)' end, t); end if; + if i % 10 = 0 then execute format('create materialized view %I.mv_%s as select id from %s', s, i, t); end if; + if i % 2 = 0 then + execute format('create function %I.f_%s(x int) returns int language sql %s %s as ''select x + 1''', s, i, + case when i % 4 = 0 then '' else 'set search_path = ''''' end, + case when i % 6 = 0 then 'security definer' else '' end); + end if; + if i % 4 = 0 then + execute format('create function %I.trg_%s() returns trigger language plpgsql set search_path = '''' as $f$ begin new.created_at := now(); return new; end $f$', s, i); + execute format('create trigger tr_%s before update on %s for each row execute function %I.trg_%s()', i, t, s, i); + end if; + end loop; +end $$; diff --git a/bench/fixture/head.sql b/bench/fixture/head.sql new file mode 100644 index 00000000..05e66861 --- /dev/null +++ b/bench/fixture/head.sql @@ -0,0 +1,47 @@ +-- Synthetic Supabase-like schema for benchmarking the lints (see bench/README.md). +-- Run as supabase_admin on a fresh image, after projects/services-stub.sql. + +-- Schema-wide objects that exercise lints a per-table loop cannot reach. +create or replace function auth.jwt() returns jsonb language sql stable as $f$ select coalesce(nullif(current_setting('request.jwt.claims', true), ''), '{}')::jsonb $f$; +create schema if not exists app; +create schema if not exists billing; +create schema if not exists private; +grant usage on schema app, billing to anon, authenticated; +do $$ begin for i in 1..20 loop execute format('create type app.enum_%s as enum (''a'',''b'',''c'')', i); end loop; end $$; + +-- 0002 auth_users_exposed +create view public.exposed_users as select id, email from auth.users; +grant select on public.exposed_users to anon, authenticated; + +-- 0014 extension_in_public, 0022 extension_versions_outdated +create extension if not exists ltree schema public; +create extension if not exists pg_graphql; + +-- 0017 foreign_table_in_api +create extension if not exists postgres_fdw schema extensions; +create server bench_fdw foreign data wrapper postgres_fdw options (host 'localhost', dbname 'postgres'); +create foreign table public.fdw_events (id int) server bench_fdw; +create foreign table app.fdw_audit (id int) server bench_fdw; + +-- 0019 insecure_queue_exposed_in_api (stubbed like test/sql/0019) +create schema if not exists pgmq; +create schema if not exists pgmq_public; +create table pgmq.q_jobs (id int); +create table pgmq.q_mail (id int); +grant usage on schema pgmq to anon, authenticated; +grant select on pgmq.q_jobs, pgmq.q_mail to anon, authenticated; + +-- 0025 public_bucket_allows_listing +insert into storage.buckets (id, name, public) values ('avatars', 'avatars', true), ('docs', 'docs', false) on conflict do nothing; +create policy "public read avatars" on storage.objects for select to anon using (bucket_id = 'avatars'); + +-- 0021 fkey_to_auth_unique +create table public.email_contacts (email varchar(255) references auth.users(email)); + +-- 0022 extension_versions_outdated +create extension if not exists amcheck version '1.0'; + +-- 0020 table_bloat: ~50 MB of dead tuples. Autovacuum is off so the pages stay until analyze. +create table public.bloated (id bigint generated always as identity primary key, pad text) with (autovacuum_enabled = false); +insert into public.bloated (pad) select repeat('x', 100) from generate_series(1, 400000); +delete from public.bloated where id > 100; diff --git a/bench/fixture/tail.sql b/bench/fixture/tail.sql new file mode 100644 index 00000000..b1e5b7c7 --- /dev/null +++ b/bench/fixture/tail.sql @@ -0,0 +1,2 @@ +grant select on all tables in schema public, app, billing to anon, authenticated; +analyze; diff --git a/bench/projects/projects.json b/bench/projects/projects.json new file mode 100644 index 00000000..f3e815c2 --- /dev/null +++ b/bench/projects/projects.json @@ -0,0 +1,7 @@ +{ + "basejump": {"repo": "usebasejump/basejump", "sha": "7a1f95ccef74eb2e638d5e4233b66b6cbbe175e6", "migrations": "supabase/migrations"}, + "atomic-crm": {"repo": "marmelab/atomic-crm", "sha": "c6a7bc842c8fe51c845f93680d15ee1f985f0ab2", "migrations": "supabase/migrations"}, + "wacrm": {"repo": "ArnasDon/wacrm", "sha": "34bce1a8abf56281505d90bac165123b6ab7d51a", "migrations": "supabase/migrations"}, + "grida": {"repo": "gridaco/grida", "sha": "9cbacacc89517971aec37d3de1c82202fb4d7986", "migrations": "supabase/migrations"}, + "carbon": {"repo": "crbnos/carbon", "sha": "493b720c3f62883c9cb72242ff9e1daef72ca844", "migrations": "packages/database/supabase/migrations"} +} diff --git a/bench/projects/services-stub.sql b/bench/projects/services-stub.sql new file mode 100644 index 00000000..87cde097 --- /dev/null +++ b/bench/projects/services-stub.sql @@ -0,0 +1,108 @@ +-- Tables that Supabase services create on a real project but the bare postgres image lacks: +-- storage-api's storage schema, GoTrue's MFA and SSO tables, realtime's messages. Enough for +-- project migrations that reference them. +create table if not exists storage.buckets ( + id text primary key, + name text not null unique, + owner uuid, + owner_id text, + public boolean default false, + avif_autodetection boolean default false, + file_size_limit bigint, + allowed_mime_types text[], + created_at timestamptz default now(), + updated_at timestamptz default now() +); +create table if not exists storage.objects ( + id uuid primary key default gen_random_uuid(), + bucket_id text references storage.buckets(id), + name text, + owner uuid, + owner_id text, + metadata jsonb, + user_metadata jsonb, + path_tokens text[] generated always as (string_to_array(name, '/')) stored, + version text, + created_at timestamptz default now(), + updated_at timestamptz default now(), + last_accessed_at timestamptz default now() +); +create unique index if not exists bucketid_objname on storage.objects (bucket_id, name); +alter table storage.buckets enable row level security; +alter table storage.objects enable row level security; +create or replace function storage.foldername(name text) returns text[] language plpgsql immutable as $f$ +declare _parts text[]; +begin + select string_to_array(name, '/') into _parts; + return _parts[1:array_length(_parts, 1) - 1]; +end +$f$; +create or replace function storage.filename(name text) returns text language plpgsql immutable as $f$ +declare _parts text[]; +begin + select string_to_array(name, '/') into _parts; + return _parts[array_length(_parts, 1)]; +end +$f$; +create or replace function storage.extension(name text) returns text language plpgsql immutable as $f$ +declare _parts text[]; _filename text; +begin + select string_to_array(name, '/') into _parts; + select _parts[array_length(_parts, 1)] into _filename; + return reverse(split_part(reverse(_filename), '.', 1)); +end +$f$; +grant usage on schema storage to anon, authenticated, service_role; +grant all on storage.buckets, storage.objects to anon, authenticated, service_role; + +create table if not exists auth.mfa_factors ( + id uuid primary key, + user_id uuid not null references auth.users(id) on delete cascade, + friendly_name text, + factor_type text not null, + status text not null, + secret text, + phone text, + created_at timestamptz not null, + updated_at timestamptz not null, + last_challenged_at timestamptz +); +create table if not exists auth.sso_providers ( + id uuid primary key, + resource_id text, + created_at timestamptz, + updated_at timestamptz +); +create table if not exists auth.sso_domains ( + id uuid primary key, + sso_provider_id uuid not null references auth.sso_providers(id) on delete cascade, + domain text not null, + created_at timestamptz, + updated_at timestamptz +); +create table if not exists realtime.messages ( + id uuid default gen_random_uuid(), + topic text not null, + extension text not null, + payload jsonb, + event text, + private boolean default false, + inserted_at timestamp not null default now(), + updated_at timestamp not null default now(), + primary key (id, inserted_at) +); +alter table realtime.messages enable row level security; +grant usage on schema realtime to anon, authenticated, service_role; +grant select, insert on realtime.messages to anon, authenticated, service_role; +create or replace function realtime.topic() returns text language sql stable as $f$ + select nullif(current_setting('realtime.topic', true), '')::text +$f$; +create or replace function realtime.send(payload jsonb, event text, topic text, private boolean default true) +returns void language plpgsql as $f$ +begin + insert into realtime.messages (payload, event, topic, private, extension) + values (payload, event, topic, private, 'broadcast'); +end +$f$; +-- On a real project postgres may add triggers and policies on these GoTrue tables. +grant all on auth.mfa_factors, auth.sso_providers, auth.sso_domains to postgres; From 5ffc8e2d6f9868359b393dd53858e00d093b2082 Mon Sep 17 00:00:00 2001 From: avallete Date: Tue, 20 Oct 2026 17:22:36 +0200 Subject: [PATCH 2/5] style(bench): format bench.py with black Co-Authored-By: Claude Opus 5.5 --- bench/bench.py | 249 ++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 194 insertions(+), 55 deletions(-) diff --git a/bench/bench.py b/bench/bench.py index a6a5f456..6d607ad6 100644 --- a/bench/bench.py +++ b/bench/bench.py @@ -61,7 +61,9 @@ def psql(sql, user, check=True, variables=None, single_transaction=False): - args = ["psql", "-X", "-q", "-At", "-F", "|", "-U", user] + (["--single-transaction"] if single_transaction else []) + args = ["psql", "-X", "-q", "-At", "-F", "|", "-U", user] + ( + ["--single-transaction"] if single_transaction else [] + ) if check: args += ["-v", "ON_ERROR_STOP=1"] for k, v in (variables or {}).items(): @@ -74,7 +76,9 @@ def psql(sql, user, check=True, variables=None, single_transaction=False): def load_lints(checkout): """Per-lint query text, compiled the way that checkout's bin/compile.py compiles it.""" - spec = importlib.util.spec_from_file_location(f"compile_{abs(hash(checkout))}", checkout / "bin" / "compile.py") + spec = importlib.util.spec_from_file_location( + f"compile_{abs(hash(checkout))}", checkout / "bin" / "compile.py" + ) mod = importlib.util.module_from_spec(spec) spec.loader.exec_module(mod) lints = dict(mod.load_sql_files(checkout / "lints")) @@ -94,13 +98,20 @@ def build_fixture(lo, hi, admin): def apply_project(migrations, user, admin): """Apply a project's migrations in order. A failing file is skipped and counted: real - projects lean on services (auth, storage) that the bare image only partly provides.""" + projects lean on services (auth, storage) that the bare image only partly provides. + """ psql((PROJECTS / "services-stub.sql").read_text(), admin) failed = [] files = sorted(Path(migrations).glob("*.sql")) for f in files: # One transaction per file, so a failed migration leaves nothing half-applied. - proc = psql(f.read_text(), user, check=False, variables={"ON_ERROR_STOP": 1}, single_transaction=True) + proc = psql( + f.read_text(), + user, + check=False, + variables={"ON_ERROR_STOP": 1}, + single_transaction=True, + ) if proc.returncode != 0: errors = [l for l in proc.stderr.splitlines() if "ERROR" in l] or ["failed"] failed.append(f"{f.name}: {errors[0].strip()}") @@ -145,9 +156,13 @@ def round_script(sides, scenarios, flip): order.reverse() for side, (header, lints) in order: if scenario in lints: - parts.append(txn(header, - f"select '{side}', '{scenario}', ms, n, digest " - f"from splinter_bench.run({QUOTE}{lints[scenario]}{QUOTE});")) + parts.append( + txn( + header, + f"select '{side}', '{scenario}', ms, n, digest " + f"from splinter_bench.run({QUOTE}{lints[scenario]}{QUOTE});", + ) + ) return "".join(parts) @@ -157,7 +172,12 @@ def plan_script(sides, scenarios): for side, (header, lints) in sides.items(): if scenario in lints: parts.append(f"\\echo {MARK}{side}|{scenario}\n") - parts.append(txn(header, f"explain (analyze, buffers, format json) {lints[scenario]};")) + parts.append( + txn( + header, + f"explain (analyze, buffers, format json) {lints[scenario]};", + ) + ) return "".join(parts) @@ -172,8 +192,11 @@ def repeated_scans(node, found): loops = node.get("Actual Loops", 1) rel = node.get("Relation Name", "") if rel.startswith("pg_") and loops > 1 and own / loops > REPEATED_SCAN_BLOCKS: - found.append(f"{node['Node Type']} on {rel}" + (f" using {node['Index Name']}" if "Index Name" in node else "") - + f": {loops} loops Γ— {own / loops:.0f} blocks") + found.append( + f"{node['Node Type']} on {rel}" + + (f" using {node['Index Name']}" if "Index Name" in node else "") + + f": {loops} loops Γ— {own / loops:.0f} blocks" + ) for c in children: repeated_scans(c, found) return found @@ -198,7 +221,9 @@ def collect_plans(sides, scenarios, user, runs): for cells in runs.values(): for cell in cells.values(): if "buffers" not in cell and not cell["error"]: - cell["error"] = "EXPLAIN failed: " + (proc.stderr.strip()[-300:] or "no plan") + cell["error"] = "EXPLAIN failed: " + ( + proc.stderr.strip()[-300:] or "no plan" + ) def measure(args): @@ -212,8 +237,11 @@ def measure(args): label, migrations = args.project.split("=", 1) t0 = time.monotonic() applied = apply_project(migrations, args.user, args.admin_user) - print(f"[{label}] {applied['migrations'] - len(applied['failed'])}/{applied['migrations']} migrations " - f"applied in {time.monotonic() - t0:.0f}s", file=sys.stderr) + print( + f"[{label}] {applied['migrations'] - len(applied['failed'])}/{applied['migrations']} migrations " + f"applied in {time.monotonic() - t0:.0f}s", + file=sys.stderr, + ) stages = [(label, lambda: applied)] else: stages = [] @@ -229,7 +257,8 @@ def build(lo=built + 1, hi=int(n)): built = int(n) result = { - "target": args.target or (args.project.split("=")[0] if args.project else "synthetic"), + "target": args.target + or (args.project.split("=")[0] if args.project else "synthetic"), "pg_version": psql("show server_version", args.user).stdout.strip(), "rounds": args.rounds, "sizes": {}, @@ -237,14 +266,24 @@ def build(lo=built + 1, hi=int(n)): for label, build in stages: t0 = time.monotonic() source = build() - print(f"[{label}] schema ready in {time.monotonic() - t0:.0f}s", file=sys.stderr) + print( + f"[{label}] schema ready in {time.monotonic() - t0:.0f}s", file=sys.stderr + ) - runs = {s: {side: {"ms": [], "n": None, "digest": None, "error": None} for side in sides if s in sides[side][1]} - for s in scenarios} + runs = { + s: { + side: {"ms": [], "n": None, "digest": None, "error": None} + for side in sides + if s in sides[side][1] + } + for s in scenarios + } # Round 0 warms caches and is discarded. for r in range(args.rounds + 1): t0 = time.monotonic() - proc = psql(round_script(sides, scenarios, flip=r % 2 == 1), args.user, check=False) + proc = psql( + round_script(sides, scenarios, flip=r % 2 == 1), args.user, check=False + ) seen = set() for line in proc.stdout.splitlines(): side, scenario, ms, count, digest = line.split("|") @@ -259,9 +298,16 @@ def build(lo=built + 1, hi=int(n)): for side, cell in cells.items(): if (side, scenario) not in seen: cell["error"] = proc.stderr.strip()[-500:] or "no result" - print(f"[{label}] round {r}/{args.rounds} in {time.monotonic() - t0:.1f}s", file=sys.stderr) + print( + f"[{label}] round {r}/{args.rounds} in {time.monotonic() - t0:.1f}s", + file=sys.stderr, + ) collect_plans(sides, scenarios, args.user, runs) - result["sizes"][label] = {"source": source, "stats": catalog_stats(args.user), "scenarios": runs} + result["sizes"][label] = { + "source": source, + "stats": catalog_stats(args.user), + "scenarios": runs, + } Path(args.out).write_text(json.dumps(result, indent=2)) @@ -270,7 +316,9 @@ def fetch(args): project = json.loads((PROJECTS / "projects.json").read_text())[args.name] dest = Path(args.dest) dest.mkdir(parents=True, exist_ok=True) - git = lambda *a: subprocess.run(["git", "-C", str(dest), *a], check=True, capture_output=True) + git = lambda *a: subprocess.run( + ["git", "-C", str(dest), *a], check=True, capture_output=True + ) git("init", "-q") git("remote", "add", "origin", f"https://github.com/{project['repo']}.git") git("sparse-checkout", "set", "--no-cone", project["migrations"]) @@ -281,14 +329,21 @@ def fetch(args): def judge(base, head): """Every (status, detail) that applies to one scenario at one size; [("same", "")] if none. - Statuses are independent so that waiving one (say, a results change) cannot hide another.""" + Statuses are independent so that waiving one (say, a results change) cannot hide another. + """ if head is None: return [("removed", "")] if head["error"]: return [("error", head["error"])] - base_scans = set() if base is None or base["error"] else {s.split(":")[0] for s in base.get("repeated_scans", [])} + base_scans = ( + set() + if base is None or base["error"] + else {s.split(":")[0] for s in base.get("repeated_scans", [])} + ) found = [] - new_scans = [s for s in head.get("repeated_scans", []) if s.split(":")[0] not in base_scans] + new_scans = [ + s for s in head.get("repeated_scans", []) if s.split(":")[0] not in base_scans + ] if new_scans: found.append(("plan", "; ".join(new_scans))) if base is None: @@ -301,9 +356,17 @@ def judge(base, head): if hb >= bb * BUFFERS_RATIO and hb - bb >= BUFFERS_MIN: found.append(("buffers", f"{bb:,} β†’ {hb:,} blocks")) b, h = statistics.median(base["ms"]), statistics.median(head["ms"]) - if h >= b * REGRESSION_RATIO and h - b >= REGRESSION_MIN_MS and min(head["ms"]) > max(base["ms"]): + if ( + h >= b * REGRESSION_RATIO + and h - b >= REGRESSION_MIN_MS + and min(head["ms"]) > max(base["ms"]) + ): found.append(("slower", "")) - elif b >= h * REGRESSION_RATIO and b - h >= REGRESSION_MIN_MS and min(base["ms"]) > max(head["ms"]): + elif ( + b >= h * REGRESSION_RATIO + and b - h >= REGRESSION_MIN_MS + and min(base["ms"]) > max(head["ms"]) + ): found.append(("faster", "")) return found or [("same", "")] @@ -321,7 +384,13 @@ def judge(base, head): "removed": "removed", } # Which label waives which status. Errors on head are never waived. -WAIVERS = {"slower": "perf", "buffers": "perf", "plan": "perf", "base-error": "perf", "results-changed": "results"} +WAIVERS = { + "slower": "perf", + "buffers": "perf", + "plan": "perf", + "base-error": "perf", + "results-changed": "results", +} def fmt_ms(cell): @@ -334,15 +403,25 @@ def fmt_buf(cell): def describe(size): st, src = size["stats"], size["source"] - what = (f"{src['fixture_tables']} fixture tables" if "fixture_tables" in src - else f"{src['migrations'] - len(src['failed'])}/{src['migrations']} migrations applied") - return (f"{what}: {st['tables']} tables, {st['views']} views, {st['indexes']} indexes, " - f"{st['functions']} functions, {st['policies']} policies, {st['pg_depend']:,} pg_depend rows") + what = ( + f"{src['fixture_tables']} fixture tables" + if "fixture_tables" in src + else f"{src['migrations'] - len(src['failed'])}/{src['migrations']} migrations applied" + ) + return ( + f"{what}: {st['tables']} tables, {st['views']} views, {st['indexes']} indexes, " + f"{st['functions']} functions, {st['policies']} policies, {st['pg_depend']:,} pg_depend rows" + ) def compare(args): results = [json.loads(Path(p).read_text()) for p in args.results] - has_base = any("base" in cells for d in results for s in d["sizes"].values() for cells in s["scenarios"].values()) + has_base = any( + "base" in cells + for d in results + for s in d["sizes"].values() + for cells in s["scenarios"].values() + ) failures = {"perf": [], "results": [], "error": []} lines = [ "\n## splinter lint benchmark\n", @@ -352,11 +431,18 @@ def compare(args): f"or its median is β‰₯{REGRESSION_RATIO - 1:.0%} and β‰₯{REGRESSION_MIN_MS:g} ms slower with every head round slower than every base round.\n", ] details = [] - header = ("| lint | rows | base ms | head ms | Ξ” | buffers | |\n|---|--:|--:|--:|--:|--:|---|" if has_base - else "| lint | rows | ms | buffers | |\n|---|--:|--:|--:|---|") + header = ( + "| lint | rows | base ms | head ms | Ξ” | buffers | |\n|---|--:|--:|--:|--:|--:|---|" + if has_base + else "| lint | rows | ms | buffers | |\n|---|--:|--:|--:|---|" + ) for data in results: for label, size in data["sizes"].items(): - title = f"{data['target']} / {label}" if data["target"] == "synthetic" else data["target"] + title = ( + f"{data['target']} / {label}" + if data["target"] == "synthetic" + else data["target"] + ) lines.append(f"\n### {title}\n{describe(size)}\n") rows, flagged = [], [] scenarios = size["scenarios"] @@ -364,7 +450,11 @@ def compare(args): base, head = scenarios[name].get("base"), scenarios[name].get("head") key = f"{title}/{name}" if not has_base: - status = "error" if head["error"] else ("plan" if head.get("repeated_scans") else "same") + status = ( + "error" + if head["error"] + else ("plan" if head.get("repeated_scans") else "same") + ) detail = head["error"] or "; ".join(head.get("repeated_scans", [])) if head["error"]: failures["error"].append(key) @@ -373,13 +463,26 @@ def compare(args): continue verdicts = judge(base, head) delta = "–" - if base and head and base["ms"] and head["ms"] and statistics.median(base["ms"]): + if ( + base + and head + and base["ms"] + and head["ms"] + and statistics.median(base["ms"]) + ): b, h = statistics.median(base["ms"]), statistics.median(head["ms"]) delta = f"{(h - b) / b:+.0%}" - buf = f"{fmt_buf(base)} β†’ {fmt_buf(head)}" if base and head and base.get("buffers") != head.get("buffers") else fmt_buf(head or base) - row = (f"| `{name}` | {head['n'] if head and head['n'] is not None else '–'} | {fmt_ms(base)} | " - f"{fmt_ms(head)} | {delta} | {buf} | " - + "
".join(f"{ICONS[s]} {d}".strip() for s, d in verdicts) + " |") + buf = ( + f"{fmt_buf(base)} β†’ {fmt_buf(head)}" + if base and head and base.get("buffers") != head.get("buffers") + else fmt_buf(head or base) + ) + row = ( + f"| `{name}` | {head['n'] if head and head['n'] is not None else '–'} | {fmt_ms(base)} | " + f"{fmt_ms(head)} | {delta} | {buf} | " + + "
".join(f"{ICONS[s]} {d}".strip() for s, d in verdicts) + + " |" + ) rows.append(row) if verdicts != [("same", "")] or name == "all": flagged.append(row) @@ -394,8 +497,11 @@ def compare(args): else: lines += [header, *rows] if size["source"].get("failed"): - lines.append(f"\n
{len(size['source']['failed'])} migrations failed to apply\n\n" - + "\n".join(f"- `{f}`" for f in size["source"]["failed"]) + "\n
\n") + lines.append( + f"\n
{len(size['source']['failed'])} migrations failed to apply\n\n" + + "\n".join(f"- `{f}`" for f in size["source"]["failed"]) + + "\n
\n" + ) if details: lines += ["\n
All lints\n", *details, "\n
"] @@ -403,10 +509,19 @@ def compare(args): if failures["error"]: blocking.append(f"lints failed on head: {', '.join(failures['error'])}") if failures["results"] and not args.accept_results: - blocking.append(f"results changed: {', '.join(failures['results'])} (add the `lint-results-changed` label if intended)") + blocking.append( + f"results changed: {', '.join(failures['results'])} (add the `lint-results-changed` label if intended)" + ) if failures["perf"] and not args.accept_perf: - blocking.append(f"performance: {', '.join(failures['perf'])} (add the `perf-accepted` label if intended)") - lines.insert(1, "**Verdict:** " + ("πŸ”΄ " + "; ".join(blocking) if blocking else "βœ… no regression") + "\n") + blocking.append( + f"performance: {', '.join(failures['perf'])} (add the `perf-accepted` label if intended)" + ) + lines.insert( + 1, + "**Verdict:** " + + ("πŸ”΄ " + "; ".join(blocking) if blocking else "βœ… no regression") + + "\n", + ) report = "\n".join(lines) + "\n" if args.markdown: Path(args.markdown).write_text(report) @@ -415,22 +530,46 @@ def compare(args): def main(): - parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) sub = parser.add_subparsers(dest="cmd", required=True) - m = sub.add_parser("measure", help="load a schema and time the lints (connects with the PG* env vars)") - m.add_argument("--head", default=".", help="checkout whose lints are measured (default: .)") - m.add_argument("--base", help="checkout to compare against; omit for absolute numbers only") - m.add_argument("--sizes", default=DEFAULT_SIZES, help=f"synthetic fixture, label=n_tables ascending (default: {DEFAULT_SIZES})") - m.add_argument("--project", help="label=path/to/migrations: apply a real project's migrations instead of the fixture") - m.add_argument("--target", help="name of this run in the report (default: synthetic or the project label)") + m = sub.add_parser( + "measure", + help="load a schema and time the lints (connects with the PG* env vars)", + ) + m.add_argument( + "--head", default=".", help="checkout whose lints are measured (default: .)" + ) + m.add_argument( + "--base", help="checkout to compare against; omit for absolute numbers only" + ) + m.add_argument( + "--sizes", + default=DEFAULT_SIZES, + help=f"synthetic fixture, label=n_tables ascending (default: {DEFAULT_SIZES})", + ) + m.add_argument( + "--project", + help="label=path/to/migrations: apply a real project's migrations instead of the fixture", + ) + m.add_argument( + "--target", + help="name of this run in the report (default: synthetic or the project label)", + ) m.add_argument("--rounds", type=int, default=5) m.add_argument("--user", default="postgres") m.add_argument("--admin-user", default="supabase_admin") m.add_argument("--out", default="bench-results.json") - f = sub.add_parser("fetch", help="download a project from projects/projects.json at its pinned commit") + f = sub.add_parser( + "fetch", + help="download a project from projects/projects.json at its pinned commit", + ) f.add_argument("name") f.add_argument("--dest", required=True) - c = sub.add_parser("compare", help="report on measure outputs; exits 1 on a regression") + c = sub.add_parser( + "compare", help="report on measure outputs; exits 1 on a regression" + ) c.add_argument("results", nargs="+") c.add_argument("--markdown", help="also write the report here") c.add_argument("--accept-perf", action="store_true") From d7c97d06297f1cbb129e71b6747e5ab22e19a131 Mon Sep 17 00:00:00 2001 From: avallete Date: Wed, 21 Oct 2026 17:20:36 +0200 Subject: [PATCH 3/5] ci(bench): report only lints that move beyond the noise bounds, with each median's range Co-Authored-By: Claude Opus 5.5 --- bench/bench.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/bench/bench.py b/bench/bench.py index 6d607ad6..ea15d4c3 100644 --- a/bench/bench.py +++ b/bench/bench.py @@ -355,6 +355,8 @@ def judge(base, head): hb, bb = head["buffers"], base["buffers"] if hb >= bb * BUFFERS_RATIO and hb - bb >= BUFFERS_MIN: found.append(("buffers", f"{bb:,} β†’ {hb:,} blocks")) + elif bb >= hb * BUFFERS_RATIO and bb - hb >= BUFFERS_MIN: + found.append(("fewer-buffers", "")) b, h = statistics.median(base["ms"]), statistics.median(head["ms"]) if ( h >= b * REGRESSION_RATIO @@ -374,6 +376,7 @@ def judge(base, head): ICONS = { "same": "", "faster": "πŸš€ faster", + "fewer-buffers": "πŸš€ fewer buffers", "slower": "πŸ”΄ slower", "buffers": "πŸ”΄ more buffers", "plan": "πŸ”΄ repeated catalog scan", @@ -394,7 +397,10 @@ def judge(base, head): def fmt_ms(cell): - return f"{statistics.median(cell['ms']):,.1f}" if cell and cell["ms"] else "–" + if not cell or not cell["ms"]: + return "–" + ms = cell["ms"] + return f"{statistics.median(ms):,.1f} ({min(ms):,.1f}–{max(ms):,.1f})" def fmt_buf(cell): @@ -425,12 +431,11 @@ def compare(args): failures = {"perf": [], "results": [], "error": []} lines = [ "\n## splinter lint benchmark\n", - f"supabase/postgres {results[0]['pg_version']}, median of {results[0]['rounds']} interleaved rounds per side. " + f"supabase/postgres {results[0]['pg_version']}, median (min–max) of {results[0]['rounds']} interleaved rounds per side. " f"Fails when a lint's rows change (count + digest of sorted rows), it errors, its plan gains a catalog scan " f"repeated per row (>{REPEATED_SCAN_BLOCKS} blocks per loop), it reads β‰₯{BUFFERS_RATIO - 1:.0%} and β‰₯{BUFFERS_MIN} more buffers, " f"or its median is β‰₯{REGRESSION_RATIO - 1:.0%} and β‰₯{REGRESSION_MIN_MS:g} ms slower with every head round slower than every base round.\n", ] - details = [] header = ( "| lint | rows | base ms | head ms | Ξ” | buffers | |\n|---|--:|--:|--:|--:|--:|---|" if has_base @@ -483,7 +488,6 @@ def compare(args): + "
".join(f"{ICONS[s]} {d}".strip() for s, d in verdicts) + " |" ) - rows.append(row) if verdicts != [("same", "")] or name == "all": flagged.append(row) for status, _ in verdicts: @@ -492,8 +496,12 @@ def compare(args): elif status in WAIVERS and key not in failures[WAIVERS[status]]: failures[WAIVERS[status]].append(key) if has_base: + unchanged = len(scenarios) - len(flagged) lines += [header, *flagged, ""] - details += [f"\n**{title}**\n", header, *rows] + if unchanged: + lines.append( + f"{unchanged} lints within noise (rows identical, time and buffers inside the bounds above).\n" + ) else: lines += [header, *rows] if size["source"].get("failed"): @@ -502,9 +510,6 @@ def compare(args): + "\n".join(f"- `{f}`" for f in size["source"]["failed"]) + "\n\n" ) - if details: - lines += ["\n
All lints\n", *details, "\n
"] - blocking = [] if failures["error"]: blocking.append(f"lints failed on head: {', '.join(failures['error'])}") From ca2dec9d069784676bb2b5c7800c195821b3089b Mon Sep 17 00:00:00 2001 From: avallete Date: Thu, 22 Oct 2026 10:47:48 +0200 Subject: [PATCH 4/5] ci(bench): time only large targets, check rows on the small projects, add deskcomm Synthetic drops its small and medium sizes. The small projects run with --rounds 0: their per-lint timings sit inside runner noise. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/bench.yml | 13 +++++++++++-- bench/README.md | 6 ++++-- bench/bench.py | 32 ++++++++++++++++++++++++-------- bench/projects/projects.json | 3 ++- 4 files changed, 41 insertions(+), 13 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index e65d55c0..2d40a676 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -27,7 +27,15 @@ jobs: strategy: fail-fast: false matrix: - target: [synthetic, basejump, atomic-crm, wacrm, grida, carbon] + # Small schemas only check rows: their timings sit inside runner noise. + include: + - { target: synthetic, rounds: 5 } + - { target: carbon, rounds: 5 } + - { target: deskcomm, rounds: 5 } + - { target: basejump, rounds: 0 } + - { target: atomic-crm, rounds: 0 } + - { target: wacrm, rounds: 0 } + - { target: grida, rounds: 0 } runs-on: ubuntu-latest timeout-minutes: 20 services: @@ -66,9 +74,10 @@ jobs: - name: Measure env: TARGET: ${{ matrix.target }} + ROUNDS: ${{ matrix.rounds }} BASE: ${{ github.event_name == 'pull_request' && 'base' || '' }} run: | - args=(--head head --out "results-$TARGET.json") + args=(--head head --rounds "$ROUNDS" --out "results-$TARGET.json") if [ -n "$BASE" ]; then args+=(--base "$BASE"); fi if [ "$TARGET" != synthetic ]; then migrations=$(python head/bench/bench.py fetch "$TARGET" --dest "project-$TARGET") diff --git a/bench/README.md b/bench/README.md index ea93ebd9..cd9a41c2 100644 --- a/bench/README.md +++ b/bench/README.md @@ -4,7 +4,7 @@ Times every lint, and the full `splinter.sql` union, on Supabase schemas, compar ## Targets -- **synthetic**: `fixture/` built at 20, 300 and 2,000 tables. The sizes are cumulative, so one database serves all three. Every lint has findings at every size. +- **synthetic**: `fixture/` built at 2,000 tables, a scaling stress test that surfaces lints whose cost grows faster than the catalog. Every lint has findings. `--sizes small=20,large=2000` builds several sizes cumulatively on one database. - **Real projects**: the open-source apps from the dogfooding corpus, fetched from GitHub at the commits pinned in `projects/projects.json`. Their `supabase/migrations` are applied in order on the bare image, after `projects/services-stub.sql` stands in for the tables storage-api, GoTrue and realtime would create. Each migration runs in one transaction; one that still fails is rolled back, skipped and listed in the report. | target | tables | policies | `pg_depend` rows | migrations applied | @@ -18,6 +18,8 @@ Times every lint, and the full `splinter.sql` union, on Supabase schemas, compar Table counts include the image's own schemas. +Only the large targets are timed. The small projects run with `--rounds 0`: their full run takes 50–100 ms, mostly fixed overhead, and per-lint differences stay inside runner noise, so they only check that every lint returns the same rows as base. Their real-world SQL still catches rewrites that break on patterns the fixture lacks. + ## What fails the check | Guard | Rule | Waiver label | @@ -34,7 +36,7 @@ Buffer counts are deterministic, so they catch plan regressions that are too sma ## When you change the benchmark - When you add a lint, extend `fixture/` so it has findings. -- When you add a project, pin a commit in `projects/projects.json` and add it to the matrix in `.github/workflows/bench.yml`. +- When you add a project, pin a commit in `projects/projects.json` and add it to the matrix in `.github/workflows/bench.yml`, timed (`rounds: 5`) only if it is large enough for lint timings to move beyond noise. ## Run locally diff --git a/bench/bench.py b/bench/bench.py index ea15d4c3..a8b43bf4 100644 --- a/bench/bench.py +++ b/bench/bench.py @@ -27,7 +27,7 @@ FIXTURE = HERE / "fixture" PROJECTS = HERE / "projects" CHUNK = 200 -DEFAULT_SIZES = "small=20,medium=300,large=2000" +DEFAULT_SIZES = "large=2000" EXPOSED_SCHEMAS = "public, app, billing, pgmq_public, graphql_public" QUOTE = "$splinter_bench$" MARK = "@@splinter_bench@@" @@ -302,7 +302,10 @@ def build(lo=built + 1, hi=int(n)): f"[{label}] round {r}/{args.rounds} in {time.monotonic() - t0:.1f}s", file=sys.stderr, ) - collect_plans(sides, scenarios, args.user, runs) + # With no timed rounds the target only checks rows; small schemas give + # buffer and plan deltas too small to judge. + if args.rounds: + collect_plans(sides, scenarios, args.user, runs) result["sizes"][label] = { "source": source, "stats": catalog_stats(args.user), @@ -352,6 +355,8 @@ def judge(base, head): return found + [("base-error", base["error"])] if (base["n"], base["digest"]) != (head["n"], head["digest"]): found.append(("results-changed", f"{base['n']} β†’ {head['n']} rows")) + if not head["ms"]: + return found or [("same", "")] hb, bb = head["buffers"], base["buffers"] if hb >= bb * BUFFERS_RATIO and hb - bb >= BUFFERS_MIN: found.append(("buffers", f"{bb:,} β†’ {hb:,} blocks")) @@ -431,7 +436,7 @@ def compare(args): failures = {"perf": [], "results": [], "error": []} lines = [ "\n## splinter lint benchmark\n", - f"supabase/postgres {results[0]['pg_version']}, median (min–max) of {results[0]['rounds']} interleaved rounds per side. " + f"supabase/postgres {results[0]['pg_version']}, median (min–max) of {max(d['rounds'] for d in results)} interleaved rounds per side. " f"Fails when a lint's rows change (count + digest of sorted rows), it errors, its plan gains a catalog scan " f"repeated per row (>{REPEATED_SCAN_BLOCKS} blocks per loop), it reads β‰₯{BUFFERS_RATIO - 1:.0%} and β‰₯{BUFFERS_MIN} more buffers, " f"or its median is β‰₯{REGRESSION_RATIO - 1:.0%} and β‰₯{REGRESSION_MIN_MS:g} ms slower with every head round slower than every base round.\n", @@ -448,7 +453,10 @@ def compare(args): if data["target"] == "synthetic" else data["target"] ) - lines.append(f"\n### {title}\n{describe(size)}\n") + timed = data["rounds"] > 0 + lines.append( + f"\n### {title}{'' if timed else ' (rows only)'}\n{describe(size)}\n" + ) rows, flagged = [], [] scenarios = size["scenarios"] for name in sorted(scenarios, key=lambda s: (s != "all", s)): @@ -488,7 +496,7 @@ def compare(args): + "
".join(f"{ICONS[s]} {d}".strip() for s, d in verdicts) + " |" ) - if verdicts != [("same", "")] or name == "all": + if verdicts != [("same", "")] or (name == "all" and timed): flagged.append(row) for status, _ in verdicts: if status == "error": @@ -497,8 +505,11 @@ def compare(args): failures[WAIVERS[status]].append(key) if has_base: unchanged = len(scenarios) - len(flagged) - lines += [header, *flagged, ""] - if unchanged: + if flagged: + lines += [header, *flagged, ""] + if not timed: + lines.append(f"{unchanged} queries return identical rows.\n") + elif unchanged: lines.append( f"{unchanged} lints within noise (rows identical, time and buffers inside the bounds above).\n" ) @@ -562,7 +573,12 @@ def main(): "--target", help="name of this run in the report (default: synthetic or the project label)", ) - m.add_argument("--rounds", type=int, default=5) + m.add_argument( + "--rounds", + type=int, + default=5, + help="timed rounds per side; 0 only checks errors and rows (default: 5)", + ) m.add_argument("--user", default="postgres") m.add_argument("--admin-user", default="supabase_admin") m.add_argument("--out", default="bench-results.json") diff --git a/bench/projects/projects.json b/bench/projects/projects.json index f3e815c2..3359b8ed 100644 --- a/bench/projects/projects.json +++ b/bench/projects/projects.json @@ -3,5 +3,6 @@ "atomic-crm": {"repo": "marmelab/atomic-crm", "sha": "c6a7bc842c8fe51c845f93680d15ee1f985f0ab2", "migrations": "supabase/migrations"}, "wacrm": {"repo": "ArnasDon/wacrm", "sha": "34bce1a8abf56281505d90bac165123b6ab7d51a", "migrations": "supabase/migrations"}, "grida": {"repo": "gridaco/grida", "sha": "9cbacacc89517971aec37d3de1c82202fb4d7986", "migrations": "supabase/migrations"}, - "carbon": {"repo": "crbnos/carbon", "sha": "493b720c3f62883c9cb72242ff9e1daef72ca844", "migrations": "packages/database/supabase/migrations"} + "carbon": {"repo": "crbnos/carbon", "sha": "493b720c3f62883c9cb72242ff9e1daef72ca844", "migrations": "packages/database/supabase/migrations"}, + "deskcomm": {"repo": "melgarafael/DeskcommCRM", "sha": "dc921fa337909ddfac71fc5aab10a18cb15cadaa", "migrations": "supabase/migrations"} } From 2970fdeb784efacd6ff6f23373eb4cb2a4b8d94e Mon Sep 17 00:00:00 2001 From: avallete Date: Thu, 22 Oct 2026 11:38:00 +0200 Subject: [PATCH 5/5] ci(bench): apply deskcomm's baseline dump before its migrations Its initial schema lives in supabase/baseline.sql, so without it only 76 of 529 migrations applied. Co-Authored-By: Claude Opus 5.5 --- bench/README.md | 2 ++ bench/bench.py | 11 ++++++++++- bench/projects/projects.json | 2 +- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/bench/README.md b/bench/README.md index cd9a41c2..8482d276 100644 --- a/bench/README.md +++ b/bench/README.md @@ -11,6 +11,7 @@ Times every lint, and the full `splinter.sql` union, on Supabase schemas, compar | -- | --: | --: | --: | --: | | synthetic / large | 2,197 | 2,736 | 64k | – | | carbon | 520 | 1,501 | 41k | 1,230 / 1,230 | +| deskcomm | 283 | 669 | 17k | 520 / 530 (baseline dump + migrations) | | grida | 174 | 89 | 7k | 76 / 87 (timescaledb is not in the image) | | wacrm | 113 | 114 | 5k | 54 / 54 | | atomic-crm | 88 | 36 | 3k | 28 / 28 | @@ -36,6 +37,7 @@ Buffer counts are deterministic, so they catch plan regressions that are too sma ## When you change the benchmark - When you add a lint, extend `fixture/` so it has findings. +- A project whose initial schema is a dump outside its migrations sets `baseline` (applied first) and, for the extensions the dump assumes, `setup` in `projects/projects.json`. - When you add a project, pin a commit in `projects/projects.json` and add it to the matrix in `.github/workflows/bench.yml`, timed (`rounds: 5`) only if it is large enough for lint timings to move beyond noise. ## Run locally diff --git a/bench/bench.py b/bench/bench.py index a8b43bf4..73fdc025 100644 --- a/bench/bench.py +++ b/bench/bench.py @@ -324,9 +324,18 @@ def fetch(args): ) git("init", "-q") git("remote", "add", "origin", f"https://github.com/{project['repo']}.git") - git("sparse-checkout", "set", "--no-cone", project["migrations"]) + paths = [project["migrations"]] + ( + [project["baseline"]] if "baseline" in project else [] + ) + git("sparse-checkout", "set", "--no-cone", *paths) git("fetch", "-q", "--depth", "1", "--filter=blob:none", "origin", project["sha"]) git("checkout", "-q", "FETCH_HEAD") + if "baseline" in project: + # Some projects keep their initial schema outside migrations/; applied first. + # A pg_dump baseline assumes the extensions its source database had. + (dest / project["migrations"] / "00000000000000_baseline.sql").write_text( + project.get("setup", "") + "\n" + (dest / project["baseline"]).read_text() + ) print(dest / project["migrations"]) diff --git a/bench/projects/projects.json b/bench/projects/projects.json index 3359b8ed..731fc4c2 100644 --- a/bench/projects/projects.json +++ b/bench/projects/projects.json @@ -4,5 +4,5 @@ "wacrm": {"repo": "ArnasDon/wacrm", "sha": "34bce1a8abf56281505d90bac165123b6ab7d51a", "migrations": "supabase/migrations"}, "grida": {"repo": "gridaco/grida", "sha": "9cbacacc89517971aec37d3de1c82202fb4d7986", "migrations": "supabase/migrations"}, "carbon": {"repo": "crbnos/carbon", "sha": "493b720c3f62883c9cb72242ff9e1daef72ca844", "migrations": "packages/database/supabase/migrations"}, - "deskcomm": {"repo": "melgarafael/DeskcommCRM", "sha": "dc921fa337909ddfac71fc5aab10a18cb15cadaa", "migrations": "supabase/migrations"} + "deskcomm": {"repo": "melgarafael/DeskcommCRM", "sha": "dc921fa337909ddfac71fc5aab10a18cb15cadaa", "migrations": "supabase/migrations", "baseline": "supabase/baseline.sql", "setup": "create extension if not exists vector with schema public; create extension if not exists citext with schema public; create extension if not exists pg_trgm with schema public;"} }