From c2d31ecb63f3235384d096b3974320e8e7bbe8d8 Mon Sep 17 00:00:00 2001 From: Luke Karrys Date: Wed, 18 Nov 2026 04:31:20 +0000 Subject: [PATCH 1/2] fix(www): average only the scenarios every run in the window has The average view took its plan from the shown run, and `averageFiles` drops a fixture missing any planned scenario, so every run before a scenario was added (the add/remove four, Nov 17) drew no trend point. `commonPlan` keeps, per fixture, the scenarios every drawn run has (excluded runs and runs past the request budget don't count). The trend and the shown run's numbers both use it, so they agree. The page lede, Markdown twin, MCP report and history name what's left out and why, and say when too few common scenarios forced a fallback to the shown run's. Closes #216 Co-Authored-By: Claude Opus 5.5 --- apps/www/AGENTS.md | 2 +- .../src/app/[section]/[[...params]]/page.tsx | 9 +- apps/www/src/lib/api.ts | 2 + apps/www/src/lib/load.ts | 33 +++- apps/www/src/lib/markdown.ts | 54 ++++++- apps/www/src/lib/measurements.ts | 79 +++++++++ apps/www/src/lib/report.ts | 20 ++- apps/www/test/average.test.ts | 150 +++++++++++++++++- 8 files changed, 327 insertions(+), 22 deletions(-) diff --git a/apps/www/AGENTS.md b/apps/www/AGENTS.md index 643577dfae..af3b7c8411 100644 --- a/apps/www/AGENTS.md +++ b/apps/www/AGENTS.md @@ -1,6 +1,6 @@ -# This is NOT the Next.js you know +## This is NOT the Next.js you know This version has breaking changes — APIs, conventions, and file structure may all differ from your training data. Read the relevant guide in `node_modules/next/dist/docs/` (resolved from this file's directory; in monorepos the `next` package may not be visible from the repo root) before writing any code. Heed deprecation notices. diff --git a/apps/www/src/app/[section]/[[...params]]/page.tsx b/apps/www/src/app/[section]/[[...params]]/page.tsx index da81d7ed21..b632c27df5 100644 --- a/apps/www/src/app/[section]/[[...params]]/page.tsx +++ b/apps/www/src/app/[section]/[[...params]]/page.tsx @@ -42,7 +42,7 @@ import { toTrendColumns } from "@/lib/chart-model"; import { loadReportParams, rangeLabel, reportHref, reportParams } from "@/lib/params"; import { loadCatalog } from "@/lib/catalog"; import { dataClient } from "@/lib/data"; -import { PER_PACKAGE_DERIVED } from "@/lib/markdown"; +import { averagedNote, PER_PACKAGE_DERIVED } from "@/lib/markdown"; import { loadReport } from "@/lib/report"; import { withAverage } from "@/lib/scenarios"; import { isSectionId, SECTION_DEFS, SECTIONS } from "@/lib/sections"; @@ -188,6 +188,7 @@ const Report = async ({ const report = result.report; const { scenario, run, filters, finding, baseline, caveats, trend } = report; + const note = scenario.averageOf === null ? null : averagedNote(scenario.averageOf, longDate); const { date } = run; const { fixture, hide: hidden } = filters; const { measure, isScript, outline, ranked, publishedFixtures, path } = report; @@ -641,6 +642,12 @@ const Report = async ({ ? " Every registry is installed from with the same npm client." : ""} {run.isLatest ? null : " This is not the latest run."} + {note === null ? null : ( + <> + {" "} + + + )} } action={ diff --git a/apps/www/src/lib/api.ts b/apps/www/src/lib/api.ts index 40d7cc8abd..1bc6a53cf2 100644 --- a/apps/www/src/lib/api.ts +++ b/apps/www/src/lib/api.ts @@ -112,6 +112,8 @@ export const historyOf = (report: ReportModel, tools?: readonly string[]) => { return { section: report.section.id, scenario: report.scenario.variation, + /* On the derived view, what the trend averages: the scenarios every run in the window has. */ + averageOf: report.scenario.averageOf, run: report.run.date, fixture: report.filters.fixture, fixtures: report.fixtures, diff --git a/apps/www/src/lib/load.ts b/apps/www/src/lib/load.ts index 8b868f572e..f9d673ad32 100644 --- a/apps/www/src/lib/load.ts +++ b/apps/www/src/lib/load.ts @@ -16,7 +16,7 @@ import { } from "@benchmarks/data"; import { probe } from "./data"; import { applyExclusions } from "./figures"; -import { averageFiles, usable } from "./measurements"; +import { averageFiles, commonPlan, usable, type CommonPlan } from "./measurements"; export interface FixtureBundle { readonly fixture: string; @@ -388,9 +388,11 @@ const loadCells = async ( /** * The derived "average" view, as a scenario: `plan` maps each fixture to the - * scenarios averaged on it. `cells` are the shown run's per-scenario files it - * was averaged from. No count files -- a package or process count is a - * fact about one scenario, and a mean of them describes no install. + * scenarios that may be averaged on it, and `averaged` to the ones that are: + * those every drawn run has (`commonPlan`), so the trend and the shown run + * cover the same scenarios. `cells` are the shown run's per-scenario files. + * No count files -- a package or process count is a fact about one scenario, + * and a mean of them describes no install. */ export const loadAverage = async ( client: DataClient, @@ -402,7 +404,9 @@ export const loadAverage = async ( /** From `loadSummaries`, possibly still loading, as for `loadScenario`. */ readonly summaries?: Summaries | Promise; } = {}, -): Promise => { +): Promise< + ScenarioData & { readonly cells: readonly RunCell[]; readonly averaged: CommonPlan } +> => { // The run shown is usually the trend's last date; fetched once either way, // and from its files, which the report reads in full, without waiting for // the summaries. @@ -415,12 +419,25 @@ export const loadAverage = async ( const loaded = await Promise.all( dates.map((d) => (d === date ? shown : loadCells(client, d, plan, summaryOf(d)))), ); - const trend = drawn.flatMap((d, i) => trendPoint(d, averageFiles(loaded[i] ?? [], plan)) ?? []); - const cells = loaded[dates.indexOf(date)] ?? []; + /* The runs the trend draws, and the shown one: an excluded run does not narrow the set. */ + const whole = new Set((options.exclude ?? []).filter((e) => !e.includes("@"))); + const averaged = commonPlan( + plan, + dates.flatMap((d, i) => + d !== date && whole.has(d) ? [] : [{ date: d, cells: loaded[i] ?? [] }], + ), + ); + const trend = drawn.flatMap( + (d, i) => trendPoint(d, averageFiles(loaded[i] ?? [], averaged.plan)) ?? [], + ); + const cells = (loaded[dates.indexOf(date)] ?? []).filter((c) => + averaged.plan[c.fixture]?.includes(c.variation), + ); return { cells, - bundles: averageFiles(cells, plan).map((b) => ({ + averaged, + bundles: averageFiles(cells, averaged.plan).map((b) => ({ ...b, counts: null, processes: null, diff --git a/apps/www/src/lib/markdown.ts b/apps/www/src/lib/markdown.ts index 96edbebd44..c5dbba6224 100644 --- a/apps/www/src/lib/markdown.ts +++ b/apps/www/src/lib/markdown.ts @@ -438,16 +438,57 @@ export const figureMarkdown = (report: ReportModel, origin: string): FigureMarkd export const PER_PACKAGE_DERIVED = "Derived: install time divided by the number of packages the tool installed. A tool with a time but no published package count is left out rather than divided by a guess."; -/** What a derived average is the mean of: its scenarios, per fixture where they differ. */ -export const meanOf = ({ - scenarios, - byFixture, -}: NonNullable): string => - Object.values(byFixture).every((v) => v.length === scenarios.length) +/** What a derived average is the mean of: its scenarios, per fixture where they differ, and what it leaves out. */ +export const meanOf = (averageOf: NonNullable): string => { + const { scenarios, byFixture } = averageOf; + const mean = Object.values(byFixture).every((v) => v.length === scenarios.length) ? `Mean of: ${scenarios.join(", ")}.` : `Mean of, per fixture: ${Object.entries(byFixture) .map(([f, v]) => `${fixtureLabel(f)}: ${v.join(", ")}`) .join("; ")}.`; + const note = averagedNote(averageOf); + return note === null ? mean : `${mean} ${note}`; +}; + +/** + * Which curated scenarios the average leaves out because some run in the trend + * window lacks them, and why; `null` when it leaves none out. Names between + * backticks, for the page's `Marked` and for Markdown alike. + */ +export const averagedNote = ( + { byFixture, leftOut, fallback }: NonNullable, + date: (d: string) => string = (d) => d, +): string | null => { + const fixtures = Object.keys(byFixture); + /* Where it applies, when that is not every fixture shown. */ + const on = (some: readonly string[]) => + some.length === fixtures.length ? [] : [`on ${some.map(fixtureLabel).join(", ")}`]; + /* Scenarios with the same reason share it: the add/remove four all start the same day. */ + const reasons = new Map(); + for (const l of leftOut) { + const why = + l.since === null + ? `missing on ${l.missing.map(date).join(", ")}` + : `first run ${date(l.since)}`; + const key = [...on(l.fixtures), why].join(", "); + reasons.set(key, [...(reasons.get(key) ?? []), `\`${l.scenario}\``]); + } + const notes = [ + ...(reasons.size === 0 + ? [] + : [ + `Left out, so every run in the trend window averages the same scenarios: ${[...reasons] + .map(([why, names]) => `${names.join(", ")} (${why})`) + .join("; ")}.`, + ]), + ...(fallback.length === 0 + ? [] + : [ + `${["Too few scenarios ran on every run in the window", ...on(fallback)].join(" ")} to average, so this run's are averaged there, and runs without one of them have no point.`, + ]), + ]; + return notes.length === 0 ? null : notes.join(" "); +}; /** The run and the filters a report is for: what the page's menus, picker and chips show. */ export const runMarkdown = (report: ReportModel): string[] => { @@ -625,6 +666,7 @@ export const historyMarkdown = (history: History, origin: string): string => { `- Window: ${history.window.from} to ${history.window.to}${history.window.custom ? "" : " (a preset window ending at the run)"}`, `- Fixture: ${history.fixture ?? `all (${history.fixtures.join(", ")})`}`, `- Values: ${history.unit}${history.derived ? " (derived from total time)" : ""}`, + ...(history.averageOf === null ? [] : [`- Averaged: ${meanOf(history.averageOf)}`]), `- Tools: ${history.tools.map((t) => (label(t) === t ? t : `${label(t)} (${t})`)).join(", ")}`, `- Excluded runs: ${history.excluded.length === 0 ? "none" : history.excluded.join(", ")}`, "", diff --git a/apps/www/src/lib/measurements.ts b/apps/www/src/lib/measurements.ts index 937233eb90..a59edf590d 100644 --- a/apps/www/src/lib/measurements.ts +++ b/apps/www/src/lib/measurements.ts @@ -207,6 +207,85 @@ export const averageFiles = ( return [{ fixture, file: { results } }]; }); +/** A scenario the window's average leaves out, and why. */ +export interface LeftOut { + readonly scenario: string; + /** The fixtures it is left out on. */ + readonly fixtures: readonly string[]; + /** The window's runs without it, ascending. */ + readonly missing: readonly string[]; + /** Its first run in the window, when every run without it is earlier; else `null`. */ + readonly since: string | null; +} + +/** The scenarios a window's average covers on each fixture. */ +export interface CommonPlan { + readonly plan: { readonly [fixture: string]: readonly string[] }; + /** In `plan`'s scenario order. */ + readonly leftOut: readonly LeftOut[]; + /** + * Fixtures where too few scenarios ran on every run to average, so they keep + * the shown run's and a run without one of them draws no point for them. + */ + readonly fallback: readonly string[]; +} + +/** + * The scenarios of `plan` every run in the window published, per fixture. + * + * `averageFiles` drops a fixture missing any planned scenario, so a plan taken + * from the shown run drew no point on any run before a scenario was added. + * Averaging only what each fixture has on every run it ran keeps the trend and + * the shown run's numbers over the same scenarios. A fixture left with fewer + * than two (when its plan has two or more) keeps the shown run's set, and is + * named in `fallback`. + */ +export const commonPlan = ( + plan: { readonly [fixture: string]: readonly string[] }, + runs: readonly { + readonly date: string; + readonly cells: readonly { readonly fixture: string; readonly variation: string }[]; + }[], +): CommonPlan => { + const has = (cells: (typeof runs)[number]["cells"], fixture: string, variation: string) => + cells.some((c) => c.fixture === fixture && c.variation === variation); + const common: { [fixture: string]: readonly string[] } = {}; + const fallback: string[] = []; + const dropped = new Map< + string, + { fixtures: string[]; missing: Set; present: string[] } + >(); + for (const [fixture, variations] of Object.entries(plan)) { + /* A run the fixture did not run on at all says nothing about its scenarios. */ + const ran = runs.filter((r) => variations.some((v) => has(r.cells, fixture, v))); + const kept = variations.filter((v) => ran.every((r) => has(r.cells, fixture, v))); + if (kept.length < Math.min(2, variations.length)) { + common[fixture] = variations; + fallback.push(fixture); + continue; + } + common[fixture] = kept; + for (const v of variations.filter((v) => !kept.includes(v))) { + const entry = dropped.get(v) ?? { fixtures: [], missing: new Set(), present: [] }; + entry.fixtures.push(fixture); + for (const r of ran) { + if (has(r.cells, fixture, v)) entry.present.push(r.date); + else entry.missing.add(r.date); + } + dropped.set(v, entry); + } + } + const leftOut = [...new Set(Object.values(plan).flat())].flatMap((scenario) => { + const entry = dropped.get(scenario); + if (entry === undefined) return []; + const missing = [...entry.missing].sort(); + const first = entry.present.sort()[0]; + const since = first !== undefined && missing.every((d) => d < first) ? first : null; + return [{ scenario, fixtures: entry.fixtures, missing, since }]; + }); + return { plan: common, leftOut, fallback }; +}; + const failedIn = (file: BenchmarkFile, tools: readonly string[]): string[] => file.results.filter((r) => tools.includes(r.command) && !usable(r)).map((r) => r.command); diff --git a/apps/www/src/lib/report.ts b/apps/www/src/lib/report.ts index a8792c84e5..c840d03f66 100644 --- a/apps/www/src/lib/report.ts +++ b/apps/www/src/lib/report.ts @@ -23,6 +23,7 @@ import { countRows, perPackageByFixture, ratioTo, + type CommonPlan, type CountRow, type FixtureRow, type ToolMean, @@ -96,10 +97,17 @@ export interface ReportModel { readonly state: ScenarioState; /** What was on disk before each run; `null` where the name encodes no state. */ readonly stateParts: ReturnType | null; - /** On the derived view, the scenarios averaged, in all and on each fixture shown; `null` on a measured one. */ + /** + * On the derived view, the scenarios averaged, in all and on each fixture + * shown: those every run in the trend window has, so they depend on the + * window. `leftOut` and `fallback` say what that cost (`commonPlan`). + * `null` on a measured one. + */ readonly averageOf: { readonly scenarios: readonly string[]; readonly byFixture: { readonly [fixture: string]: readonly string[] }; + readonly leftOut: CommonPlan["leftOut"]; + readonly fallback: CommonPlan["fallback"]; } | null; }; readonly run: { @@ -583,11 +591,15 @@ export const loadReport = async ({ state: selected.state, stateParts: selected.kind === "other" ? null : stateParts(selected.state), averageOf: - averageOf === null + averageOf === null || average === null ? null : { - scenarios: averageOf.filter((v) => Object.values(shownPlan).flat().includes(v)), - byFixture: shownPlan, + scenarios: averageOf.filter((v) => + Object.values(average.averaged.plan).flat().includes(v), + ), + byFixture: average.averaged.plan, + leftOut: average.averaged.leftOut, + fallback: average.averaged.fallback, }, }, run: { date, latest: index.latest, isLatest: date === index.latest, dates: index.dates }, diff --git a/apps/www/test/average.test.ts b/apps/www/test/average.test.ts index b205c000ec..5cb28cca6c 100644 --- a/apps/www/test/average.test.ts +++ b/apps/www/test/average.test.ts @@ -6,12 +6,13 @@ import { GET as api } from "../src/app/api/v1/[section]/[scenario]/route.ts"; import reportTwin, { slots } from "../src/app/[section]/[[...params]]/markdown.ts"; import { GET as llms } from "../src/app/llms.txt/route.ts"; import { TREND_BUDGET, trendDates } from "../src/lib/load.ts"; -import { averageFiles } from "../src/lib/measurements.ts"; +import { averagedNote, reportMarkdown } from "../src/lib/markdown.ts"; +import { averageFiles, commonPlan } from "../src/lib/measurements.ts"; import { treeFor } from "../src/lib/page-tree.ts"; import { loadReportParams } from "../src/lib/params.ts"; import { loadReport } from "../src/lib/report.ts"; import { AVERAGE, scenariosFor, withAverage } from "../src/lib/scenarios.ts"; -import { DATA_URL, fakeFetch } from "./fixture-tree.ts"; +import { DATA_URL, DATES, fakeFetch } from "./fixture-tree.ts"; const ORIGIN = "https://benchmarks.vlt.io"; @@ -97,6 +98,99 @@ describe("averageFiles", () => { }); }); +describe("commonPlan", () => { + const plan = { + astro: ["clean", "cache", "add"], + babylon: ["clean", "cache", "add"], + }; + const run = (date: string, cells: { [fixture: string]: readonly string[] }) => ({ + date, + cells: Object.entries(cells).flatMap(([fixture, vs]) => + vs.map((variation) => ({ fixture, variation })), + ), + }); + const all = { astro: plan.astro, babylon: plan.babylon }; + const before = { astro: ["clean", "cache"], babylon: ["clean", "cache"] }; + + it("leaves out a scenario added mid-window, and says when it starts", () => { + const common = commonPlan(plan, [run("d1", before), run("d2", before), run("d3", all)]); + expect(common.plan).toEqual(before); + expect(common.leftOut).toEqual([ + { scenario: "add", fixtures: ["astro", "babylon"], missing: ["d1", "d2"], since: "d3" }, + ]); + expect(common.fallback).toEqual([]); + }); + + it("averages everything in a window that starts after it was added", () => { + expect(commonPlan(plan, [run("d3", all), run("d4", all)])).toEqual({ + plan: all, + leftOut: [], + fallback: [], + }); + }); + + it("leaves out a scenario missing on one run, on that fixture only, and names the run", () => { + const gap = { astro: ["clean", "add"], babylon: plan.babylon }; + const common = commonPlan(plan, [run("d1", all), run("d2", gap), run("d3", all)]); + expect(common.plan).toEqual({ astro: ["clean", "add"], babylon: plan.babylon }); + expect(common.leftOut).toEqual([ + { scenario: "cache", fixtures: ["astro"], missing: ["d2"], since: null }, + ]); + }); + + it("ignores a run a fixture did not run on at all", () => { + const common = commonPlan(plan, [run("d1", { babylon: plan.babylon }), run("d2", all)]); + expect(common.plan).toEqual(all); + }); + + it("falls back to the shown run's set when fewer than two are common", () => { + const common = commonPlan(plan, [ + run("d1", { astro: ["clean"], babylon: ["clean"] }), + run("d2", all), + ]); + expect(common.plan).toEqual(all); + expect(common.fallback).toEqual(["astro", "babylon"]); + expect(common.leftOut).toEqual([]); + }); + + it("covers only the fixtures it is given", () => { + const common = commonPlan({ astro: plan.astro }, [run("d1", before), run("d2", all)]); + expect(common.plan).toEqual({ astro: ["clean", "cache"] }); + expect(common.leftOut).toEqual([ + { scenario: "add", fixtures: ["astro"], missing: ["d1"], since: "d2" }, + ]); + }); +}); + +describe("averagedNote", () => { + const averageOf = { + scenarios: ["clean"], + byFixture: { astro: ["clean"], babylon: ["clean", "cache"] }, + leftOut: [ + { scenario: "add", fixtures: ["astro", "babylon"], missing: ["d1"], since: "d2" }, + { scenario: "remove", fixtures: ["astro", "babylon"], missing: ["d1"], since: "d2" }, + { scenario: "cache", fixtures: ["astro"], missing: ["d2"], since: null }, + ], + fallback: [], + }; + + it("names what is left out, grouped by why", () => { + expect(averagedNote(averageOf)).toBe( + "Left out, so every run in the trend window averages the same scenarios: `add`, `remove` (first run d2); `cache` (on Astro, missing on d2).", + ); + }); + + it("says nothing when nothing is left out", () => { + expect(averagedNote({ ...averageOf, leftOut: [] })).toBeNull(); + }); + + it("says when it fell back", () => { + expect(averagedNote({ ...averageOf, leftOut: [], fallback: ["astro", "babylon"] })).toBe( + "Too few scenarios ran on every run in the window to average, so this run's are averaged there, and runs without one of them have no point.", + ); + }); +}); + describe("withAverage", () => { it("leads with the derived view, over the curated scenarios only", () => { const list = withAverage(scenariosFor("package-managers", ["cache", "clean", "ci+setup"])); @@ -160,11 +254,63 @@ describe("the average report", () => { expect(report.scenario.averageOf).toEqual({ scenarios: ["clean", "cache"], byFixture: { astro: ["clean", "cache"], babylon: ["clean", "cache"] }, + leftOut: [], + fallback: [], }); expect(report.finding.sentence).toBe("bun is fastest at 1.50s, 10.0× faster than npm."); expect(report.timeRows.find((r) => r.fixture === "babylon")?.failed).toEqual(["zpm"]); }); + it("averages only what every run in the window has, and says what it left out", async () => { + /* `add-singleton` first published on the latest run: `clean`'s files again, on that run only. */ + const added = async (url: string | URL | Request) => { + const path = decodeURIComponent(String(url).slice(DATA_URL.length + 1)); + const add = /^2026-10-13\/(.+)-add-singleton\.json$/.exec(path); + if (add !== null) return fakeFetch(`${DATA_URL}/2026-10-13/${add[1]}-clean.json`); + const res = await withCache(url); + if (path !== "manifest/2026-10-13.json" || !res.ok) return res; + const entry = (await res.json()) as { benchmarks: { [f: string]: string[] } }; + const benchmarks = Object.fromEntries( + Object.entries(entry.benchmarks).map(([f, v]) => [f, [...v, "add-singleton"]]), + ); + return { ...res, json: async () => ({ ...entry, benchmarks }) }; + }; + const at = async (search: string) => { + const r = await loadReport({ + section: "package-managers", + scenario: AVERAGE, + query: loadReportParams(new URLSearchParams(search)), + client: new DataClient({ baseUrl: DATA_URL, fetch: added }), + }); + if (r.status !== "ok") throw new Error(r.status); + return r.report; + }; + + const window = await at("from=2026-10-10&to=2026-10-13"); + expect(window.scenario.averageOf?.scenarios).toEqual(["clean", "cache"]); + expect(window.scenario.averageOf?.leftOut).toEqual([ + { + scenario: "add-singleton", + fixtures: ["astro", "babylon"], + missing: ["2026-10-10", "2026-10-12"], + since: "2026-10-13", + }, + ]); + expect(window.trend.points.map((p) => p.date)).toEqual(DATES); + expect(window.raw.map((r) => r.label)).not.toContain("Astro · add-singleton"); + expect(reportMarkdown(window, ORIGIN)).toContain( + "Left out, so every run in the trend window averages the same scenarios: `add-singleton` (first run 2026-10-13).", + ); + + const latest = await at("from=2026-10-13&to=2026-10-13"); + expect(latest.scenario.averageOf?.scenarios).toEqual(["clean", "cache", "add-singleton"]); + expect(latest.scenario.averageOf?.leftOut).toEqual([]); + + const astro = await at("from=2026-10-10&to=2026-10-13&fixture=astro"); + expect(astro.scenario.averageOf?.byFixture).toEqual({ astro: ["clean", "cache"] }); + expect(astro.scenario.averageOf?.leftOut[0]?.fixtures).toEqual(["astro"]); + }); + it("has no counts, no per-package metric, and cites every file it averaged", async () => { const report = await load("metric=per-package"); expect(report.outline.map((s) => s.id)).toEqual(["leaderboard", "time"]); From 2d5877fec5427d0e7e69845dffd9494d8cf31197 Mon Sep 17 00:00:00 2001 From: Luke Karrys Date: Wed, 18 Nov 2026 05:47:21 +0000 Subject: [PATCH 2/2] fix(www): name every scenario the window's average leaves out, truthfully Review fixes for #217: - The note no longer calls a run a scenario's "first run": it describes only the runs considered ("not in runs through ", "since", or the runs it is missing from), since the real first run can be excluded or outside the window. LeftOut.since becomes LeftOut.when. - The average's candidates are the section's full curated list on each fixture the shown run averages anything on, and commonPlan treats the shown run like any other. A scenario the shown run lacks but other runs have is now named, not silently dropped. Scenarios no run has aren't offered; the fallback uses the scenarios the shown run has. - Fallback wording: only that fixture leaves the runs without them. - OpenAPI: document History.averageOf. - Revert the unrelated apps/www/AGENTS.md heading change. Co-Authored-By: Claude Opus 5.5 --- apps/www/AGENTS.md | 2 +- apps/www/src/app/openapi.json/route.ts | 25 +++++++ apps/www/src/lib/load.ts | 4 +- apps/www/src/lib/markdown.ts | 13 ++-- apps/www/src/lib/measurements.ts | 48 +++++++++----- apps/www/src/lib/report.ts | 16 +++-- apps/www/test/average.test.ts | 92 +++++++++++++++++++------- 7 files changed, 146 insertions(+), 54 deletions(-) diff --git a/apps/www/AGENTS.md b/apps/www/AGENTS.md index af3b7c8411..643577dfae 100644 --- a/apps/www/AGENTS.md +++ b/apps/www/AGENTS.md @@ -1,6 +1,6 @@ -## This is NOT the Next.js you know +# This is NOT the Next.js you know This version has breaking changes — APIs, conventions, and file structure may all differ from your training data. Read the relevant guide in `node_modules/next/dist/docs/` (resolved from this file's directory; in monorepos the `next` package may not be visible from the repo root) before writing any code. Heed deprecation notices. diff --git a/apps/www/src/app/openapi.json/route.ts b/apps/www/src/app/openapi.json/route.ts index f65ae141bf..6cbb53874e 100644 --- a/apps/www/src/app/openapi.json/route.ts +++ b/apps/www/src/app/openapi.json/route.ts @@ -284,6 +284,31 @@ export const GET = (request: Request) => properties: { section: { type: "string" }, scenario: { type: "string" }, + averageOf: { + type: ["object", "null"], + description: + "On the derived average: the scenarios averaged (those every run in the window has), in all and per fixture, the ones left out and why, and the fixtures that fell back to the shown run's set. Null on a measured scenario.", + properties: { + scenarios: { type: "array", items: { type: "string" } }, + byFixture: { + type: "object", + additionalProperties: { type: "array", items: { type: "string" } }, + }, + leftOut: { + type: "array", + items: { + type: "object", + properties: { + scenario: { type: "string" }, + fixtures: { type: "array", items: { type: "string" } }, + missing: { type: "array", items: date }, + when: { type: "string", enum: ["before", "after", "between"] }, + }, + }, + }, + fallback: { type: "array", items: { type: "string" } }, + }, + }, run: date, fixture: { type: ["string", "null"] }, fixtures: { type: "array", items: { type: "string" } }, diff --git a/apps/www/src/lib/load.ts b/apps/www/src/lib/load.ts index f9d673ad32..300a3e224a 100644 --- a/apps/www/src/lib/load.ts +++ b/apps/www/src/lib/load.ts @@ -388,7 +388,8 @@ const loadCells = async ( /** * The derived "average" view, as a scenario: `plan` maps each fixture to the - * scenarios that may be averaged on it, and `averaged` to the ones that are: + * scenarios that may be averaged on it (the curated list, whether or not the + * shown run has them), and `averaged` to the ones that are: * those every drawn run has (`commonPlan`), so the trend and the shown run * cover the same scenarios. `cells` are the shown run's per-scenario files. * No count files -- a package or process count is a fact about one scenario, @@ -426,6 +427,7 @@ export const loadAverage = async ( dates.flatMap((d, i) => d !== date && whole.has(d) ? [] : [{ date: d, cells: loaded[i] ?? [] }], ), + date, ); const trend = drawn.flatMap( (d, i) => trendPoint(d, averageFiles(loaded[i] ?? [], averaged.plan)) ?? [], diff --git a/apps/www/src/lib/markdown.ts b/apps/www/src/lib/markdown.ts index c5dbba6224..c028388e54 100644 --- a/apps/www/src/lib/markdown.ts +++ b/apps/www/src/lib/markdown.ts @@ -463,13 +463,16 @@ export const averagedNote = ( /* Where it applies, when that is not every fixture shown. */ const on = (some: readonly string[]) => some.length === fixtures.length ? [] : [`on ${some.map(fixtureLabel).join(", ")}`]; - /* Scenarios with the same reason share it: the add/remove four all start the same day. */ + /* Scenarios with the same reason share it: the add/remove four all start on the same run. */ const reasons = new Map(); for (const l of leftOut) { + /* Said of the runs considered only: its real first run may be excluded or outside the window. */ const why = - l.since === null - ? `missing on ${l.missing.map(date).join(", ")}` - : `first run ${date(l.since)}`; + l.when === "before" + ? `not in runs through ${date(l.missing.at(-1) ?? "")}` + : l.when === "after" + ? `not in runs since ${date(l.missing[0] ?? "")}` + : `not in the runs of ${l.missing.map(date).join(", ")}`; const key = [...on(l.fixtures), why].join(", "); reasons.set(key, [...(reasons.get(key) ?? []), `\`${l.scenario}\``]); } @@ -484,7 +487,7 @@ export const averagedNote = ( ...(fallback.length === 0 ? [] : [ - `${["Too few scenarios ran on every run in the window", ...on(fallback)].join(" ")} to average, so this run's are averaged there, and runs without one of them have no point.`, + `${["Too few scenarios ran on every run in the window", ...on(fallback)].join(" ")} to average, so this run's are averaged there, and ${fallback.length === 1 ? "that fixture is" : "those fixtures are"} left out of runs without one of them.`, ]), ]; return notes.length === 0 ? null : notes.join(" "); diff --git a/apps/www/src/lib/measurements.ts b/apps/www/src/lib/measurements.ts index a59edf590d..9545d076a8 100644 --- a/apps/www/src/lib/measurements.ts +++ b/apps/www/src/lib/measurements.ts @@ -212,10 +212,14 @@ export interface LeftOut { readonly scenario: string; /** The fixtures it is left out on. */ readonly fixtures: readonly string[]; - /** The window's runs without it, ascending. */ + /** The runs considered that lack it, ascending. */ readonly missing: readonly string[]; - /** Its first run in the window, when every run without it is earlier; else `null`. */ - readonly since: string | null; + /** + * Where `missing` sits among the runs considered: all before every run that + * has it, all after, or neither. Only those runs: the scenario's real first + * run may be excluded, or outside the window. + */ + readonly when: "before" | "after" | "between"; } /** The scenarios a window's average covers on each fixture. */ @@ -225,20 +229,21 @@ export interface CommonPlan { readonly leftOut: readonly LeftOut[]; /** * Fixtures where too few scenarios ran on every run to average, so they keep - * the shown run's and a run without one of them draws no point for them. + * the shown run's, and are left out of the runs without one of them. */ readonly fallback: readonly string[]; } /** - * The scenarios of `plan` every run in the window published, per fixture. + * The scenarios of `plan` every run considered published, per fixture. * * `averageFiles` drops a fixture missing any planned scenario, so a plan taken * from the shown run drew no point on any run before a scenario was added. * Averaging only what each fixture has on every run it ran keeps the trend and - * the shown run's numbers over the same scenarios. A fixture left with fewer - * than two (when its plan has two or more) keeps the shown run's set, and is - * named in `fallback`. + * the shown run's numbers over the same scenarios. `plan` is every candidate + * (the curated list); one no run has is not offered, and is not named. The + * shown run is one of `runs` like any other. A fixture left with fewer than + * two keeps the scenarios the `shown` run has, and is named in `fallback`. */ export const commonPlan = ( plan: { readonly [fixture: string]: readonly string[] }, @@ -246,6 +251,7 @@ export const commonPlan = ( readonly date: string; readonly cells: readonly { readonly fixture: string; readonly variation: string }[]; }[], + shown?: string, ): CommonPlan => { const has = (cells: (typeof runs)[number]["cells"], fixture: string, variation: string) => cells.some((c) => c.fixture === fixture && c.variation === variation); @@ -255,17 +261,19 @@ export const commonPlan = ( string, { fixtures: string[]; missing: Set; present: string[] } >(); - for (const [fixture, variations] of Object.entries(plan)) { + for (const [fixture, candidates] of Object.entries(plan)) { /* A run the fixture did not run on at all says nothing about its scenarios. */ - const ran = runs.filter((r) => variations.some((v) => has(r.cells, fixture, v))); - const kept = variations.filter((v) => ran.every((r) => has(r.cells, fixture, v))); - if (kept.length < Math.min(2, variations.length)) { - common[fixture] = variations; + const ran = runs.filter((r) => candidates.some((v) => has(r.cells, fixture, v))); + const offered = candidates.filter((v) => ran.some((r) => has(r.cells, fixture, v))); + const kept = offered.filter((v) => ran.every((r) => has(r.cells, fixture, v))); + if (kept.length < Math.min(2, offered.length)) { + const own = runs.find((r) => r.date === shown)?.cells; + common[fixture] = own === undefined ? offered : offered.filter((v) => has(own, fixture, v)); fallback.push(fixture); continue; } common[fixture] = kept; - for (const v of variations.filter((v) => !kept.includes(v))) { + for (const v of offered.filter((v) => !kept.includes(v))) { const entry = dropped.get(v) ?? { fixtures: [], missing: new Set(), present: [] }; entry.fixtures.push(fixture); for (const r of ran) { @@ -279,9 +287,15 @@ export const commonPlan = ( const entry = dropped.get(scenario); if (entry === undefined) return []; const missing = [...entry.missing].sort(); - const first = entry.present.sort()[0]; - const since = first !== undefined && missing.every((d) => d < first) ? first : null; - return [{ scenario, fixtures: entry.fixtures, missing, since }]; + const present = entry.present.sort(); + const first = present[0] ?? ""; + const last = present.at(-1) ?? ""; + const when: LeftOut["when"] = missing.every((d) => d < first) + ? "before" + : missing.every((d) => d > last) + ? "after" + : "between"; + return [{ scenario, fixtures: entry.fixtures, missing, when }]; }); return { plan: common, leftOut, fallback }; }; diff --git a/apps/www/src/lib/report.ts b/apps/www/src/lib/report.ts index c840d03f66..e4b6ce8bfa 100644 --- a/apps/www/src/lib/report.ts +++ b/apps/www/src/lib/report.ts @@ -316,14 +316,16 @@ export const loadReport = async ({ } const averageOf = selected.averageOf ?? null; - /* The derived view: each fixture, and the scenarios averaged on it. */ + /* + * The derived view: each fixture the shown run averages anything on, and + * every curated scenario as a candidate, published by this run or not. + * `loadAverage` keeps those every run in the window has, so a scenario this + * run lacks and others have is named as left out, not silently missing. + */ const plan: { [fixture: string]: readonly string[] } = Object.fromEntries( Object.entries(entry.benchmarks) - .map(([f, published]): [string, string[]] => [ - f, - (averageOf ?? []).filter((v) => published?.includes(v)), - ]) - .filter(([, averaged]) => averaged.length > 0) + .filter(([, published]) => (averageOf ?? []).some((v) => published?.includes(v))) + .map(([f]): [string, readonly string[]] => [f, def.order]) .sort(([a], [b]) => a.localeCompare(b)), ); const publishedFixtures = @@ -594,7 +596,7 @@ export const loadReport = async ({ averageOf === null || average === null ? null : { - scenarios: averageOf.filter((v) => + scenarios: def.order.filter((v) => Object.values(average.averaged.plan).flat().includes(v), ), byFixture: average.averaged.plan, diff --git a/apps/www/test/average.test.ts b/apps/www/test/average.test.ts index 5cb28cca6c..2f1efe8ac8 100644 --- a/apps/www/test/average.test.ts +++ b/apps/www/test/average.test.ts @@ -116,7 +116,7 @@ describe("commonPlan", () => { const common = commonPlan(plan, [run("d1", before), run("d2", before), run("d3", all)]); expect(common.plan).toEqual(before); expect(common.leftOut).toEqual([ - { scenario: "add", fixtures: ["astro", "babylon"], missing: ["d1", "d2"], since: "d3" }, + { scenario: "add", fixtures: ["astro", "babylon"], missing: ["d1", "d2"], when: "before" }, ]); expect(common.fallback).toEqual([]); }); @@ -134,7 +134,7 @@ describe("commonPlan", () => { const common = commonPlan(plan, [run("d1", all), run("d2", gap), run("d3", all)]); expect(common.plan).toEqual({ astro: ["clean", "add"], babylon: plan.babylon }); expect(common.leftOut).toEqual([ - { scenario: "cache", fixtures: ["astro"], missing: ["d2"], since: null }, + { scenario: "cache", fixtures: ["astro"], missing: ["d2"], when: "between" }, ]); }); @@ -144,20 +144,46 @@ describe("commonPlan", () => { }); it("falls back to the shown run's set when fewer than two are common", () => { - const common = commonPlan(plan, [ - run("d1", { astro: ["clean"], babylon: ["clean"] }), - run("d2", all), - ]); - expect(common.plan).toEqual(all); + const common = commonPlan( + plan, + [run("d1", { astro: ["clean"], babylon: ["clean"] }), run("d2", before)], + "d2", + ); + expect(common.plan).toEqual(before); expect(common.fallback).toEqual(["astro", "babylon"]); expect(common.leftOut).toEqual([]); }); + it("does not offer a scenario no run has", () => { + expect(commonPlan(plan, [run("d1", before), run("d2", before)])).toEqual({ + plan: before, + leftOut: [], + fallback: [], + }); + }); + + it("names a scenario the shown run lacks and an earlier run has", () => { + const shown = { astro: ["clean", "cache"], babylon: plan.babylon }; + const common = commonPlan(plan, [run("d1", all), run("d2", shown)], "d2"); + expect(common.plan).toEqual({ astro: ["clean", "cache"], babylon: plan.babylon }); + expect(common.leftOut).toEqual([ + { scenario: "add", fixtures: ["astro"], missing: ["d2"], when: "after" }, + ]); + }); + + it("describes only the runs it is given, not an excluded first run", () => { + // d2 is the scenario's real first run, excluded by the caller: d3 is not its first. + const common = commonPlan(plan, [run("d1", before), run("d3", all), run("d4", all)]); + expect(common.leftOut).toEqual([ + { scenario: "add", fixtures: ["astro", "babylon"], missing: ["d1"], when: "before" }, + ]); + }); + it("covers only the fixtures it is given", () => { const common = commonPlan({ astro: plan.astro }, [run("d1", before), run("d2", all)]); expect(common.plan).toEqual({ astro: ["clean", "cache"] }); expect(common.leftOut).toEqual([ - { scenario: "add", fixtures: ["astro"], missing: ["d1"], since: "d2" }, + { scenario: "add", fixtures: ["astro"], missing: ["d1"], when: "before" }, ]); }); }); @@ -167,16 +193,16 @@ describe("averagedNote", () => { scenarios: ["clean"], byFixture: { astro: ["clean"], babylon: ["clean", "cache"] }, leftOut: [ - { scenario: "add", fixtures: ["astro", "babylon"], missing: ["d1"], since: "d2" }, - { scenario: "remove", fixtures: ["astro", "babylon"], missing: ["d1"], since: "d2" }, - { scenario: "cache", fixtures: ["astro"], missing: ["d2"], since: null }, + { scenario: "add", fixtures: ["astro", "babylon"], missing: ["d1"], when: "before" }, + { scenario: "remove", fixtures: ["astro", "babylon"], missing: ["d3"], when: "after" }, + { scenario: "cache", fixtures: ["astro"], missing: ["d2", "d4"], when: "between" }, ], fallback: [], - }; + } satisfies Parameters[0]; it("names what is left out, grouped by why", () => { expect(averagedNote(averageOf)).toBe( - "Left out, so every run in the trend window averages the same scenarios: `add`, `remove` (first run d2); `cache` (on Astro, missing on d2).", + "Left out, so every run in the trend window averages the same scenarios: `add` (not in runs through d1); `remove` (not in runs since d3); `cache` (on Astro, not in the runs of d2, d4).", ); }); @@ -186,7 +212,7 @@ describe("averagedNote", () => { it("says when it fell back", () => { expect(averagedNote({ ...averageOf, leftOut: [], fallback: ["astro", "babylon"] })).toBe( - "Too few scenarios ran on every run in the window to average, so this run's are averaged there, and runs without one of them have no point.", + "Too few scenarios ran on every run in the window to average, so this run's are averaged there, and those fixtures are left out of runs without one of them.", ); }); }); @@ -262,25 +288,30 @@ describe("the average report", () => { }); it("averages only what every run in the window has, and says what it left out", async () => { - /* `add-singleton` first published on the latest run: `clean`'s files again, on that run only. */ - const added = async (url: string | URL | Request) => { + /* `add-singleton` published, as `clean`'s files again, on the runs given only. */ + const addedOn = (runs: readonly string[]) => async (url: string | URL | Request) => { const path = decodeURIComponent(String(url).slice(DATA_URL.length + 1)); - const add = /^2026-10-13\/(.+)-add-singleton\.json$/.exec(path); - if (add !== null) return fakeFetch(`${DATA_URL}/2026-10-13/${add[1]}-clean.json`); + const add = /^([\d-]+)\/(.+)-add-singleton\.json$/.exec(path); + if (add !== null) { + return runs.includes(add[1] ?? "") + ? fakeFetch(`${DATA_URL}/${add[1]}/${add[2]}-clean.json`) + : { ok: false, status: 404, json: async () => undefined }; + } const res = await withCache(url); - if (path !== "manifest/2026-10-13.json" || !res.ok) return res; + const run = /^manifest\/(.+)\.json$/.exec(path)?.[1]; + if (run === undefined || !runs.includes(run) || !res.ok) return res; const entry = (await res.json()) as { benchmarks: { [f: string]: string[] } }; const benchmarks = Object.fromEntries( Object.entries(entry.benchmarks).map(([f, v]) => [f, [...v, "add-singleton"]]), ); return { ...res, json: async () => ({ ...entry, benchmarks }) }; }; - const at = async (search: string) => { + const at = async (search: string, runs: readonly string[] = ["2026-10-13"]) => { const r = await loadReport({ section: "package-managers", scenario: AVERAGE, query: loadReportParams(new URLSearchParams(search)), - client: new DataClient({ baseUrl: DATA_URL, fetch: added }), + client: new DataClient({ baseUrl: DATA_URL, fetch: addedOn(runs) }), }); if (r.status !== "ok") throw new Error(r.status); return r.report; @@ -293,13 +324,28 @@ describe("the average report", () => { scenario: "add-singleton", fixtures: ["astro", "babylon"], missing: ["2026-10-10", "2026-10-12"], - since: "2026-10-13", + when: "before", }, ]); expect(window.trend.points.map((p) => p.date)).toEqual(DATES); expect(window.raw.map((r) => r.label)).not.toContain("Astro · add-singleton"); expect(reportMarkdown(window, ORIGIN)).toContain( - "Left out, so every run in the trend window averages the same scenarios: `add-singleton` (first run 2026-10-13).", + "Left out, so every run in the trend window averages the same scenarios: `add-singleton` (not in runs through 2026-10-12).", + ); + + /* Shown at a run without it, after runs that have it: still named, not silently missing. */ + const dropped = await at("from=2026-10-10&to=2026-10-13", ["2026-10-10", "2026-10-12"]); + expect(dropped.scenario.averageOf?.scenarios).toEqual(["clean", "cache"]); + expect(dropped.scenario.averageOf?.leftOut).toEqual([ + { + scenario: "add-singleton", + fixtures: ["astro", "babylon"], + missing: ["2026-10-13"], + when: "after", + }, + ]); + expect(reportMarkdown(dropped, ORIGIN)).toContain( + "`add-singleton` (not in runs since 2026-10-13).", ); const latest = await at("from=2026-10-13&to=2026-10-13");