diff --git a/apps/www/src/app/[section]/[[...params]]/page.tsx b/apps/www/src/app/[section]/[[...params]]/page.tsx
index da81d7ed21..b632c27df5 100644
--- a/apps/www/src/app/[section]/[[...params]]/page.tsx
+++ b/apps/www/src/app/[section]/[[...params]]/page.tsx
@@ -42,7 +42,7 @@ import { toTrendColumns } from "@/lib/chart-model";
import { loadReportParams, rangeLabel, reportHref, reportParams } from "@/lib/params";
import { loadCatalog } from "@/lib/catalog";
import { dataClient } from "@/lib/data";
-import { PER_PACKAGE_DERIVED } from "@/lib/markdown";
+import { averagedNote, PER_PACKAGE_DERIVED } from "@/lib/markdown";
import { loadReport } from "@/lib/report";
import { withAverage } from "@/lib/scenarios";
import { isSectionId, SECTION_DEFS, SECTIONS } from "@/lib/sections";
@@ -188,6 +188,7 @@ const Report = async ({
const report = result.report;
const { scenario, run, filters, finding, baseline, caveats, trend } = report;
+ const note = scenario.averageOf === null ? null : averagedNote(scenario.averageOf, longDate);
const { date } = run;
const { fixture, hide: hidden } = filters;
const { measure, isScript, outline, ranked, publishedFixtures, path } = report;
@@ -641,6 +642,12 @@ const Report = async ({
? " Every registry is installed from with the same npm client."
: ""}
{run.isLatest ? null : " This is not the latest run."}
+ {note === null ? null : (
+ <>
+ {" "}
+
+ >
+ )}
>
}
action={
diff --git a/apps/www/src/app/openapi.json/route.ts b/apps/www/src/app/openapi.json/route.ts
index f65ae141bf..6cbb53874e 100644
--- a/apps/www/src/app/openapi.json/route.ts
+++ b/apps/www/src/app/openapi.json/route.ts
@@ -284,6 +284,31 @@ export const GET = (request: Request) =>
properties: {
section: { type: "string" },
scenario: { type: "string" },
+ averageOf: {
+ type: ["object", "null"],
+ description:
+ "On the derived average: the scenarios averaged (those every run in the window has), in all and per fixture, the ones left out and why, and the fixtures that fell back to the shown run's set. Null on a measured scenario.",
+ properties: {
+ scenarios: { type: "array", items: { type: "string" } },
+ byFixture: {
+ type: "object",
+ additionalProperties: { type: "array", items: { type: "string" } },
+ },
+ leftOut: {
+ type: "array",
+ items: {
+ type: "object",
+ properties: {
+ scenario: { type: "string" },
+ fixtures: { type: "array", items: { type: "string" } },
+ missing: { type: "array", items: date },
+ when: { type: "string", enum: ["before", "after", "between"] },
+ },
+ },
+ },
+ fallback: { type: "array", items: { type: "string" } },
+ },
+ },
run: date,
fixture: { type: ["string", "null"] },
fixtures: { type: "array", items: { type: "string" } },
diff --git a/apps/www/src/lib/api.ts b/apps/www/src/lib/api.ts
index 40d7cc8abd..1bc6a53cf2 100644
--- a/apps/www/src/lib/api.ts
+++ b/apps/www/src/lib/api.ts
@@ -112,6 +112,8 @@ export const historyOf = (report: ReportModel, tools?: readonly string[]) => {
return {
section: report.section.id,
scenario: report.scenario.variation,
+ /* On the derived view, what the trend averages: the scenarios every run in the window has. */
+ averageOf: report.scenario.averageOf,
run: report.run.date,
fixture: report.filters.fixture,
fixtures: report.fixtures,
diff --git a/apps/www/src/lib/load.ts b/apps/www/src/lib/load.ts
index 8b868f572e..300a3e224a 100644
--- a/apps/www/src/lib/load.ts
+++ b/apps/www/src/lib/load.ts
@@ -16,7 +16,7 @@ import {
} from "@benchmarks/data";
import { probe } from "./data";
import { applyExclusions } from "./figures";
-import { averageFiles, usable } from "./measurements";
+import { averageFiles, commonPlan, usable, type CommonPlan } from "./measurements";
export interface FixtureBundle {
readonly fixture: string;
@@ -388,9 +388,12 @@ const loadCells = async (
/**
* The derived "average" view, as a scenario: `plan` maps each fixture to the
- * scenarios averaged on it. `cells` are the shown run's per-scenario files it
- * was averaged from. No count files -- a package or process count is a
- * fact about one scenario, and a mean of them describes no install.
+ * scenarios that may be averaged on it (the curated list, whether or not the
+ * shown run has them), and `averaged` to the ones that are:
+ * those every drawn run has (`commonPlan`), so the trend and the shown run
+ * cover the same scenarios. `cells` are the shown run's per-scenario files.
+ * No count files -- a package or process count is a fact about one scenario,
+ * and a mean of them describes no install.
*/
export const loadAverage = async (
client: DataClient,
@@ -402,7 +405,9 @@ export const loadAverage = async (
/** From `loadSummaries`, possibly still loading, as for `loadScenario`. */
readonly summaries?: Summaries | Promise;
} = {},
-): Promise => {
+): Promise<
+ ScenarioData & { readonly cells: readonly RunCell[]; readonly averaged: CommonPlan }
+> => {
// The run shown is usually the trend's last date; fetched once either way,
// and from its files, which the report reads in full, without waiting for
// the summaries.
@@ -415,12 +420,26 @@ export const loadAverage = async (
const loaded = await Promise.all(
dates.map((d) => (d === date ? shown : loadCells(client, d, plan, summaryOf(d)))),
);
- const trend = drawn.flatMap((d, i) => trendPoint(d, averageFiles(loaded[i] ?? [], plan)) ?? []);
- const cells = loaded[dates.indexOf(date)] ?? [];
+ /* The runs the trend draws, and the shown one: an excluded run does not narrow the set. */
+ const whole = new Set((options.exclude ?? []).filter((e) => !e.includes("@")));
+ const averaged = commonPlan(
+ plan,
+ dates.flatMap((d, i) =>
+ d !== date && whole.has(d) ? [] : [{ date: d, cells: loaded[i] ?? [] }],
+ ),
+ date,
+ );
+ const trend = drawn.flatMap(
+ (d, i) => trendPoint(d, averageFiles(loaded[i] ?? [], averaged.plan)) ?? [],
+ );
+ const cells = (loaded[dates.indexOf(date)] ?? []).filter((c) =>
+ averaged.plan[c.fixture]?.includes(c.variation),
+ );
return {
cells,
- bundles: averageFiles(cells, plan).map((b) => ({
+ averaged,
+ bundles: averageFiles(cells, averaged.plan).map((b) => ({
...b,
counts: null,
processes: null,
diff --git a/apps/www/src/lib/markdown.ts b/apps/www/src/lib/markdown.ts
index 96edbebd44..c028388e54 100644
--- a/apps/www/src/lib/markdown.ts
+++ b/apps/www/src/lib/markdown.ts
@@ -438,16 +438,60 @@ export const figureMarkdown = (report: ReportModel, origin: string): FigureMarkd
export const PER_PACKAGE_DERIVED =
"Derived: install time divided by the number of packages the tool installed. A tool with a time but no published package count is left out rather than divided by a guess.";
-/** What a derived average is the mean of: its scenarios, per fixture where they differ. */
-export const meanOf = ({
- scenarios,
- byFixture,
-}: NonNullable): string =>
- Object.values(byFixture).every((v) => v.length === scenarios.length)
+/** What a derived average is the mean of: its scenarios, per fixture where they differ, and what it leaves out. */
+export const meanOf = (averageOf: NonNullable): string => {
+ const { scenarios, byFixture } = averageOf;
+ const mean = Object.values(byFixture).every((v) => v.length === scenarios.length)
? `Mean of: ${scenarios.join(", ")}.`
: `Mean of, per fixture: ${Object.entries(byFixture)
.map(([f, v]) => `${fixtureLabel(f)}: ${v.join(", ")}`)
.join("; ")}.`;
+ const note = averagedNote(averageOf);
+ return note === null ? mean : `${mean} ${note}`;
+};
+
+/**
+ * Which curated scenarios the average leaves out because some run in the trend
+ * window lacks them, and why; `null` when it leaves none out. Names between
+ * backticks, for the page's `Marked` and for Markdown alike.
+ */
+export const averagedNote = (
+ { byFixture, leftOut, fallback }: NonNullable,
+ date: (d: string) => string = (d) => d,
+): string | null => {
+ const fixtures = Object.keys(byFixture);
+ /* Where it applies, when that is not every fixture shown. */
+ const on = (some: readonly string[]) =>
+ some.length === fixtures.length ? [] : [`on ${some.map(fixtureLabel).join(", ")}`];
+ /* Scenarios with the same reason share it: the add/remove four all start on the same run. */
+ const reasons = new Map();
+ for (const l of leftOut) {
+ /* Said of the runs considered only: its real first run may be excluded or outside the window. */
+ const why =
+ l.when === "before"
+ ? `not in runs through ${date(l.missing.at(-1) ?? "")}`
+ : l.when === "after"
+ ? `not in runs since ${date(l.missing[0] ?? "")}`
+ : `not in the runs of ${l.missing.map(date).join(", ")}`;
+ const key = [...on(l.fixtures), why].join(", ");
+ reasons.set(key, [...(reasons.get(key) ?? []), `\`${l.scenario}\``]);
+ }
+ const notes = [
+ ...(reasons.size === 0
+ ? []
+ : [
+ `Left out, so every run in the trend window averages the same scenarios: ${[...reasons]
+ .map(([why, names]) => `${names.join(", ")} (${why})`)
+ .join("; ")}.`,
+ ]),
+ ...(fallback.length === 0
+ ? []
+ : [
+ `${["Too few scenarios ran on every run in the window", ...on(fallback)].join(" ")} to average, so this run's are averaged there, and ${fallback.length === 1 ? "that fixture is" : "those fixtures are"} left out of runs without one of them.`,
+ ]),
+ ];
+ return notes.length === 0 ? null : notes.join(" ");
+};
/** The run and the filters a report is for: what the page's menus, picker and chips show. */
export const runMarkdown = (report: ReportModel): string[] => {
@@ -625,6 +669,7 @@ export const historyMarkdown = (history: History, origin: string): string => {
`- Window: ${history.window.from} to ${history.window.to}${history.window.custom ? "" : " (a preset window ending at the run)"}`,
`- Fixture: ${history.fixture ?? `all (${history.fixtures.join(", ")})`}`,
`- Values: ${history.unit}${history.derived ? " (derived from total time)" : ""}`,
+ ...(history.averageOf === null ? [] : [`- Averaged: ${meanOf(history.averageOf)}`]),
`- Tools: ${history.tools.map((t) => (label(t) === t ? t : `${label(t)} (${t})`)).join(", ")}`,
`- Excluded runs: ${history.excluded.length === 0 ? "none" : history.excluded.join(", ")}`,
"",
diff --git a/apps/www/src/lib/measurements.ts b/apps/www/src/lib/measurements.ts
index 937233eb90..9545d076a8 100644
--- a/apps/www/src/lib/measurements.ts
+++ b/apps/www/src/lib/measurements.ts
@@ -207,6 +207,99 @@ export const averageFiles = (
return [{ fixture, file: { results } }];
});
+/** A scenario the window's average leaves out, and why. */
+export interface LeftOut {
+ readonly scenario: string;
+ /** The fixtures it is left out on. */
+ readonly fixtures: readonly string[];
+ /** The runs considered that lack it, ascending. */
+ readonly missing: readonly string[];
+ /**
+ * Where `missing` sits among the runs considered: all before every run that
+ * has it, all after, or neither. Only those runs: the scenario's real first
+ * run may be excluded, or outside the window.
+ */
+ readonly when: "before" | "after" | "between";
+}
+
+/** The scenarios a window's average covers on each fixture. */
+export interface CommonPlan {
+ readonly plan: { readonly [fixture: string]: readonly string[] };
+ /** In `plan`'s scenario order. */
+ readonly leftOut: readonly LeftOut[];
+ /**
+ * Fixtures where too few scenarios ran on every run to average, so they keep
+ * the shown run's, and are left out of the runs without one of them.
+ */
+ readonly fallback: readonly string[];
+}
+
+/**
+ * The scenarios of `plan` every run considered published, per fixture.
+ *
+ * `averageFiles` drops a fixture missing any planned scenario, so a plan taken
+ * from the shown run drew no point on any run before a scenario was added.
+ * Averaging only what each fixture has on every run it ran keeps the trend and
+ * the shown run's numbers over the same scenarios. `plan` is every candidate
+ * (the curated list); one no run has is not offered, and is not named. The
+ * shown run is one of `runs` like any other. A fixture left with fewer than
+ * two keeps the scenarios the `shown` run has, and is named in `fallback`.
+ */
+export const commonPlan = (
+ plan: { readonly [fixture: string]: readonly string[] },
+ runs: readonly {
+ readonly date: string;
+ readonly cells: readonly { readonly fixture: string; readonly variation: string }[];
+ }[],
+ shown?: string,
+): CommonPlan => {
+ const has = (cells: (typeof runs)[number]["cells"], fixture: string, variation: string) =>
+ cells.some((c) => c.fixture === fixture && c.variation === variation);
+ const common: { [fixture: string]: readonly string[] } = {};
+ const fallback: string[] = [];
+ const dropped = new Map<
+ string,
+ { fixtures: string[]; missing: Set; present: string[] }
+ >();
+ for (const [fixture, candidates] of Object.entries(plan)) {
+ /* A run the fixture did not run on at all says nothing about its scenarios. */
+ const ran = runs.filter((r) => candidates.some((v) => has(r.cells, fixture, v)));
+ const offered = candidates.filter((v) => ran.some((r) => has(r.cells, fixture, v)));
+ const kept = offered.filter((v) => ran.every((r) => has(r.cells, fixture, v)));
+ if (kept.length < Math.min(2, offered.length)) {
+ const own = runs.find((r) => r.date === shown)?.cells;
+ common[fixture] = own === undefined ? offered : offered.filter((v) => has(own, fixture, v));
+ fallback.push(fixture);
+ continue;
+ }
+ common[fixture] = kept;
+ for (const v of offered.filter((v) => !kept.includes(v))) {
+ const entry = dropped.get(v) ?? { fixtures: [], missing: new Set(), present: [] };
+ entry.fixtures.push(fixture);
+ for (const r of ran) {
+ if (has(r.cells, fixture, v)) entry.present.push(r.date);
+ else entry.missing.add(r.date);
+ }
+ dropped.set(v, entry);
+ }
+ }
+ const leftOut = [...new Set(Object.values(plan).flat())].flatMap((scenario) => {
+ const entry = dropped.get(scenario);
+ if (entry === undefined) return [];
+ const missing = [...entry.missing].sort();
+ const present = entry.present.sort();
+ const first = present[0] ?? "";
+ const last = present.at(-1) ?? "";
+ const when: LeftOut["when"] = missing.every((d) => d < first)
+ ? "before"
+ : missing.every((d) => d > last)
+ ? "after"
+ : "between";
+ return [{ scenario, fixtures: entry.fixtures, missing, when }];
+ });
+ return { plan: common, leftOut, fallback };
+};
+
const failedIn = (file: BenchmarkFile, tools: readonly string[]): string[] =>
file.results.filter((r) => tools.includes(r.command) && !usable(r)).map((r) => r.command);
diff --git a/apps/www/src/lib/report.ts b/apps/www/src/lib/report.ts
index a8792c84e5..e4b6ce8bfa 100644
--- a/apps/www/src/lib/report.ts
+++ b/apps/www/src/lib/report.ts
@@ -23,6 +23,7 @@ import {
countRows,
perPackageByFixture,
ratioTo,
+ type CommonPlan,
type CountRow,
type FixtureRow,
type ToolMean,
@@ -96,10 +97,17 @@ export interface ReportModel {
readonly state: ScenarioState;
/** What was on disk before each run; `null` where the name encodes no state. */
readonly stateParts: ReturnType | null;
- /** On the derived view, the scenarios averaged, in all and on each fixture shown; `null` on a measured one. */
+ /**
+ * On the derived view, the scenarios averaged, in all and on each fixture
+ * shown: those every run in the trend window has, so they depend on the
+ * window. `leftOut` and `fallback` say what that cost (`commonPlan`).
+ * `null` on a measured one.
+ */
readonly averageOf: {
readonly scenarios: readonly string[];
readonly byFixture: { readonly [fixture: string]: readonly string[] };
+ readonly leftOut: CommonPlan["leftOut"];
+ readonly fallback: CommonPlan["fallback"];
} | null;
};
readonly run: {
@@ -308,14 +316,16 @@ export const loadReport = async ({
}
const averageOf = selected.averageOf ?? null;
- /* The derived view: each fixture, and the scenarios averaged on it. */
+ /*
+ * The derived view: each fixture the shown run averages anything on, and
+ * every curated scenario as a candidate, published by this run or not.
+ * `loadAverage` keeps those every run in the window has, so a scenario this
+ * run lacks and others have is named as left out, not silently missing.
+ */
const plan: { [fixture: string]: readonly string[] } = Object.fromEntries(
Object.entries(entry.benchmarks)
- .map(([f, published]): [string, string[]] => [
- f,
- (averageOf ?? []).filter((v) => published?.includes(v)),
- ])
- .filter(([, averaged]) => averaged.length > 0)
+ .filter(([, published]) => (averageOf ?? []).some((v) => published?.includes(v)))
+ .map(([f]): [string, readonly string[]] => [f, def.order])
.sort(([a], [b]) => a.localeCompare(b)),
);
const publishedFixtures =
@@ -583,11 +593,15 @@ export const loadReport = async ({
state: selected.state,
stateParts: selected.kind === "other" ? null : stateParts(selected.state),
averageOf:
- averageOf === null
+ averageOf === null || average === null
? null
: {
- scenarios: averageOf.filter((v) => Object.values(shownPlan).flat().includes(v)),
- byFixture: shownPlan,
+ scenarios: def.order.filter((v) =>
+ Object.values(average.averaged.plan).flat().includes(v),
+ ),
+ byFixture: average.averaged.plan,
+ leftOut: average.averaged.leftOut,
+ fallback: average.averaged.fallback,
},
},
run: { date, latest: index.latest, isLatest: date === index.latest, dates: index.dates },
diff --git a/apps/www/test/average.test.ts b/apps/www/test/average.test.ts
index b205c000ec..2f1efe8ac8 100644
--- a/apps/www/test/average.test.ts
+++ b/apps/www/test/average.test.ts
@@ -6,12 +6,13 @@ import { GET as api } from "../src/app/api/v1/[section]/[scenario]/route.ts";
import reportTwin, { slots } from "../src/app/[section]/[[...params]]/markdown.ts";
import { GET as llms } from "../src/app/llms.txt/route.ts";
import { TREND_BUDGET, trendDates } from "../src/lib/load.ts";
-import { averageFiles } from "../src/lib/measurements.ts";
+import { averagedNote, reportMarkdown } from "../src/lib/markdown.ts";
+import { averageFiles, commonPlan } from "../src/lib/measurements.ts";
import { treeFor } from "../src/lib/page-tree.ts";
import { loadReportParams } from "../src/lib/params.ts";
import { loadReport } from "../src/lib/report.ts";
import { AVERAGE, scenariosFor, withAverage } from "../src/lib/scenarios.ts";
-import { DATA_URL, fakeFetch } from "./fixture-tree.ts";
+import { DATA_URL, DATES, fakeFetch } from "./fixture-tree.ts";
const ORIGIN = "https://benchmarks.vlt.io";
@@ -97,6 +98,125 @@ describe("averageFiles", () => {
});
});
+describe("commonPlan", () => {
+ const plan = {
+ astro: ["clean", "cache", "add"],
+ babylon: ["clean", "cache", "add"],
+ };
+ const run = (date: string, cells: { [fixture: string]: readonly string[] }) => ({
+ date,
+ cells: Object.entries(cells).flatMap(([fixture, vs]) =>
+ vs.map((variation) => ({ fixture, variation })),
+ ),
+ });
+ const all = { astro: plan.astro, babylon: plan.babylon };
+ const before = { astro: ["clean", "cache"], babylon: ["clean", "cache"] };
+
+ it("leaves out a scenario added mid-window, and says when it starts", () => {
+ const common = commonPlan(plan, [run("d1", before), run("d2", before), run("d3", all)]);
+ expect(common.plan).toEqual(before);
+ expect(common.leftOut).toEqual([
+ { scenario: "add", fixtures: ["astro", "babylon"], missing: ["d1", "d2"], when: "before" },
+ ]);
+ expect(common.fallback).toEqual([]);
+ });
+
+ it("averages everything in a window that starts after it was added", () => {
+ expect(commonPlan(plan, [run("d3", all), run("d4", all)])).toEqual({
+ plan: all,
+ leftOut: [],
+ fallback: [],
+ });
+ });
+
+ it("leaves out a scenario missing on one run, on that fixture only, and names the run", () => {
+ const gap = { astro: ["clean", "add"], babylon: plan.babylon };
+ const common = commonPlan(plan, [run("d1", all), run("d2", gap), run("d3", all)]);
+ expect(common.plan).toEqual({ astro: ["clean", "add"], babylon: plan.babylon });
+ expect(common.leftOut).toEqual([
+ { scenario: "cache", fixtures: ["astro"], missing: ["d2"], when: "between" },
+ ]);
+ });
+
+ it("ignores a run a fixture did not run on at all", () => {
+ const common = commonPlan(plan, [run("d1", { babylon: plan.babylon }), run("d2", all)]);
+ expect(common.plan).toEqual(all);
+ });
+
+ it("falls back to the shown run's set when fewer than two are common", () => {
+ const common = commonPlan(
+ plan,
+ [run("d1", { astro: ["clean"], babylon: ["clean"] }), run("d2", before)],
+ "d2",
+ );
+ expect(common.plan).toEqual(before);
+ expect(common.fallback).toEqual(["astro", "babylon"]);
+ expect(common.leftOut).toEqual([]);
+ });
+
+ it("does not offer a scenario no run has", () => {
+ expect(commonPlan(plan, [run("d1", before), run("d2", before)])).toEqual({
+ plan: before,
+ leftOut: [],
+ fallback: [],
+ });
+ });
+
+ it("names a scenario the shown run lacks and an earlier run has", () => {
+ const shown = { astro: ["clean", "cache"], babylon: plan.babylon };
+ const common = commonPlan(plan, [run("d1", all), run("d2", shown)], "d2");
+ expect(common.plan).toEqual({ astro: ["clean", "cache"], babylon: plan.babylon });
+ expect(common.leftOut).toEqual([
+ { scenario: "add", fixtures: ["astro"], missing: ["d2"], when: "after" },
+ ]);
+ });
+
+ it("describes only the runs it is given, not an excluded first run", () => {
+ // d2 is the scenario's real first run, excluded by the caller: d3 is not its first.
+ const common = commonPlan(plan, [run("d1", before), run("d3", all), run("d4", all)]);
+ expect(common.leftOut).toEqual([
+ { scenario: "add", fixtures: ["astro", "babylon"], missing: ["d1"], when: "before" },
+ ]);
+ });
+
+ it("covers only the fixtures it is given", () => {
+ const common = commonPlan({ astro: plan.astro }, [run("d1", before), run("d2", all)]);
+ expect(common.plan).toEqual({ astro: ["clean", "cache"] });
+ expect(common.leftOut).toEqual([
+ { scenario: "add", fixtures: ["astro"], missing: ["d1"], when: "before" },
+ ]);
+ });
+});
+
+describe("averagedNote", () => {
+ const averageOf = {
+ scenarios: ["clean"],
+ byFixture: { astro: ["clean"], babylon: ["clean", "cache"] },
+ leftOut: [
+ { scenario: "add", fixtures: ["astro", "babylon"], missing: ["d1"], when: "before" },
+ { scenario: "remove", fixtures: ["astro", "babylon"], missing: ["d3"], when: "after" },
+ { scenario: "cache", fixtures: ["astro"], missing: ["d2", "d4"], when: "between" },
+ ],
+ fallback: [],
+ } satisfies Parameters[0];
+
+ it("names what is left out, grouped by why", () => {
+ expect(averagedNote(averageOf)).toBe(
+ "Left out, so every run in the trend window averages the same scenarios: `add` (not in runs through d1); `remove` (not in runs since d3); `cache` (on Astro, not in the runs of d2, d4).",
+ );
+ });
+
+ it("says nothing when nothing is left out", () => {
+ expect(averagedNote({ ...averageOf, leftOut: [] })).toBeNull();
+ });
+
+ it("says when it fell back", () => {
+ expect(averagedNote({ ...averageOf, leftOut: [], fallback: ["astro", "babylon"] })).toBe(
+ "Too few scenarios ran on every run in the window to average, so this run's are averaged there, and those fixtures are left out of runs without one of them.",
+ );
+ });
+});
+
describe("withAverage", () => {
it("leads with the derived view, over the curated scenarios only", () => {
const list = withAverage(scenariosFor("package-managers", ["cache", "clean", "ci+setup"]));
@@ -160,11 +280,83 @@ describe("the average report", () => {
expect(report.scenario.averageOf).toEqual({
scenarios: ["clean", "cache"],
byFixture: { astro: ["clean", "cache"], babylon: ["clean", "cache"] },
+ leftOut: [],
+ fallback: [],
});
expect(report.finding.sentence).toBe("bun is fastest at 1.50s, 10.0× faster than npm.");
expect(report.timeRows.find((r) => r.fixture === "babylon")?.failed).toEqual(["zpm"]);
});
+ it("averages only what every run in the window has, and says what it left out", async () => {
+ /* `add-singleton` published, as `clean`'s files again, on the runs given only. */
+ const addedOn = (runs: readonly string[]) => async (url: string | URL | Request) => {
+ const path = decodeURIComponent(String(url).slice(DATA_URL.length + 1));
+ const add = /^([\d-]+)\/(.+)-add-singleton\.json$/.exec(path);
+ if (add !== null) {
+ return runs.includes(add[1] ?? "")
+ ? fakeFetch(`${DATA_URL}/${add[1]}/${add[2]}-clean.json`)
+ : { ok: false, status: 404, json: async () => undefined };
+ }
+ const res = await withCache(url);
+ const run = /^manifest\/(.+)\.json$/.exec(path)?.[1];
+ if (run === undefined || !runs.includes(run) || !res.ok) return res;
+ const entry = (await res.json()) as { benchmarks: { [f: string]: string[] } };
+ const benchmarks = Object.fromEntries(
+ Object.entries(entry.benchmarks).map(([f, v]) => [f, [...v, "add-singleton"]]),
+ );
+ return { ...res, json: async () => ({ ...entry, benchmarks }) };
+ };
+ const at = async (search: string, runs: readonly string[] = ["2026-10-13"]) => {
+ const r = await loadReport({
+ section: "package-managers",
+ scenario: AVERAGE,
+ query: loadReportParams(new URLSearchParams(search)),
+ client: new DataClient({ baseUrl: DATA_URL, fetch: addedOn(runs) }),
+ });
+ if (r.status !== "ok") throw new Error(r.status);
+ return r.report;
+ };
+
+ const window = await at("from=2026-10-10&to=2026-10-13");
+ expect(window.scenario.averageOf?.scenarios).toEqual(["clean", "cache"]);
+ expect(window.scenario.averageOf?.leftOut).toEqual([
+ {
+ scenario: "add-singleton",
+ fixtures: ["astro", "babylon"],
+ missing: ["2026-10-10", "2026-10-12"],
+ when: "before",
+ },
+ ]);
+ expect(window.trend.points.map((p) => p.date)).toEqual(DATES);
+ expect(window.raw.map((r) => r.label)).not.toContain("Astro · add-singleton");
+ expect(reportMarkdown(window, ORIGIN)).toContain(
+ "Left out, so every run in the trend window averages the same scenarios: `add-singleton` (not in runs through 2026-10-12).",
+ );
+
+ /* Shown at a run without it, after runs that have it: still named, not silently missing. */
+ const dropped = await at("from=2026-10-10&to=2026-10-13", ["2026-10-10", "2026-10-12"]);
+ expect(dropped.scenario.averageOf?.scenarios).toEqual(["clean", "cache"]);
+ expect(dropped.scenario.averageOf?.leftOut).toEqual([
+ {
+ scenario: "add-singleton",
+ fixtures: ["astro", "babylon"],
+ missing: ["2026-10-13"],
+ when: "after",
+ },
+ ]);
+ expect(reportMarkdown(dropped, ORIGIN)).toContain(
+ "`add-singleton` (not in runs since 2026-10-13).",
+ );
+
+ const latest = await at("from=2026-10-13&to=2026-10-13");
+ expect(latest.scenario.averageOf?.scenarios).toEqual(["clean", "cache", "add-singleton"]);
+ expect(latest.scenario.averageOf?.leftOut).toEqual([]);
+
+ const astro = await at("from=2026-10-10&to=2026-10-13&fixture=astro");
+ expect(astro.scenario.averageOf?.byFixture).toEqual({ astro: ["clean", "cache"] });
+ expect(astro.scenario.averageOf?.leftOut[0]?.fixtures).toEqual(["astro"]);
+ });
+
it("has no counts, no per-package metric, and cites every file it averaged", async () => {
const report = await load("metric=per-package");
expect(report.outline.map((s) => s.id)).toEqual(["leaderboard", "time"]);