Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion apps/www/src/app/[section]/[[...params]]/page.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ import { toTrendColumns } from "@/lib/chart-model";
import { loadReportParams, rangeLabel, reportHref, reportParams } from "@/lib/params";
import { loadCatalog } from "@/lib/catalog";
import { dataClient } from "@/lib/data";
import { PER_PACKAGE_DERIVED } from "@/lib/markdown";
import { averagedNote, PER_PACKAGE_DERIVED } from "@/lib/markdown";
import { loadReport } from "@/lib/report";
import { withAverage } from "@/lib/scenarios";
import { isSectionId, SECTION_DEFS, SECTIONS } from "@/lib/sections";
Expand Down Expand Up @@ -188,6 +188,7 @@ const Report = async ({

const report = result.report;
const { scenario, run, filters, finding, baseline, caveats, trend } = report;
const note = scenario.averageOf === null ? null : averagedNote(scenario.averageOf, longDate);
const { date } = run;
const { fixture, hide: hidden } = filters;
const { measure, isScript, outline, ranked, publishedFixtures, path } = report;
Expand Down Expand Up @@ -641,6 +642,12 @@ const Report = async ({
? " Every registry is installed from with the same npm client."
: ""}
{run.isLatest ? null : " This is not the latest run."}
{note === null ? null : (
<>
{" "}
<Marked text={note} />
</>
)}
</>
}
action={
Expand Down
25 changes: 25 additions & 0 deletions apps/www/src/app/openapi.json/route.ts
Original file line number Diff line number Diff line change
Expand Up @@ -284,6 +284,31 @@ export const GET = (request: Request) =>
properties: {
section: { type: "string" },
scenario: { type: "string" },
averageOf: {
type: ["object", "null"],
description:
"On the derived average: the scenarios averaged (those every run in the window has), in all and per fixture, the ones left out and why, and the fixtures that fell back to the shown run's set. Null on a measured scenario.",
properties: {
scenarios: { type: "array", items: { type: "string" } },
byFixture: {
type: "object",
additionalProperties: { type: "array", items: { type: "string" } },
},
leftOut: {
type: "array",
items: {
type: "object",
properties: {
scenario: { type: "string" },
fixtures: { type: "array", items: { type: "string" } },
missing: { type: "array", items: date },
when: { type: "string", enum: ["before", "after", "between"] },
},
},
},
fallback: { type: "array", items: { type: "string" } },
},
},
run: date,
fixture: { type: ["string", "null"] },
fixtures: { type: "array", items: { type: "string" } },
Expand Down
2 changes: 2 additions & 0 deletions apps/www/src/lib/api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,8 @@ export const historyOf = (report: ReportModel, tools?: readonly string[]) => {
return {
section: report.section.id,
scenario: report.scenario.variation,
/* On the derived view, what the trend averages: the scenarios every run in the window has. */
averageOf: report.scenario.averageOf,
run: report.run.date,
fixture: report.filters.fixture,
fixtures: report.fixtures,
Expand Down
35 changes: 27 additions & 8 deletions apps/www/src/lib/load.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import {
} from "@benchmarks/data";
import { probe } from "./data";
import { applyExclusions } from "./figures";
import { averageFiles, usable } from "./measurements";
import { averageFiles, commonPlan, usable, type CommonPlan } from "./measurements";

export interface FixtureBundle {
readonly fixture: string;
Expand Down Expand Up @@ -388,9 +388,12 @@ const loadCells = async (

/**
* The derived "average" view, as a scenario: `plan` maps each fixture to the
* scenarios averaged on it. `cells` are the shown run's per-scenario files it
* was averaged from. No count files -- a package or process count is a
* fact about one scenario, and a mean of them describes no install.
* scenarios that may be averaged on it (the curated list, whether or not the
* shown run has them), and `averaged` to the ones that are:
* those every drawn run has (`commonPlan`), so the trend and the shown run
* cover the same scenarios. `cells` are the shown run's per-scenario files.
* No count files -- a package or process count is a fact about one scenario,
* and a mean of them describes no install.
*/
export const loadAverage = async (
client: DataClient,
Expand All @@ -402,7 +405,9 @@ export const loadAverage = async (
/** From `loadSummaries`, possibly still loading, as for `loadScenario`. */
readonly summaries?: Summaries | Promise<Summaries>;
} = {},
): Promise<ScenarioData & { readonly cells: readonly RunCell[] }> => {
): Promise<
ScenarioData & { readonly cells: readonly RunCell[]; readonly averaged: CommonPlan }
> => {
// The run shown is usually the trend's last date; fetched once either way,
// and from its files, which the report reads in full, without waiting for
// the summaries.
Expand All @@ -415,12 +420,26 @@ export const loadAverage = async (
const loaded = await Promise.all(
dates.map((d) => (d === date ? shown : loadCells(client, d, plan, summaryOf(d)))),
);
const trend = drawn.flatMap((d, i) => trendPoint(d, averageFiles(loaded[i] ?? [], plan)) ?? []);
const cells = loaded[dates.indexOf(date)] ?? [];
/* The runs the trend draws, and the shown one: an excluded run does not narrow the set. */
const whole = new Set((options.exclude ?? []).filter((e) => !e.includes("@")));
const averaged = commonPlan(
plan,
dates.flatMap((d, i) =>
d !== date && whole.has(d) ? [] : [{ date: d, cells: loaded[i] ?? [] }],
),
date,
);
const trend = drawn.flatMap(
(d, i) => trendPoint(d, averageFiles(loaded[i] ?? [], averaged.plan)) ?? [],
);
const cells = (loaded[dates.indexOf(date)] ?? []).filter((c) =>
averaged.plan[c.fixture]?.includes(c.variation),
);

return {
cells,
bundles: averageFiles(cells, plan).map((b) => ({
averaged,
bundles: averageFiles(cells, averaged.plan).map((b) => ({
...b,
counts: null,
processes: null,
Expand Down
57 changes: 51 additions & 6 deletions apps/www/src/lib/markdown.ts
Original file line number Diff line number Diff line change
Expand Up @@ -438,16 +438,60 @@ export const figureMarkdown = (report: ReportModel, origin: string): FigureMarkd
export const PER_PACKAGE_DERIVED =
"Derived: install time divided by the number of packages the tool installed. A tool with a time but no published package count is left out rather than divided by a guess.";

/** What a derived average is the mean of: its scenarios, per fixture where they differ. */
export const meanOf = ({
scenarios,
byFixture,
}: NonNullable<ReportModel["scenario"]["averageOf"]>): string =>
Object.values(byFixture).every((v) => v.length === scenarios.length)
/** What a derived average is the mean of: its scenarios, per fixture where they differ, and what it leaves out. */
export const meanOf = (averageOf: NonNullable<ReportModel["scenario"]["averageOf"]>): string => {
const { scenarios, byFixture } = averageOf;
const mean = Object.values(byFixture).every((v) => v.length === scenarios.length)
? `Mean of: ${scenarios.join(", ")}.`
: `Mean of, per fixture: ${Object.entries(byFixture)
.map(([f, v]) => `${fixtureLabel(f)}: ${v.join(", ")}`)
.join("; ")}.`;
const note = averagedNote(averageOf);
return note === null ? mean : `${mean} ${note}`;
};

/**
* Which curated scenarios the average leaves out because some run in the trend
* window lacks them, and why; `null` when it leaves none out. Names between
* backticks, for the page's `Marked` and for Markdown alike.
*/
export const averagedNote = (
{ byFixture, leftOut, fallback }: NonNullable<ReportModel["scenario"]["averageOf"]>,
date: (d: string) => string = (d) => d,
): string | null => {
const fixtures = Object.keys(byFixture);
/* Where it applies, when that is not every fixture shown. */
const on = (some: readonly string[]) =>
some.length === fixtures.length ? [] : [`on ${some.map(fixtureLabel).join(", ")}`];
/* Scenarios with the same reason share it: the add/remove four all start on the same run. */
const reasons = new Map<string, string[]>();
for (const l of leftOut) {
/* Said of the runs considered only: its real first run may be excluded or outside the window. */
const why =
l.when === "before"
? `not in runs through ${date(l.missing.at(-1) ?? "")}`
: l.when === "after"
? `not in runs since ${date(l.missing[0] ?? "")}`
: `not in the runs of ${l.missing.map(date).join(", ")}`;
const key = [...on(l.fixtures), why].join(", ");
reasons.set(key, [...(reasons.get(key) ?? []), `\`${l.scenario}\``]);
}
const notes = [
...(reasons.size === 0
? []
: [
`Left out, so every run in the trend window averages the same scenarios: ${[...reasons]
.map(([why, names]) => `${names.join(", ")} (${why})`)
.join("; ")}.`,
]),
...(fallback.length === 0
? []
: [
`${["Too few scenarios ran on every run in the window", ...on(fallback)].join(" ")} to average, so this run's are averaged there, and ${fallback.length === 1 ? "that fixture is" : "those fixtures are"} left out of runs without one of them.`,
]),
];
return notes.length === 0 ? null : notes.join(" ");
};

/** The run and the filters a report is for: what the page's menus, picker and chips show. */
export const runMarkdown = (report: ReportModel): string[] => {
Expand Down Expand Up @@ -625,6 +669,7 @@ export const historyMarkdown = (history: History, origin: string): string => {
`- Window: ${history.window.from} to ${history.window.to}${history.window.custom ? "" : " (a preset window ending at the run)"}`,
`- Fixture: ${history.fixture ?? `all (${history.fixtures.join(", ")})`}`,
`- Values: ${history.unit}${history.derived ? " (derived from total time)" : ""}`,
...(history.averageOf === null ? [] : [`- Averaged: ${meanOf(history.averageOf)}`]),
`- Tools: ${history.tools.map((t) => (label(t) === t ? t : `${label(t)} (${t})`)).join(", ")}`,
`- Excluded runs: ${history.excluded.length === 0 ? "none" : history.excluded.join(", ")}`,
"",
Expand Down
93 changes: 93 additions & 0 deletions apps/www/src/lib/measurements.ts
Original file line number Diff line number Diff line change
Expand Up @@ -207,6 +207,99 @@ export const averageFiles = (
return [{ fixture, file: { results } }];
});

/** A scenario the window's average leaves out, and why. */
export interface LeftOut {
readonly scenario: string;
/** The fixtures it is left out on. */
readonly fixtures: readonly string[];
/** The runs considered that lack it, ascending. */
readonly missing: readonly string[];
/**
* Where `missing` sits among the runs considered: all before every run that
* has it, all after, or neither. Only those runs: the scenario's real first
* run may be excluded, or outside the window.
*/
readonly when: "before" | "after" | "between";
}

/** The scenarios a window's average covers on each fixture. */
export interface CommonPlan {
readonly plan: { readonly [fixture: string]: readonly string[] };
/** In `plan`'s scenario order. */
readonly leftOut: readonly LeftOut[];
/**
* Fixtures where too few scenarios ran on every run to average, so they keep
* the shown run's, and are left out of the runs without one of them.
*/
readonly fallback: readonly string[];
}

/**
* The scenarios of `plan` every run considered published, per fixture.
*
* `averageFiles` drops a fixture missing any planned scenario, so a plan taken
* from the shown run drew no point on any run before a scenario was added.
* Averaging only what each fixture has on every run it ran keeps the trend and
* the shown run's numbers over the same scenarios. `plan` is every candidate
* (the curated list); one no run has is not offered, and is not named. The
* shown run is one of `runs` like any other. A fixture left with fewer than
* two keeps the scenarios the `shown` run has, and is named in `fallback`.
*/
export const commonPlan = (
plan: { readonly [fixture: string]: readonly string[] },
runs: readonly {
readonly date: string;
readonly cells: readonly { readonly fixture: string; readonly variation: string }[];
}[],
shown?: string,
): CommonPlan => {
const has = (cells: (typeof runs)[number]["cells"], fixture: string, variation: string) =>
cells.some((c) => c.fixture === fixture && c.variation === variation);
const common: { [fixture: string]: readonly string[] } = {};
const fallback: string[] = [];
const dropped = new Map<
string,
{ fixtures: string[]; missing: Set<string>; present: string[] }
>();
for (const [fixture, candidates] of Object.entries(plan)) {
/* A run the fixture did not run on at all says nothing about its scenarios. */
const ran = runs.filter((r) => candidates.some((v) => has(r.cells, fixture, v)));
const offered = candidates.filter((v) => ran.some((r) => has(r.cells, fixture, v)));
const kept = offered.filter((v) => ran.every((r) => has(r.cells, fixture, v)));
if (kept.length < Math.min(2, offered.length)) {
const own = runs.find((r) => r.date === shown)?.cells;
common[fixture] = own === undefined ? offered : offered.filter((v) => has(own, fixture, v));
fallback.push(fixture);
continue;
}
common[fixture] = kept;
for (const v of offered.filter((v) => !kept.includes(v))) {
const entry = dropped.get(v) ?? { fixtures: [], missing: new Set(), present: [] };
entry.fixtures.push(fixture);
for (const r of ran) {
if (has(r.cells, fixture, v)) entry.present.push(r.date);
else entry.missing.add(r.date);
}
dropped.set(v, entry);
}
}
const leftOut = [...new Set(Object.values(plan).flat())].flatMap((scenario) => {
const entry = dropped.get(scenario);
if (entry === undefined) return [];
const missing = [...entry.missing].sort();
const present = entry.present.sort();
const first = present[0] ?? "";
const last = present.at(-1) ?? "";
const when: LeftOut["when"] = missing.every((d) => d < first)
? "before"
: missing.every((d) => d > last)
? "after"
: "between";
return [{ scenario, fixtures: entry.fixtures, missing, when }];
});
return { plan: common, leftOut, fallback };
};

const failedIn = (file: BenchmarkFile, tools: readonly string[]): string[] =>
file.results.filter((r) => tools.includes(r.command) && !usable(r)).map((r) => r.command);

Expand Down
34 changes: 24 additions & 10 deletions apps/www/src/lib/report.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import {
countRows,
perPackageByFixture,
ratioTo,
type CommonPlan,
type CountRow,
type FixtureRow,
type ToolMean,
Expand Down Expand Up @@ -96,10 +97,17 @@ export interface ReportModel {
readonly state: ScenarioState;
/** What was on disk before each run; `null` where the name encodes no state. */
readonly stateParts: ReturnType<typeof stateParts> | null;
/** On the derived view, the scenarios averaged, in all and on each fixture shown; `null` on a measured one. */
/**
* On the derived view, the scenarios averaged, in all and on each fixture
* shown: those every run in the trend window has, so they depend on the
* window. `leftOut` and `fallback` say what that cost (`commonPlan`).
* `null` on a measured one.
*/
readonly averageOf: {
readonly scenarios: readonly string[];
readonly byFixture: { readonly [fixture: string]: readonly string[] };
readonly leftOut: CommonPlan["leftOut"];
readonly fallback: CommonPlan["fallback"];
} | null;
};
readonly run: {
Expand Down Expand Up @@ -308,14 +316,16 @@ export const loadReport = async ({
}

const averageOf = selected.averageOf ?? null;
/* The derived view: each fixture, and the scenarios averaged on it. */
/*
* The derived view: each fixture the shown run averages anything on, and
* every curated scenario as a candidate, published by this run or not.
* `loadAverage` keeps those every run in the window has, so a scenario this
* run lacks and others have is named as left out, not silently missing.
*/
const plan: { [fixture: string]: readonly string[] } = Object.fromEntries(
Object.entries(entry.benchmarks)
.map(([f, published]): [string, string[]] => [
f,
(averageOf ?? []).filter((v) => published?.includes(v)),
])
.filter(([, averaged]) => averaged.length > 0)
.filter(([, published]) => (averageOf ?? []).some((v) => published?.includes(v)))
.map(([f]): [string, readonly string[]] => [f, def.order])
.sort(([a], [b]) => a.localeCompare(b)),
);
const publishedFixtures =
Expand Down Expand Up @@ -583,11 +593,15 @@ export const loadReport = async ({
state: selected.state,
stateParts: selected.kind === "other" ? null : stateParts(selected.state),
averageOf:
averageOf === null
averageOf === null || average === null
? null
: {
scenarios: averageOf.filter((v) => Object.values(shownPlan).flat().includes(v)),
byFixture: shownPlan,
scenarios: def.order.filter((v) =>
Object.values(average.averaged.plan).flat().includes(v),
),
byFixture: average.averaged.plan,
leftOut: average.averaged.leftOut,
fallback: average.averaged.fallback,
},
},
run: { date, latest: index.latest, isLatest: date === index.latest, dates: index.dates },
Expand Down
Loading