Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion apps/www/src/app/about/components.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -281,7 +281,9 @@ export const LatestResults = async () => {
const scenario = scenarios[0];
if (scenario === undefined) return { section, scenario: null };
const fixtures = fixturesWith(entry.benchmarks, scenario.variation);
const data = await loadScenario(client, date, scenario.variation, fixtures, []);
const data = await loadScenario(client, date, scenario.variation, fixtures, [], {
entry,
});
return {
section,
scenario,
Expand Down
40 changes: 34 additions & 6 deletions apps/www/src/lib/data.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,11 @@ export const dataBaseUrl = (): string => {
* fans out to hundreds of files, on every build worker at once. So at most
* MAX_IN_FLIGHT requests run per process, and a 429 waits and tries again.
*
* It also answers some requests with a 5xx that a retry does not see again --
* seen on 2026-11-18, where a path that does not exist was a 503 one minute
* and a 404 the next. A 5xx is tried SERVER_RETRIES more times with a short
* backoff before it is believed.
*
* The queue is inside "use cache", in the fill: a prerender's second pass
* must find every entry already filled, so nothing may wait before the cached
* call. `next dev` skips it, since it reports a fill idle for 10s as a
Expand All @@ -52,6 +57,7 @@ export const dataBaseUrl = (): string => {
*/
const MAX_IN_FLIGHT = 12;
const RETRIES = 5;
const SERVER_RETRIES = 2;
let inFlight = 0;
const waiting: (() => void)[] = [];

Expand Down Expand Up @@ -80,12 +86,16 @@ export const throttle = async <T>(task: () => Promise<T>): Promise<T> => {
};

export const fetchRetrying = async (url: string): Promise<Response> => {
for (let attempt = 0; ; attempt++) {
let limited = 0;
let broken = 0;
for (;;) {
const response = await fetch(url);
if (response.status !== 429 || attempt === RETRIES) return response;
if (response.status === 429 && limited < RETRIES) limited++;
else if (response.status >= 500 && broken < SERVER_RETRIES) broken++;
else return response;
await response.body?.cancel();
const after = Number(response.headers.get("retry-after"));
const ms = after > 0 ? after * 1000 : 250 * 2 ** attempt;
const ms = after > 0 ? after * 1000 : 250 * 2 ** (limited + broken - 1);
await new Promise((resolve) => setTimeout(resolve, ms));
}
};
Expand All @@ -99,12 +109,20 @@ const throttledFetch = (url: string): Promise<Response> =>
* kept for a day and refreshed behind the reader; the index, the full
* manifest and `latest/` move with each run, and a 404 may be a date not yet
* published, so those are kept a minute -- the shortest life that still lets
* a page prerender. A failure other than a 404 throws, and is not cached.
* a page prerender.
*
* A 5xx that outlasts `fetchRetrying` is returned like a 404, for the same
* minute, and `DataClient` turns it into a `DataFetchError` the caller sees.
* It is not thrown here: on 2026-11-18 every build failed on a 503 thrown from
* this function for a process-count file reached only through {@link probe},
* which catches it -- the prerender fails a page on an error thrown inside
* "use cache" whether or not the caller catches it. Any other failure throws,
* and is not cached.
*/
const cachedGet = async (url: string) => {
"use cache";
const response = await throttledFetch(url);
if (!response.ok && response.status !== 404) {
if (!response.ok && response.status !== 404 && response.status < 500) {
throw new Error(`Request for ${url} failed with status ${response.status}`);
}
if (response.ok && /\/\d{4}-\d{2}-\d{2}(\/|\.json$)/.test(url)) cacheLife("days");
Expand Down Expand Up @@ -133,6 +151,14 @@ export const dataClient = (): DataClient =>
* every `registry-*` variation and `versions.json` is genuinely absent from
* `latest/`, so a 404 there is the honest answer and must never be rendered as
* a zero or an em dash. A 500 is a different thing and says so.
*
* Everything probed is data a page renders without: a count file, a run's
* `versions.json`, a summary. So a 5xx that outlasts the retries is reported
* as `failed` -- which every reader shows as missing -- and warned about on
* the server, rather than failing the page. Data a page cannot render without
* (the index, a run's shard) is never probed, and its `DataFetchError` is
* thrown, so a real outage still fails the build instead of publishing an
* empty site.
*/
export type Probe<T> =
| { readonly state: "present"; readonly value: T }
Expand All @@ -146,6 +172,8 @@ export const probe = async <T>(fetcher: () => Promise<T>): Promise<Probe<T>> =>
if (error instanceof DataFetchError && error.status === 404) {
return { state: "absent", status: error.status, url: error.url };
}
return { state: "failed", message: error instanceof Error ? error.message : String(error) };
const message = error instanceof Error ? error.message : String(error);
console.warn(`Treating optional data as missing: ${message}`);
return { state: "failed", message };
}
};
35 changes: 29 additions & 6 deletions apps/www/src/lib/load.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,9 @@ import {
DataClient,
summaryPackageCounts,
type DateSummary,
type DirectoryEntry,
type SummaryResult,
type VariationsByFixture,
} from "@benchmarks/data";
import { probe } from "./data";
import { applyExclusions } from "./figures";
Expand Down Expand Up @@ -73,28 +75,43 @@ export interface ScenarioData {

const asFixture = (name: string): Fixture => name as Fixture;

/** One fixture's three files. The two count files are independently optional. */
/** Whether the run's shard lists a file for this (fixture, variation). */
const published = (listed: VariationsByFixture, fixture: string, variation: string): boolean =>
listed[fixture]?.includes(variation) ?? false;

/**
* One fixture's three files. The two count files are independently optional,
* and only the ones the run's shard lists are requested: `registry-*` publishes
* no process count and `build` no count at all (docs/METHODOLOGY.md), and a
* request for a file that does not exist is a 404 on a good day and a 503
* from GitHub Pages under a burst.
*/
const loadFixture = async (
client: DataClient,
date: string,
fixture: string,
variation: string,
entry: DirectoryEntry,
): Promise<FixtureBundle | null> => {
const file = await probe<BenchmarkFile>(() =>
client.getBenchmark(date, asFixture(fixture), variation),
);
if (file.state !== "present") return null;

const [counts, processes] = await Promise.all([
probe<PackageCount>(() => client.getPackageCount(date, asFixture(fixture), variation)),
probe<ProcessCount>(() => client.getProcessCount(date, asFixture(fixture), variation)),
published(entry.packageCounts, fixture, variation)
? probe<PackageCount>(() => client.getPackageCount(date, asFixture(fixture), variation))
: null,
published(entry.processCounts, fixture, variation)
? probe<ProcessCount>(() => client.getProcessCount(date, asFixture(fixture), variation))
: null,
]);

return {
fixture,
file: file.value,
counts: counts.state === "present" ? counts.value : null,
processes: processes.state === "present" ? processes.value : null,
counts: counts?.state === "present" ? counts.value : null,
processes: processes?.state === "present" ? processes.value : null,
};
};

Expand Down Expand Up @@ -280,13 +297,19 @@ export const loadScenario = async (
readonly perPackage?: boolean;
/** From `loadSummaries`: a summarised run costs one fetch, not one per fixture. */
readonly summaries?: Summaries | Promise<Summaries>;
/**
* The shown run's shard (`getDirectoryEntry`), which says which count files
* it has. Every caller has fetched it already; left out, it is fetched here.
*/
readonly entry?: DirectoryEntry | Promise<DirectoryEntry>;
} = {},
): Promise<ScenarioData> => {
const withCounts = options.perPackage === true;
/* A run without a summary: a file per fixture, and its count where that is wanted. */
const perRun = fixtures.length * (withCounts ? 2 : 1);
const entry = await (options.entry ?? client.getDirectoryEntry(date));
const [bundles, trend] = await Promise.all([
Promise.all(fixtures.map((f) => loadFixture(client, date, f, variation))),
Promise.all(fixtures.map((f) => loadFixture(client, date, f, variation, entry))),
(async () => {
const summaries = await options.summaries;
const dates = trendDates(windowDates, perRun, new Set(summaries?.keys()));
Expand Down
1 change: 1 addition & 0 deletions apps/www/src/lib/report.ts
Original file line number Diff line number Diff line change
Expand Up @@ -396,6 +396,7 @@ export const loadReport = async ({
/* The per-package figures by fixture draw every run, so the counts are always wanted where there are any. */
perPackage: section !== "task-runners",
summaries,
entry,
}));
const facts = scenarioFacts(
section,
Expand Down
163 changes: 163 additions & 0 deletions apps/www/test/data.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,163 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { DataClient, DataFetchError, type DirectoryEntry } from "@benchmarks/data";
import { dataClient, fetchRetrying, probe } from "@/lib/data";
import { loadScenario } from "@/lib/load";

afterEach(() => {
vi.unstubAllGlobals();
vi.restoreAllMocks();
delete process.env["DATA_URL"];
});

/* Answer each request to a URL from its queue of statuses, the last one repeating. */
const answering = (plan: { [url: string]: number[] }) => {
const seen: string[] = [];
vi.stubGlobal("fetch", async (url: string) => {
seen.push(url);
const queue = plan[url] ?? [200];
const status = queue.length > 1 ? (queue.shift() as number) : (queue[0] as number);
return status === 200
? new Response('{"ok":true}', { status })
: new Response(null, { status, headers: { "retry-after": "0" } });
});
return seen;
};

describe("fetchRetrying", () => {
it("retries a 5xx with a short backoff and returns the answer that follows", async () => {
const seen = answering({ "https://x/a.json": [503, 502, 200] });
const response = await fetchRetrying("https://x/a.json");
expect(response.status).toBe(200);
expect(seen).toHaveLength(3);
});

it("gives up on a persistent 5xx after a couple of attempts", async () => {
const seen = answering({ "https://x/a.json": [503] });
const response = await fetchRetrying("https://x/a.json");
expect(response.status).toBe(503);
expect(seen).toHaveLength(3);
});

it("does not retry a 404", async () => {
const seen = answering({ "https://x/a.json": [404] });
expect((await fetchRetrying("https://x/a.json")).status).toBe(404);
expect(seen).toHaveLength(1);
});
});

describe("optional against required data", () => {
it("treats a persistent 5xx on probed data as missing, with a warning", async () => {
process.env["DATA_URL"] = "https://x";
answering({ "https://x/2026-11-18/vue-registry-clean-process-count.json": [503] });
const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
const client = dataClient();
const result = await probe(() =>
client.getProcessCount("2026-11-18", "vue" as never, "registry-clean"),
);
expect(result.state).toBe("failed");
expect(warn).toHaveBeenCalledTimes(1);
expect(warn.mock.calls[0]?.[0]).toContain("503");
});

it("keeps a 404 on probed data as absent, without a warning", async () => {
process.env["DATA_URL"] = "https://x";
answering({ "https://x/2026-11-18/vue-registry-clean-process-count.json": [404] });
const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
const client = dataClient();
const result = await probe(() =>
client.getProcessCount("2026-11-18", "vue" as never, "registry-clean"),
);
expect(result).toMatchObject({ state: "absent", status: 404 });
expect(warn).not.toHaveBeenCalled();
});

it("still fails loudly when data a page cannot render without answers 5xx", async () => {
process.env["DATA_URL"] = "https://x";
const seen = answering({ "https://x/index.json": [503] });
const client = dataClient();
const error = await client.getIndex().catch((e: unknown) => e);
expect(error).toBeInstanceOf(DataFetchError);
expect((error as DataFetchError).status).toBe(503);
/* After its retries, not before. */
expect(seen).toHaveLength(3);
});
});

describe("loadScenario", () => {
const DATE = "2026-11-18";
const results = [{ command: "npm", mean: 1, stddev: 0, exit_codes: [0] }];
/* The run's shard: registry-clean publishes package counts but no process counts, build neither. */
const entry: DirectoryEntry = {
name: DATE,
isLatest: false,
fileCount: 0,
benchmarks: {
astro: ["clean", "registry-clean"],
vue: ["clean", "registry-clean"],
next: ["build"],
},
packageCounts: { astro: ["clean", "registry-clean"], vue: ["clean", "registry-clean"] },
processCounts: { astro: ["clean"], vue: ["clean"] },
hasChartData: false,
hasVersions: true,
};

const recording = () => {
const seen: string[] = [];
const client = new DataClient({
baseUrl: "https://x",
fetch: async (url) => {
seen.push(url.slice("https://x/".length));
if (url.endsWith("manifest/" + DATE + ".json"))
return { ok: true, status: 200, json: async () => entry };
const count = /-(package|process)-count\.json$/.test(url);
return {
ok: true,
status: 200,
json: async () => (count ? { npm: { count: 10 } } : { results }),
};
},
});
return { client, seen };
};

it("requests only the count files the run's shard lists", async () => {
const { client, seen } = recording();
const data = await loadScenario(client, DATE, "registry-clean", ["astro", "vue"], [], {
entry,
});
expect(seen.filter((u) => u.includes("process-count"))).toEqual([]);
expect(seen.filter((u) => u.includes("package-count")).sort()).toEqual([
`${DATE}/astro-registry-clean-package-count.json`,
`${DATE}/vue-registry-clean-package-count.json`,
]);
expect(data.bundles.map((b) => [b.fixture, b.counts !== null, b.processes])).toEqual([
["astro", true, null],
["vue", true, null],
]);
});

it("requests no count file for a scenario that publishes none", async () => {
const { client, seen } = recording();
const data = await loadScenario(client, DATE, "build", ["next"], [], { entry });
expect(seen.filter((u) => u.includes("-count"))).toEqual([]);
expect(data.bundles).toHaveLength(1);
});

it("requests both where the shard lists both", async () => {
const { client, seen } = recording();
const data = await loadScenario(client, DATE, "clean", ["astro"], [], { entry });
expect(seen.filter((u) => u.includes("-count")).sort()).toEqual([
`${DATE}/astro-clean-package-count.json`,
`${DATE}/astro-clean-process-count.json`,
]);
expect(data.bundles[0]?.processes).not.toBeNull();
});

it("fetches the shard itself when the caller has not", async () => {
const { client, seen } = recording();
await loadScenario(client, DATE, "registry-clean", ["astro"], []);
expect(seen).toContain(`manifest/${DATE}.json`);
expect(seen.filter((u) => u.includes("process-count"))).toEqual([]);
});
});
4 changes: 3 additions & 1 deletion apps/www/test/report.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -203,8 +203,10 @@ describe("a trend's fetch budget", () => {
// Two fixtures, each with its count: four files a run, so the newest 100 runs.
expect(data.trendCost).toMatchObject({ perRun: 4, dates: TREND_BUDGET / 4 });
const newest = new Set(window.slice(-TREND_BUDGET / 4));
/* The shown run's files and shard are not the trend's. */
const trendFiles = fetched.filter(
(p) => !p.startsWith("summary/") && !p.startsWith("2026-10-13/"),
(p) =>
!p.startsWith("summary/") && !p.startsWith("manifest/") && !p.startsWith("2026-10-13/"),
);
expect(trendFiles.length).toBeLessThanOrEqual(TREND_BUDGET);
expect(trendFiles.every((p) => newest.has(p.split("/")[0] ?? ""))).toBe(true);
Expand Down