diff --git a/apps/www/src/app/about/components.tsx b/apps/www/src/app/about/components.tsx index 74f0bec959..80d13a8952 100644 --- a/apps/www/src/app/about/components.tsx +++ b/apps/www/src/app/about/components.tsx @@ -281,7 +281,9 @@ export const LatestResults = async () => { const scenario = scenarios[0]; if (scenario === undefined) return { section, scenario: null }; const fixtures = fixturesWith(entry.benchmarks, scenario.variation); - const data = await loadScenario(client, date, scenario.variation, fixtures, []); + const data = await loadScenario(client, date, scenario.variation, fixtures, [], { + entry, + }); return { section, scenario, diff --git a/apps/www/src/lib/data.ts b/apps/www/src/lib/data.ts index 95b7cfb30c..327b267c83 100644 --- a/apps/www/src/lib/data.ts +++ b/apps/www/src/lib/data.ts @@ -43,6 +43,11 @@ export const dataBaseUrl = (): string => { * fans out to hundreds of files, on every build worker at once. So at most * MAX_IN_FLIGHT requests run per process, and a 429 waits and tries again. * + * It also answers some requests with a 5xx that a retry does not see again -- + * seen on 2026-11-18, where a path that does not exist was a 503 one minute + * and a 404 the next. A 5xx is tried SERVER_RETRIES more times with a short + * backoff before it is believed. + * * The queue is inside "use cache", in the fill: a prerender's second pass * must find every entry already filled, so nothing may wait before the cached * call. `next dev` skips it, since it reports a fill idle for 10s as a @@ -52,6 +57,7 @@ export const dataBaseUrl = (): string => { */ const MAX_IN_FLIGHT = 12; const RETRIES = 5; +const SERVER_RETRIES = 2; let inFlight = 0; const waiting: (() => void)[] = []; @@ -80,12 +86,16 @@ export const throttle = async (task: () => Promise): Promise => { }; export const fetchRetrying = async (url: string): Promise => { - for (let attempt = 0; ; attempt++) { + let limited = 0; + let broken = 0; + for (;;) { const response = await fetch(url); - if (response.status !== 429 || attempt === RETRIES) return response; + if (response.status === 429 && limited < RETRIES) limited++; + else if (response.status >= 500 && broken < SERVER_RETRIES) broken++; + else return response; await response.body?.cancel(); const after = Number(response.headers.get("retry-after")); - const ms = after > 0 ? after * 1000 : 250 * 2 ** attempt; + const ms = after > 0 ? after * 1000 : 250 * 2 ** (limited + broken - 1); await new Promise((resolve) => setTimeout(resolve, ms)); } }; @@ -99,12 +109,20 @@ const throttledFetch = (url: string): Promise => * kept for a day and refreshed behind the reader; the index, the full * manifest and `latest/` move with each run, and a 404 may be a date not yet * published, so those are kept a minute -- the shortest life that still lets - * a page prerender. A failure other than a 404 throws, and is not cached. + * a page prerender. + * + * A 5xx that outlasts `fetchRetrying` is returned like a 404, for the same + * minute, and `DataClient` turns it into a `DataFetchError` the caller sees. + * It is not thrown here: on 2026-11-18 every build failed on a 503 thrown from + * this function for a process-count file reached only through {@link probe}, + * which catches it -- the prerender fails a page on an error thrown inside + * "use cache" whether or not the caller catches it. Any other failure throws, + * and is not cached. */ const cachedGet = async (url: string) => { "use cache"; const response = await throttledFetch(url); - if (!response.ok && response.status !== 404) { + if (!response.ok && response.status !== 404 && response.status < 500) { throw new Error(`Request for ${url} failed with status ${response.status}`); } if (response.ok && /\/\d{4}-\d{2}-\d{2}(\/|\.json$)/.test(url)) cacheLife("days"); @@ -133,6 +151,14 @@ export const dataClient = (): DataClient => * every `registry-*` variation and `versions.json` is genuinely absent from * `latest/`, so a 404 there is the honest answer and must never be rendered as * a zero or an em dash. A 500 is a different thing and says so. + * + * Everything probed is data a page renders without: a count file, a run's + * `versions.json`, a summary. So a 5xx that outlasts the retries is reported + * as `failed` -- which every reader shows as missing -- and warned about on + * the server, rather than failing the page. Data a page cannot render without + * (the index, a run's shard) is never probed, and its `DataFetchError` is + * thrown, so a real outage still fails the build instead of publishing an + * empty site. */ export type Probe = | { readonly state: "present"; readonly value: T } @@ -146,6 +172,8 @@ export const probe = async (fetcher: () => Promise): Promise> => if (error instanceof DataFetchError && error.status === 404) { return { state: "absent", status: error.status, url: error.url }; } - return { state: "failed", message: error instanceof Error ? error.message : String(error) }; + const message = error instanceof Error ? error.message : String(error); + console.warn(`Treating optional data as missing: ${message}`); + return { state: "failed", message }; } }; diff --git a/apps/www/src/lib/load.ts b/apps/www/src/lib/load.ts index 300a3e224a..849d6b51c1 100644 --- a/apps/www/src/lib/load.ts +++ b/apps/www/src/lib/load.ts @@ -12,7 +12,9 @@ import { DataClient, summaryPackageCounts, type DateSummary, + type DirectoryEntry, type SummaryResult, + type VariationsByFixture, } from "@benchmarks/data"; import { probe } from "./data"; import { applyExclusions } from "./figures"; @@ -73,12 +75,23 @@ export interface ScenarioData { const asFixture = (name: string): Fixture => name as Fixture; -/** One fixture's three files. The two count files are independently optional. */ +/** Whether the run's shard lists a file for this (fixture, variation). */ +const published = (listed: VariationsByFixture, fixture: string, variation: string): boolean => + listed[fixture]?.includes(variation) ?? false; + +/** + * One fixture's three files. The two count files are independently optional, + * and only the ones the run's shard lists are requested: `registry-*` publishes + * no process count and `build` no count at all (docs/METHODOLOGY.md), and a + * request for a file that does not exist is a 404 on a good day and a 503 + * from GitHub Pages under a burst. + */ const loadFixture = async ( client: DataClient, date: string, fixture: string, variation: string, + entry: DirectoryEntry, ): Promise => { const file = await probe(() => client.getBenchmark(date, asFixture(fixture), variation), @@ -86,15 +99,19 @@ const loadFixture = async ( if (file.state !== "present") return null; const [counts, processes] = await Promise.all([ - probe(() => client.getPackageCount(date, asFixture(fixture), variation)), - probe(() => client.getProcessCount(date, asFixture(fixture), variation)), + published(entry.packageCounts, fixture, variation) + ? probe(() => client.getPackageCount(date, asFixture(fixture), variation)) + : null, + published(entry.processCounts, fixture, variation) + ? probe(() => client.getProcessCount(date, asFixture(fixture), variation)) + : null, ]); return { fixture, file: file.value, - counts: counts.state === "present" ? counts.value : null, - processes: processes.state === "present" ? processes.value : null, + counts: counts?.state === "present" ? counts.value : null, + processes: processes?.state === "present" ? processes.value : null, }; }; @@ -280,13 +297,19 @@ export const loadScenario = async ( readonly perPackage?: boolean; /** From `loadSummaries`: a summarised run costs one fetch, not one per fixture. */ readonly summaries?: Summaries | Promise; + /** + * The shown run's shard (`getDirectoryEntry`), which says which count files + * it has. Every caller has fetched it already; left out, it is fetched here. + */ + readonly entry?: DirectoryEntry | Promise; } = {}, ): Promise => { const withCounts = options.perPackage === true; /* A run without a summary: a file per fixture, and its count where that is wanted. */ const perRun = fixtures.length * (withCounts ? 2 : 1); + const entry = await (options.entry ?? client.getDirectoryEntry(date)); const [bundles, trend] = await Promise.all([ - Promise.all(fixtures.map((f) => loadFixture(client, date, f, variation))), + Promise.all(fixtures.map((f) => loadFixture(client, date, f, variation, entry))), (async () => { const summaries = await options.summaries; const dates = trendDates(windowDates, perRun, new Set(summaries?.keys())); diff --git a/apps/www/src/lib/report.ts b/apps/www/src/lib/report.ts index e4b6ce8bfa..509e9b7b29 100644 --- a/apps/www/src/lib/report.ts +++ b/apps/www/src/lib/report.ts @@ -396,6 +396,7 @@ export const loadReport = async ({ /* The per-package figures by fixture draw every run, so the counts are always wanted where there are any. */ perPackage: section !== "task-runners", summaries, + entry, })); const facts = scenarioFacts( section, diff --git a/apps/www/test/data.test.ts b/apps/www/test/data.test.ts new file mode 100644 index 0000000000..5f2bdb0a00 --- /dev/null +++ b/apps/www/test/data.test.ts @@ -0,0 +1,163 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { DataClient, DataFetchError, type DirectoryEntry } from "@benchmarks/data"; +import { dataClient, fetchRetrying, probe } from "@/lib/data"; +import { loadScenario } from "@/lib/load"; + +afterEach(() => { + vi.unstubAllGlobals(); + vi.restoreAllMocks(); + delete process.env["DATA_URL"]; +}); + +/* Answer each request to a URL from its queue of statuses, the last one repeating. */ +const answering = (plan: { [url: string]: number[] }) => { + const seen: string[] = []; + vi.stubGlobal("fetch", async (url: string) => { + seen.push(url); + const queue = plan[url] ?? [200]; + const status = queue.length > 1 ? (queue.shift() as number) : (queue[0] as number); + return status === 200 + ? new Response('{"ok":true}', { status }) + : new Response(null, { status, headers: { "retry-after": "0" } }); + }); + return seen; +}; + +describe("fetchRetrying", () => { + it("retries a 5xx with a short backoff and returns the answer that follows", async () => { + const seen = answering({ "https://x/a.json": [503, 502, 200] }); + const response = await fetchRetrying("https://x/a.json"); + expect(response.status).toBe(200); + expect(seen).toHaveLength(3); + }); + + it("gives up on a persistent 5xx after a couple of attempts", async () => { + const seen = answering({ "https://x/a.json": [503] }); + const response = await fetchRetrying("https://x/a.json"); + expect(response.status).toBe(503); + expect(seen).toHaveLength(3); + }); + + it("does not retry a 404", async () => { + const seen = answering({ "https://x/a.json": [404] }); + expect((await fetchRetrying("https://x/a.json")).status).toBe(404); + expect(seen).toHaveLength(1); + }); +}); + +describe("optional against required data", () => { + it("treats a persistent 5xx on probed data as missing, with a warning", async () => { + process.env["DATA_URL"] = "https://x"; + answering({ "https://x/2026-11-18/vue-registry-clean-process-count.json": [503] }); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const client = dataClient(); + const result = await probe(() => + client.getProcessCount("2026-11-18", "vue" as never, "registry-clean"), + ); + expect(result.state).toBe("failed"); + expect(warn).toHaveBeenCalledTimes(1); + expect(warn.mock.calls[0]?.[0]).toContain("503"); + }); + + it("keeps a 404 on probed data as absent, without a warning", async () => { + process.env["DATA_URL"] = "https://x"; + answering({ "https://x/2026-11-18/vue-registry-clean-process-count.json": [404] }); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const client = dataClient(); + const result = await probe(() => + client.getProcessCount("2026-11-18", "vue" as never, "registry-clean"), + ); + expect(result).toMatchObject({ state: "absent", status: 404 }); + expect(warn).not.toHaveBeenCalled(); + }); + + it("still fails loudly when data a page cannot render without answers 5xx", async () => { + process.env["DATA_URL"] = "https://x"; + const seen = answering({ "https://x/index.json": [503] }); + const client = dataClient(); + const error = await client.getIndex().catch((e: unknown) => e); + expect(error).toBeInstanceOf(DataFetchError); + expect((error as DataFetchError).status).toBe(503); + /* After its retries, not before. */ + expect(seen).toHaveLength(3); + }); +}); + +describe("loadScenario", () => { + const DATE = "2026-11-18"; + const results = [{ command: "npm", mean: 1, stddev: 0, exit_codes: [0] }]; + /* The run's shard: registry-clean publishes package counts but no process counts, build neither. */ + const entry: DirectoryEntry = { + name: DATE, + isLatest: false, + fileCount: 0, + benchmarks: { + astro: ["clean", "registry-clean"], + vue: ["clean", "registry-clean"], + next: ["build"], + }, + packageCounts: { astro: ["clean", "registry-clean"], vue: ["clean", "registry-clean"] }, + processCounts: { astro: ["clean"], vue: ["clean"] }, + hasChartData: false, + hasVersions: true, + }; + + const recording = () => { + const seen: string[] = []; + const client = new DataClient({ + baseUrl: "https://x", + fetch: async (url) => { + seen.push(url.slice("https://x/".length)); + if (url.endsWith("manifest/" + DATE + ".json")) + return { ok: true, status: 200, json: async () => entry }; + const count = /-(package|process)-count\.json$/.test(url); + return { + ok: true, + status: 200, + json: async () => (count ? { npm: { count: 10 } } : { results }), + }; + }, + }); + return { client, seen }; + }; + + it("requests only the count files the run's shard lists", async () => { + const { client, seen } = recording(); + const data = await loadScenario(client, DATE, "registry-clean", ["astro", "vue"], [], { + entry, + }); + expect(seen.filter((u) => u.includes("process-count"))).toEqual([]); + expect(seen.filter((u) => u.includes("package-count")).sort()).toEqual([ + `${DATE}/astro-registry-clean-package-count.json`, + `${DATE}/vue-registry-clean-package-count.json`, + ]); + expect(data.bundles.map((b) => [b.fixture, b.counts !== null, b.processes])).toEqual([ + ["astro", true, null], + ["vue", true, null], + ]); + }); + + it("requests no count file for a scenario that publishes none", async () => { + const { client, seen } = recording(); + const data = await loadScenario(client, DATE, "build", ["next"], [], { entry }); + expect(seen.filter((u) => u.includes("-count"))).toEqual([]); + expect(data.bundles).toHaveLength(1); + }); + + it("requests both where the shard lists both", async () => { + const { client, seen } = recording(); + const data = await loadScenario(client, DATE, "clean", ["astro"], [], { entry }); + expect(seen.filter((u) => u.includes("-count")).sort()).toEqual([ + `${DATE}/astro-clean-package-count.json`, + `${DATE}/astro-clean-process-count.json`, + ]); + expect(data.bundles[0]?.processes).not.toBeNull(); + }); + + it("fetches the shard itself when the caller has not", async () => { + const { client, seen } = recording(); + await loadScenario(client, DATE, "registry-clean", ["astro"], []); + expect(seen).toContain(`manifest/${DATE}.json`); + expect(seen.filter((u) => u.includes("process-count"))).toEqual([]); + }); +}); diff --git a/apps/www/test/report.test.ts b/apps/www/test/report.test.ts index 25d366893b..6c50b18ce7 100644 --- a/apps/www/test/report.test.ts +++ b/apps/www/test/report.test.ts @@ -203,8 +203,10 @@ describe("a trend's fetch budget", () => { // Two fixtures, each with its count: four files a run, so the newest 100 runs. expect(data.trendCost).toMatchObject({ perRun: 4, dates: TREND_BUDGET / 4 }); const newest = new Set(window.slice(-TREND_BUDGET / 4)); + /* The shown run's files and shard are not the trend's. */ const trendFiles = fetched.filter( - (p) => !p.startsWith("summary/") && !p.startsWith("2026-10-13/"), + (p) => + !p.startsWith("summary/") && !p.startsWith("manifest/") && !p.startsWith("2026-10-13/"), ); expect(trendFiles.length).toBeLessThanOrEqual(TREND_BUDGET); expect(trendFiles.every((p) => newest.has(p.split("/")[0] ?? ""))).toBe(true);