From 2224c4fca9000e3328ed48a99b2d4ea92ccb5d20 Mon Sep 17 00:00:00 2001 From: thelastbackspace <23155651+thelastbackspace@users.noreply.github.com> Date: Sat, 7 Nov 2026 10:22:25 +0100 Subject: [PATCH] feat(run): traceperf.config.json and a login step Options can live in traceperf.config.json (or --config file); the command line wins. A login step runs before the traffic and sends the token from its JSON response (or its cookies) with every request, so a stale token can't turn a run into a benchmark of 401 pages. --- CHANGELOG.md | 1 + README.md | 21 ++++- src/__tests__/run.test.ts | 49 +++++++++++ src/cli.ts | 14 +++- src/run.ts | 167 +++++++++++++++++++++++++++++++++++++- 5 files changed, 246 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2264f58..b43b9ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `traceperf/auto` wrote each request tree with its own `appendFileSync`, and under load that showed up as the app's "CPU cost" (`writeFileUtf8`). Writes are batched every 20 ms. ### Added +- `traceperf run` reads `traceperf.config.json` (or `--config file`); command-line options win. A `login` step logs in before the traffic and sends the returned token or cookies with every request; `${NAME}` in the file is read from the environment. - `npm run eval` scores `traceperf run` on 20 apps in `examples/slow-apps`: 12 with a planted problem, 8 clean ones where the right answer is "no issues". CI fails on any missed problem or false alarm. Before these fixes: 17/17 found, 15 false alarms; after: 17/17, 0. ### Added diff --git a/README.md b/README.md index 67e2708..a6b4ba1 100644 --- a/README.md +++ b/README.md @@ -105,6 +105,25 @@ Checked on real apps, not only on the planted ones: Prisma's REST example, the R - `--warmup 3`: untimed requests per path first, so cold-start work doesn't look like a problem. - `--no-profile`: don't sample the CPU. - Any start command works: `npx traceperf run -- npm start`. +- `--config file.json`: read the options from a file (`traceperf.config.json` is read automatically when present). Options given on the command line win. + +**Config file and login.** Put the routes in `traceperf.config.json` so the command is just `npx traceperf run`. A `login` step runs once before the traffic, and its token (or cookie) is sent with every request, so an expired token can't silently turn the run into a benchmark of 401 pages. `${NAME}` is read from the environment: + + +```json +{ + "app": "dist/server.js", + "paths": ["/api/articles", "/api/user"], + "login": { + "path": "/api/users/login", + "body": { "user": { "email": "${TEST_EMAIL}", "password": "${TEST_PASSWORD}" } }, + "token": "user.token", + "prefix": "Token " + } +} +``` + +`token` is the dotted path of the token in the JSON response, sent as `authorization: Bearer ` unless `header` / `prefix` say otherwise. Leave `token` out to use the cookies the login sets. `method` defaults to POST. ### In CI: fail a PR that adds queries @@ -130,7 +149,7 @@ It fails when a route makes more calls of any kind per request (database queries - run: npx traceperf run dist/server.js --path /orders --path /users/1 --baseline perf-baseline.json ``` -`examples/slow-apps/` has small apps with one planted problem each (blocking in your code, blocking in built-ins, sequential awaits, N+1, a slow dependency, errors); CI checks that `run` names each one. +`examples/slow-apps/` has 20 small apps: 12 with one planted problem each (blocking in your code and in built-ins, sequential awaits, N+1 over SQL and over HTTP, a slow or flaky dependency, failing requests…) and 8 clean ones where the right answer is "no issues". `npm run eval` scores `run` on them, and CI fails on a missed problem or a false alarm. The same naming works without `run`: `TRACEPERF_PROFILE_BLOCKING=1 node --require traceperf/auto server.js` adds the function names to the exit summary (sampling costs a few percent CPU, so use it while investigating). In code: `profileSlowCalls({ select: root => blockingRanges(root), mergeInlined: true, onProfile })`, then `explain(roots, { hotspots })`. diff --git a/src/__tests__/run.test.ts b/src/__tests__/run.test.ts index 53c37a8..dfc2320 100644 --- a/src/__tests__/run.test.ts +++ b/src/__tests__/run.test.ts @@ -363,3 +363,52 @@ describe('queueFindings', () => { expect(busy.fix).toMatch(/throughput limit/); }); }); + +describe('config file and login', () => { + const read = (text: string) => (file?: string) => (file === 'missing.json' ? undefined : text); + + it('reads options from the file; the command line wins and its --path replaces the file paths', () => { + const config = JSON.stringify({ app: 'server.js', paths: ['/a', '/b'], concurrency: 4, duration: '2s', headers: { 'X-Key': '${KEY}' }, probe: 0 }); + const fromFile = parseRunArgs([], read(config), { KEY: 'secret' }) as RunArgs; + expect(fromFile).toMatchObject({ command: ['server.js'], paths: ['/a', '/b'], concurrency: 4, durationMs: 2000, headers: { 'x-key': 'secret' }, probe: 0 }); + const overridden = parseRunArgs(['-c', '8', '--path', '/c'], read(config), {}) as RunArgs; + expect(overridden).toMatchObject({ command: ['server.js'], paths: ['/c'], concurrency: 8 }); + expect((parseRunArgs(['other.js'], read(config), {}) as RunArgs).command).toEqual(['other.js']); + expect(parseRunArgs([], read('{'), {})).toMatch(/not valid JSON/); + expect(parseRunArgs(['--config', 'missing.json', 'x.js'], read(''), {})).toMatch(/could not read missing.json/); + expect(parseRunArgs([], read('{}'), {})).toMatch(/"app" in the config file/); + }); + + it('logs in first and sends the token from the JSON response with every request', async () => { + const seen: Array = []; + const server = http.createServer((req, res) => { + let body = ''; + req.on('data', c => { body += c; }); + req.on('end', () => { + if (req.url === '/login') { + const { password } = JSON.parse(body); + if (password !== 'pw') { res.statusCode = 401; return res.end('bad password'); } + return res.end(JSON.stringify({ user: { token: 'abc' } })); + } + if (req.url === '/cookie-login') { res.setHeader('set-cookie', ['sid=42; HttpOnly', 'theme=dark']); return res.end('{}'); } + seen.push(req.headers.authorization || req.headers.cookie); + res.end('ok'); + }); + }); + await new Promise(r => server.listen(0, '127.0.0.1', () => r())); + const port = (server.address() as AddressInfo).port; + try { + const args = parseRunArgs(['x.js']) as RunArgs; + const { login } = await import('../run'); + expect(await login(http, args, port, { path: '/login', body: { password: 'pw' }, token: 'user.token' })).toEqual({ authorization: 'Bearer abc' }); + expect(await login(http, args, port, { path: '/login', body: { password: 'pw' }, token: 'user.token', header: 'X-Token', prefix: '' })).toEqual({ 'x-token': 'abc' }); + expect(await login(http, args, port, { path: '/cookie-login' })).toEqual({ cookie: 'sid=42; theme=dark' }); + expect(await login(http, args, port, { path: '/login', body: { password: 'no' }, token: 'user.token' })).toMatch(/answered 401: bad password/); + expect(await login(http, args, port, { path: '/login', body: { password: 'pw' }, token: 'missing.path' })).toMatch(/no "missing.path"/); + await sendLoad(http, { ...args, paths: ['/x'], requests: 2, concurrency: 1, headers: { authorization: 'Bearer abc' } }, port); + expect(seen).toEqual(['Bearer abc', 'Bearer abc']); + } finally { + server.close(); + } + }); +}); diff --git a/src/cli.ts b/src/cli.ts index 05a2c3a..851a310 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -15,6 +15,16 @@ import { renderHtmlReport } from './report-html'; import { createWatcher, WatchOptions } from './watch'; import { parseRunArgs, runApp, RUN_USAGE } from './run'; +/** --config FILE, or traceperf.config.json in the working directory when there is one */ +function readRunConfig(file?: string): string | undefined { + /* eslint-disable @typescript-eslint/no-require-imports */ + const fs = require('fs'); + const target = require('path').resolve(file || 'traceperf.config.json'); + /* eslint-enable @typescript-eslint/no-require-imports */ + if (!file && !fs.existsSync(target)) return undefined; + return fs.readFileSync(target, 'utf8'); +} + export interface ReportOptions { top?: number; /** Return a self-contained HTML report instead of text */ @@ -381,7 +391,7 @@ export function main(argv: string[], io: { return runWatchOnce(rest, io); } if (command === 'run') { - const args = parseRunArgs(rest); + const args = parseRunArgs(rest, readRunConfig, process.env); io.err(typeof args === 'string' ? `traceperf: ${args}\n\n${RUN_USAGE}` : 'traceperf: run starts an app; use the traceperf command'); return 1; } @@ -559,7 +569,7 @@ if (typeof require !== 'undefined' && typeof module !== 'undefined' && require.m const fs = require('fs'); const argv = process.argv.slice(2); const watchArgs = argv[0] === 'watch' ? parseWatchArgs(argv.slice(1)) : undefined; - const runArgs = argv[0] === 'run' ? parseRunArgs(argv.slice(1)) : undefined; + const runArgs = argv[0] === 'run' ? parseRunArgs(argv.slice(1), readRunConfig, process.env) : undefined; if (watchArgs && typeof watchArgs !== 'string' && !watchArgs.once) { watchLive(watchArgs, fs); } else if (runArgs && typeof runArgs !== 'string') { diff --git a/src/run.ts b/src/run.ts index 6608205..7068c8c 100644 --- a/src/run.ts +++ b/src/run.ts @@ -45,6 +45,8 @@ export interface RunArgs { profile: boolean; /** Ms to wait for the app to listen before giving up */ startTimeoutMs: number; + /** From the config file: log in first and send the token with every request */ + login?: RunLogin; } export const RUN_USAGE = ` traceperf run [app args] [options] @@ -67,6 +69,9 @@ export const RUN_USAGE = ` traceperf run [app args] [options] --baseline f.json compare with it; exit 1 if a route makes more calls (e.g. an N+1) or a new problem appears. Counts, not timings: deterministic, safe for CI + --config f.json read these options from a file (default: traceperf.config.json + if present; command-line options win). It can also log in + first: "login": {"path": "/api/login", "body": {...}, "token": "user.token"} --no-profile don't sample the CPU (blocking code is then named only by its tracked call)`; @@ -78,8 +83,113 @@ function parseDuration(value: string | undefined): number | undefined { } /** Parse `run` arguments; returns an error message on bad input */ -export function parseRunArgs(argv: string[]): RunArgs | string { - const args: RunArgs = { +/** Log in before the traffic and send the token (or cookie) it returns with every request */ +export interface RunLogin { + path: string; + /** Default POST */ + method?: string; + /** JSON request body; "${NAME}" in strings is replaced by the environment variable */ + body?: unknown; + headers?: Record; + /** Dotted path of the token in the JSON response, e.g. "user.token"; without it the Set-Cookie header is used */ + token?: string; + /** Header to send it in (default authorization) */ + header?: string; + /** Text before the token (default "Bearer ") */ + prefix?: string; +} + +/** traceperf.config.json: the same settings as the command-line options */ +export interface RunConfig { + app?: string | string[]; + paths?: string[]; + concurrency?: number; + duration?: string | number; + requests?: number; + method?: string; + headers?: Record; + warmup?: number; + probe?: number; + port?: number; + baseline?: string; + saveBaseline?: string; + html?: string; + login?: RunLogin; +} + +/** Replace "${NAME}" with process environment values, in every string of a JSON value */ +export function interpolateEnv(value: T, env: Record): T { + if (typeof value === 'string') return value.replace(/\$\{(\w+)\}/g, (_, name) => env[name] ?? '') as unknown as T; + if (Array.isArray(value)) return value.map(v => interpolateEnv(v, env)) as unknown as T; + if (value && typeof value === 'object') { + return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, interpolateEnv(v, env)])) as T; + } + return value; +} + +function configArgv(c: RunConfig): string[] { + const out: string[] = []; + const opt = (flag: string, v: unknown) => { if (v !== undefined) out.push(flag, String(v)); }; + for (const p of c.paths || []) out.push('--path', p); + opt('-c', c.concurrency); + opt('-d', typeof c.duration === 'number' ? `${c.duration}ms` : c.duration); + opt('-n', c.requests); + opt('--method', c.method); + for (const [k, v] of Object.entries(c.headers || {})) out.push('-H', `${k}: ${v}`); + opt('--warmup', c.warmup); + opt('--probe', c.probe); + opt('--port', c.port); + opt('--baseline', c.baseline); + opt('--save-baseline', c.saveBaseline); + opt('--html', c.html); + return out; +} + +/** + * Read run options; `readConfig(file?)` returns the text of --config FILE, or + * of traceperf.config.json when there is one (undefined: none). Command-line + * options win over the file; --path on the command line replaces its paths. + */ +export function parseRunArgs(argv: string[], readConfig?: (file?: string) => string | undefined, env: Record = {}): RunArgs | string { + const at = argv.indexOf('--config'); + const dashdash = argv.indexOf('--'); + let configFile: string | undefined; + if (at >= 0 && (dashdash < 0 || at < dashdash)) { + configFile = argv[at + 1]; + if (!configFile) return '--config expects a file'; + argv = [...argv.slice(0, at), ...argv.slice(at + 2)]; + } + let config: RunConfig | undefined; + if (readConfig) { + let text: string | undefined; + try { + text = readConfig(configFile); + } catch (e) { + return `could not read ${configFile}: ${e instanceof Error ? e.message : String(e)}`; + } + if (configFile && text === undefined) return `could not read ${configFile}`; + if (text !== undefined) { + try { + config = interpolateEnv(JSON.parse(text) as RunConfig, env); + } catch (e) { + return `${configFile || 'traceperf.config.json'} is not valid JSON: ${e instanceof Error ? e.message : String(e)}`; + } + } + } + if (!config) return parseArgv(argv); + const base = parseArgv([...configArgv(config), '--', ...(typeof config.app === 'string' ? [config.app] : config.app || [])], true); + if (typeof base === 'string') return `in the config file: ${base}`; + base.login = config.login; + const cli = parseArgv(argv, true, { ...base, paths: [], command: [] }); + if (typeof cli === 'string') return cli; + if (!cli.paths.length) cli.paths = base.paths; + if (!cli.command.length) cli.command = base.command; + if (!cli.command.length) return 'run expects an app, e.g. traceperf run server.js (or "app" in the config file)'; + return cli; +} + +function parseArgv(argv: string[], partial = false, start?: RunArgs): RunArgs | string { + const args: RunArgs = start ? { ...start } : { command: [], paths: [], concurrency: 10, durationMs: 5000, host: '127.0.0.1', method: 'GET', warmup: 3, probe: 10, json: false, profile: true, max: 10, startTimeoutMs: 15000, }; @@ -158,10 +268,50 @@ export function parseRunArgs(argv: string[]): RunArgs | string { args.command.push(a); } } - if (!args.command.length) return 'run expects an app, e.g. traceperf run server.js'; + if (!args.command.length && !partial) return 'run expects an app, e.g. traceperf run server.js'; return args; } +/** Read a dotted path ("user.token", "data.0.jwt") from a JSON value */ +function dig(value: unknown, dotted: string): unknown { + return dotted.split('.').reduce((v, key) => (v !== null && typeof v === 'object' ? (v as Record)[key] : undefined), value); +} + +/** Run the login request; returns the headers to send from now on, or an error message */ +export function login(http: any, args: RunArgs, port: number, spec: RunLogin): Promise | string> { + const body = spec.body === undefined ? undefined : JSON.stringify(spec.body); + return new Promise(resolve => { + const req = http.request({ + host: args.host, port, path: spec.path, method: (spec.method || 'POST').toUpperCase(), + headers: { ...(body ? { 'content-type': 'application/json', 'content-length': Buffer.byteLength(body) } : {}), ...(spec.headers || {}) }, + }, (res: any) => { + let text = ''; + res.setEncoding('utf8'); + res.on('data', (chunk: string) => { text += chunk; }); + res.on('end', () => { + const where = `login ${spec.method || 'POST'} ${spec.path}`; + if (res.statusCode >= 400) return resolve(`${where} answered ${res.statusCode}: ${text.slice(0, 200)}`); + if (!spec.token) { + const cookies: string[] = [].concat(res.headers['set-cookie'] || []); + if (!cookies.length) return resolve(`${where} set no cookie; give "token": the path of the token in its JSON response`); + return resolve({ cookie: cookies.map(c => c.split(';')[0]).join('; ') }); + } + let token: unknown; + try { + token = dig(JSON.parse(text), spec.token); + } catch { + return resolve(`${where} did not answer JSON: ${text.slice(0, 200)}`); + } + if (typeof token !== 'string' && typeof token !== 'number') return resolve(`${where}: no "${spec.token}" in the response: ${text.slice(0, 200)}`); + resolve({ [(spec.header || 'authorization').toLowerCase()]: `${spec.prefix ?? 'Bearer '}${token}` }); + }); + }); + req.on('error', (e: Error) => resolve(`login failed: ${e.message}`)); + if (body) req.write(body); + req.end(); + }); +} + export interface LoadResult { requests: number; errors: number; @@ -576,6 +726,17 @@ export async function runApp(args: RunArgs, deps: RunDeps): Promise { return 1; } deps.err(`traceperf: sending ${args.requests ? `${args.requests} requests` : `${args.durationMs / 1000} s of traffic`} to ${args.paths.join(', ')} on port ${port}, ${args.concurrency} at a time`); + if (args.login) { + const got = await login(deps.http, args, port, args.login); + if (typeof got === 'string') { + deps.err(`traceperf: ${got}`); + await stopAndWait(); + return 1; + } + // Later headers from -H win over the login's + args = { ...args, headers: { ...got, ...(args.headers || {}) } }; + deps.err(`traceperf: logged in (${Object.keys(got).join(', ')} set)`); + } if (args.warmup > 0) { await sendLoad(deps.http, { ...args, concurrency: 1, requests: args.warmup * args.paths.length }, port); skipBytes = await sizeNow();