diff --git a/apps/server/test/__artifacts__/metrics.simulation.json b/apps/server/test/__artifacts__/metrics.simulation.json index 67511d82..25844363 100644 --- a/apps/server/test/__artifacts__/metrics.simulation.json +++ b/apps/server/test/__artifacts__/metrics.simulation.json @@ -2,42 +2,42 @@ "seeded": { "cw-a": { "sessions": 300, - "prompts": 2245, - "requests": 7273, - "aborted": 209, - "failed": 140, - "calls": 7827 + "prompts": 2125, + "requests": 6815, + "aborted": 193, + "failed": 160, + "calls": 7251 }, "cw-b": { "sessions": 120, - "prompts": 881, - "requests": 2839, - "aborted": 98, - "failed": 60, - "calls": 3042 + "prompts": 878, + "requests": 2835, + "aborted": 94, + "failed": 50, + "calls": 3036 } }, "connected": { "before": { "all": { - "requests": 10112, - "tokens": 490774644, - "cost": 221.19770285, - "timed": 9805, - "prompts": 3126, - "tools": 10347 + "requests": 9650, + "tokens": 455262754, + "cost": 173.70565119999998, + "timed": 9363, + "prompts": 3003, + "tools": 9791 }, "sessions": 420, "deleted": 0 }, "after": { "all": { - "requests": 10112, - "tokens": 490774644, - "cost": 221.19770285, - "timed": 9805, - "prompts": 3126, - "tools": 10347 + "requests": 9650, + "tokens": 455262754, + "cost": 173.70565119999998, + "timed": 9363, + "prompts": 3003, + "tools": 9791 }, "sessions": 420, "deleted": 7 @@ -45,12 +45,12 @@ }, "live": { "all": { - "requests": 10118, - "tokens": 490774656, - "cost": 221.19770285, - "timed": 9811, - "prompts": 3129, - "tools": 10350 + "requests": 9656, + "tokens": 455262766, + "cost": 173.70565119999998, + "timed": 9369, + "prompts": 3006, + "tools": 9794 }, "sessions": 421, "deleted": 7 @@ -62,12 +62,12 @@ ], "totals": { "all": { - "requests": 10130, - "tokens": 490774680, - "cost": 221.19770285, - "timed": 9823, - "prompts": 3135, - "tools": 10356 + "requests": 9668, + "tokens": 455262790, + "cost": 173.70565119999998, + "timed": 9381, + "prompts": 3012, + "tools": 9800 }, "sessions": 423, "deleted": 7 @@ -76,24 +76,24 @@ "paused": { "paused": { "all": { - "requests": 10134, - "tokens": 490774688, - "cost": 221.19770285, - "timed": 9827, - "prompts": 3137, - "tools": 10358 + "requests": 9672, + "tokens": 455262798, + "cost": 173.70565119999998, + "timed": 9385, + "prompts": 3014, + "tools": 9802 }, "sessions": 424, "deleted": 7 }, "resumed": { "all": { - "requests": 10134, - "tokens": 490774688, - "cost": 221.19770285, - "timed": 9827, - "prompts": 3137, - "tools": 10358 + "requests": 9672, + "tokens": 455262798, + "cost": 173.70565119999998, + "timed": 9385, + "prompts": 3014, + "tools": 9802 }, "sessions": 424, "deleted": 7 @@ -101,36 +101,36 @@ }, "restartedAway": { "all": { - "requests": 10138, - "tokens": 490774696, - "cost": 221.19770285, - "timed": 9831, - "prompts": 3139, - "tools": 10360 + "requests": 9676, + "tokens": 455262806, + "cost": 173.70565119999998, + "timed": 9389, + "prompts": 3016, + "tools": 9804 }, "sessions": 425, "deleted": 7 }, "cut": { "all": { - "requests": 10140, - "tokens": 490774700, - "cost": 221.19770285, - "timed": 9833, - "prompts": 3140, - "tools": 10361 + "requests": 9678, + "tokens": 455262810, + "cost": 173.70565119999998, + "timed": 9391, + "prompts": 3017, + "tools": 9805 }, "sessions": 426, "deleted": 7 }, "restored": { "all": { - "requests": 10144, - "tokens": 490774708, - "cost": 221.19770285, - "timed": 9837, - "prompts": 3142, - "tools": 10363 + "requests": 9682, + "tokens": 455262818, + "cost": 173.70565119999998, + "timed": 9395, + "prompts": 3019, + "tools": 9807 }, "sessions": 428, "deleted": 7 @@ -138,24 +138,24 @@ "readded": { "kept": { "all": { - "requests": 10144, - "tokens": 490774708, - "cost": 221.19770285, - "timed": 9837, - "prompts": 3142, - "tools": 10363 + "requests": 9682, + "tokens": 455262818, + "cost": 173.70565119999998, + "timed": 9395, + "prompts": 3019, + "tools": 9807 }, "sessions": 428, "deleted": 128 }, "again": { "all": { - "requests": 10144, - "tokens": 490774708, - "cost": 221.19770285, - "timed": 9837, - "prompts": 3142, - "tools": 10363 + "requests": 9682, + "tokens": 455262818, + "cost": 173.70565119999998, + "timed": 9395, + "prompts": 3019, + "tools": 9807 }, "sessions": 428, "deleted": 7 @@ -163,12 +163,12 @@ }, "recreated": { "all": { - "requests": 10148, - "tokens": 490774716, - "cost": 221.19770285, - "timed": 9841, - "prompts": 3144, - "tools": 10365 + "requests": 9686, + "tokens": 455262826, + "cost": 173.70565119999998, + "timed": 9399, + "prompts": 3021, + "tools": 9809 }, "sessions": 429, "deleted": 128 diff --git a/apps/server/test/machines/seed.ts b/apps/server/test/machines/seed.ts index f95932ad..5be1eeb7 100644 --- a/apps/server/test/machines/seed.ts +++ b/apps/server/test/machines/seed.ts @@ -1,9 +1,9 @@ /* @effect-diagnostics nodeBuiltinImport:off -- a one-shot script run inside an OrbStack machine. */ /** - * Fills every session of a stopped CodeWork store with a history of turns across the past `days` - * days, through the thread store's own writes, so the usage rollup, thread totals and revisions come + * Fills every session of a stopped CodeWork store with a history of turns across the `days` + * days up to `until`, through the thread store's own writes, so the usage rollup, thread totals and revisions come * out as a live run makes them: - * node --conditions=development seed.ts [] + * node --conditions=development seed.ts * Each session gets a start day (a little busier recently, quieter at weekends) and 2-14 turns on its days; * each turn is a prompt and 1-6 requests to one model of its project's mix, the last answering and * the others calling tools. A few requests fail, a few are interrupted (aborted, never timed), and a @@ -17,10 +17,11 @@ import * as ThreadSchema from "../../../../packages/harness/src/thread/schema.ts import * as Thread from "../../../../packages/harness/src/thread/thread.ts"; import type { ToolStatus } from "../../../../packages/harness/src/db/schema.sql.ts"; -const [database, daysArg = "365", seedArg = "1", untilArg] = process.argv.slice(2); -if (database === undefined) throw new Error("usage: seed.ts []"); +const [database, daysArg, seedArg, untilArg] = process.argv.slice(2); +if (database === undefined || untilArg === undefined) + throw new Error("usage: seed.ts "); const DAYS = Number(daysArg); -const UNTIL = untilArg === undefined ? Date.now() : Number(untilArg); +const UNTIL = Number(untilArg); const DAY = 86_400_000; // mulberry32: the same seed writes the same history. @@ -140,7 +141,9 @@ const program = Effect.gen(function* () { for (let turn = 0, turns = int(2, 14); turn < turns && day >= 0; turn++) { // Work hours, 08:00-20:00 UTC, so most zones see a working day. let at = UNTIL - day * DAY - (UNTIL % DAY) + int(8, 19) * 3_600_000 + int(0, 3_599_999); - if (at > UNTIL) at = UNTIL - int(60_000, 3_600_000); + // Drawn either way, so where `until` falls in its day does not shift the draws after it. + const back = int(60_000, 3_600_000); + if (at > UNTIL) at = UNTIL - back; const promptId = randomUUID(); yield* threads.append({ id: promptId, diff --git a/apps/server/test/metrics.simulation.live.test.ts b/apps/server/test/metrics.simulation.live.test.ts index c598ab9b..15afe52e 100644 --- a/apps/server/test/metrics.simulation.live.test.ts +++ b/apps/server/test/metrics.simulation.live.test.ts @@ -44,6 +44,8 @@ const until = () => new Date(Date.now() + 2 * DAY).toISOString().slice(0, 10); const everything = (zone = "UTC") => ({ zone, since: "1970-01-01", until: until() }); const PROJECTS = ["codework", "billing-api", "mobile-app", "infra", "docs-site", "ml-pipeline"]; const live = { CODEWORK_FIXTURE_HOST_OS: "1", CODEWORK_FIXTURE_NOW: "1" }; +/** Where seeded history ends: fixed, so every run seeds the same history; past, so `until()` covers it. */ +const SEEDED_UNTIL = Date.UTC(2026, 10, 14, 12); /** The machine's server stopped and its home moved aside: the next start makes a new store. */ const aside = async (m: Machine) => { @@ -69,7 +71,7 @@ const seeded = async (m: Machine, count: number, days: number, seed: number) => } await m.stop("TERM"); const wrote = await m.sh( - `cd ~/codework && ~/.local/node/bin/node --conditions=development apps/server/test/machines/seed.ts ${m.database} ${days} ${seed}`, + `cd ~/codework && ~/.local/node/bin/node --conditions=development apps/server/test/machines/seed.ts ${m.database} ${days} ${seed} ${SEEDED_UNTIL}`, ); await m.start(live); return JSON.parse(wrote.trim().split("\n").at(-1)!) as Record;