Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
164 changes: 82 additions & 82 deletions apps/server/test/__artifacts__/metrics.simulation.json
Original file line number Diff line number Diff line change
Expand Up @@ -2,55 +2,55 @@
"seeded": {
"cw-a": {
"sessions": 300,
"prompts": 2245,
"requests": 7273,
"aborted": 209,
"failed": 140,
"calls": 7827
"prompts": 2125,
"requests": 6815,
"aborted": 193,
"failed": 160,
"calls": 7251
},
"cw-b": {
"sessions": 120,
"prompts": 881,
"requests": 2839,
"aborted": 98,
"failed": 60,
"calls": 3042
"prompts": 878,
"requests": 2835,
"aborted": 94,
"failed": 50,
"calls": 3036
}
},
"connected": {
"before": {
"all": {
"requests": 10112,
"tokens": 490774644,
"cost": 221.19770285,
"timed": 9805,
"prompts": 3126,
"tools": 10347
"requests": 9650,
"tokens": 455262754,
"cost": 173.70565119999998,
"timed": 9363,
"prompts": 3003,
"tools": 9791
},
"sessions": 420,
"deleted": 0
},
"after": {
"all": {
"requests": 10112,
"tokens": 490774644,
"cost": 221.19770285,
"timed": 9805,
"prompts": 3126,
"tools": 10347
"requests": 9650,
"tokens": 455262754,
"cost": 173.70565119999998,
"timed": 9363,
"prompts": 3003,
"tools": 9791
},
"sessions": 420,
"deleted": 7
}
},
"live": {
"all": {
"requests": 10118,
"tokens": 490774656,
"cost": 221.19770285,
"timed": 9811,
"prompts": 3129,
"tools": 10350
"requests": 9656,
"tokens": 455262766,
"cost": 173.70565119999998,
"timed": 9369,
"prompts": 3006,
"tools": 9794
},
"sessions": 421,
"deleted": 7
Expand All @@ -62,12 +62,12 @@
],
"totals": {
"all": {
"requests": 10130,
"tokens": 490774680,
"cost": 221.19770285,
"timed": 9823,
"prompts": 3135,
"tools": 10356
"requests": 9668,
"tokens": 455262790,
"cost": 173.70565119999998,
"timed": 9381,
"prompts": 3012,
"tools": 9800
},
"sessions": 423,
"deleted": 7
Expand All @@ -76,99 +76,99 @@
"paused": {
"paused": {
"all": {
"requests": 10134,
"tokens": 490774688,
"cost": 221.19770285,
"timed": 9827,
"prompts": 3137,
"tools": 10358
"requests": 9672,
"tokens": 455262798,
"cost": 173.70565119999998,
"timed": 9385,
"prompts": 3014,
"tools": 9802
},
"sessions": 424,
"deleted": 7
},
"resumed": {
"all": {
"requests": 10134,
"tokens": 490774688,
"cost": 221.19770285,
"timed": 9827,
"prompts": 3137,
"tools": 10358
"requests": 9672,
"tokens": 455262798,
"cost": 173.70565119999998,
"timed": 9385,
"prompts": 3014,
"tools": 9802
},
"sessions": 424,
"deleted": 7
}
},
"restartedAway": {
"all": {
"requests": 10138,
"tokens": 490774696,
"cost": 221.19770285,
"timed": 9831,
"prompts": 3139,
"tools": 10360
"requests": 9676,
"tokens": 455262806,
"cost": 173.70565119999998,
"timed": 9389,
"prompts": 3016,
"tools": 9804
},
"sessions": 425,
"deleted": 7
},
"cut": {
"all": {
"requests": 10140,
"tokens": 490774700,
"cost": 221.19770285,
"timed": 9833,
"prompts": 3140,
"tools": 10361
"requests": 9678,
"tokens": 455262810,
"cost": 173.70565119999998,
"timed": 9391,
"prompts": 3017,
"tools": 9805
},
"sessions": 426,
"deleted": 7
},
"restored": {
"all": {
"requests": 10144,
"tokens": 490774708,
"cost": 221.19770285,
"timed": 9837,
"prompts": 3142,
"tools": 10363
"requests": 9682,
"tokens": 455262818,
"cost": 173.70565119999998,
"timed": 9395,
"prompts": 3019,
"tools": 9807
},
"sessions": 428,
"deleted": 7
},
"readded": {
"kept": {
"all": {
"requests": 10144,
"tokens": 490774708,
"cost": 221.19770285,
"timed": 9837,
"prompts": 3142,
"tools": 10363
"requests": 9682,
"tokens": 455262818,
"cost": 173.70565119999998,
"timed": 9395,
"prompts": 3019,
"tools": 9807
},
"sessions": 428,
"deleted": 128
},
"again": {
"all": {
"requests": 10144,
"tokens": 490774708,
"cost": 221.19770285,
"timed": 9837,
"prompts": 3142,
"tools": 10363
"requests": 9682,
"tokens": 455262818,
"cost": 173.70565119999998,
"timed": 9395,
"prompts": 3019,
"tools": 9807
},
"sessions": 428,
"deleted": 7
}
},
"recreated": {
"all": {
"requests": 10148,
"tokens": 490774716,
"cost": 221.19770285,
"timed": 9841,
"prompts": 3144,
"tools": 10365
"requests": 9686,
"tokens": 455262826,
"cost": 173.70565119999998,
"timed": 9399,
"prompts": 3021,
"tools": 9809
},
"sessions": 429,
"deleted": 128
Expand Down
17 changes: 10 additions & 7 deletions apps/server/test/machines/seed.ts
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
/* @effect-diagnostics nodeBuiltinImport:off -- a one-shot script run inside an OrbStack machine. */
/**
* Fills every session of a stopped CodeWork store with a history of turns across the past `days`
* days, through the thread store's own writes, so the usage rollup, thread totals and revisions come
* Fills every session of a stopped CodeWork store with a history of turns across the `days`
* days up to `until`, through the thread store's own writes, so the usage rollup, thread totals and revisions come
* out as a live run makes them:
* node --conditions=development seed.ts <database> <days> <seed> [<until ms>]
* node --conditions=development seed.ts <database> <days> <seed> <until ms>
* Each session gets a start day (a little busier recently, quieter at weekends) and 2-14 turns on its days;
* each turn is a prompt and 1-6 requests to one model of its project's mix, the last answering and
* the others calling tools. A few requests fail, a few are interrupted (aborted, never timed), and a
Expand All @@ -17,10 +17,11 @@ import * as ThreadSchema from "../../../../packages/harness/src/thread/schema.ts
import * as Thread from "../../../../packages/harness/src/thread/thread.ts";
import type { ToolStatus } from "../../../../packages/harness/src/db/schema.sql.ts";

const [database, daysArg = "365", seedArg = "1", untilArg] = process.argv.slice(2);
if (database === undefined) throw new Error("usage: seed.ts <database> <days> <seed> [<until ms>]");
const [database, daysArg, seedArg, untilArg] = process.argv.slice(2);
if (database === undefined || untilArg === undefined)
throw new Error("usage: seed.ts <database> <days> <seed> <until ms>");
const DAYS = Number(daysArg);
const UNTIL = untilArg === undefined ? Date.now() : Number(untilArg);
const UNTIL = Number(untilArg);
const DAY = 86_400_000;

// mulberry32: the same seed writes the same history.
Expand Down Expand Up @@ -140,7 +141,9 @@ const program = Effect.gen(function* () {
for (let turn = 0, turns = int(2, 14); turn < turns && day >= 0; turn++) {
// Work hours, 08:00-20:00 UTC, so most zones see a working day.
let at = UNTIL - day * DAY - (UNTIL % DAY) + int(8, 19) * 3_600_000 + int(0, 3_599_999);
if (at > UNTIL) at = UNTIL - int(60_000, 3_600_000);
// Drawn either way, so where `until` falls in its day does not shift the draws after it.
const back = int(60_000, 3_600_000);
if (at > UNTIL) at = UNTIL - back;
const promptId = randomUUID();
yield* threads.append({
id: promptId,
Expand Down
4 changes: 3 additions & 1 deletion apps/server/test/metrics.simulation.live.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@ const until = () => new Date(Date.now() + 2 * DAY).toISOString().slice(0, 10);
const everything = (zone = "UTC") => ({ zone, since: "1970-01-01", until: until() });
const PROJECTS = ["codework", "billing-api", "mobile-app", "infra", "docs-site", "ml-pipeline"];
const live = { CODEWORK_FIXTURE_HOST_OS: "1", CODEWORK_FIXTURE_NOW: "1" };
/** Where seeded history ends: fixed, so every run seeds the same history; past, so `until()` covers it. */
const SEEDED_UNTIL = Date.UTC(2026, 10, 14, 12);

/** The machine's server stopped and its home moved aside: the next start makes a new store. */
const aside = async (m: Machine) => {
Expand All @@ -69,7 +71,7 @@ const seeded = async (m: Machine, count: number, days: number, seed: number) =>
}
await m.stop("TERM");
const wrote = await m.sh(
`cd ~/codework && ~/.local/node/bin/node --conditions=development apps/server/test/machines/seed.ts ${m.database} ${days} ${seed}`,
`cd ~/codework && ~/.local/node/bin/node --conditions=development apps/server/test/machines/seed.ts ${m.database} ${days} ${seed} ${SEEDED_UNTIL}`,
);
await m.start(live);
return JSON.parse(wrote.trim().split("\n").at(-1)!) as Record<string, number>;
Expand Down