From 9dd9552cc1f451be7e35d6968a344af2ac4c971a Mon Sep 17 00:00:00 2001 From: Sanchit Rk <0xsanchit@gmail.com> Date: Sat, 14 Nov 2026 14:16:44 +0530 Subject: [PATCH 1/8] feat(harness): keep a durable usage rollup per hour, thread and model Two tables written in the transaction that finalises an entry: usage (requests, tokens by type, reasoning, cost by type, response time and a duration histogram per UTC hour, thread, provider, model and thinking level) and tally (user messages and tool calls that ran). Rows carry the session and project they were written for and no foreign key, so they outlive a deleted session. Aborted requests are billed and counted but never timed. A migration backfills from existing entries with the same code as the live path. Sync.usage reads both tables by revision for a follower. Co-Authored-By: Claude Fable 5.2 --- packages/harness/src/db/migrations.ts | 63 +++ packages/harness/src/db/schema.sql.ts | 43 ++ packages/harness/src/effect.ts | 2 + packages/harness/src/effect/sync.ts | 43 +- packages/harness/src/session/session.ts | 7 +- packages/harness/src/thread/thread.ts | 24 +- packages/harness/src/usage/schema.ts | 33 ++ packages/harness/src/usage/usage.ts | 244 ++++++++ .../test/__artifacts__/usage.rollup.json | 519 +++++++++++++++++ packages/harness/test/db/db.test.ts | 9 +- packages/harness/test/session.test.ts | 4 + packages/harness/test/usage.test.ts | 526 ++++++++++++++++++ 12 files changed, 1506 insertions(+), 11 deletions(-) create mode 100644 packages/harness/src/usage/schema.ts create mode 100644 packages/harness/src/usage/usage.ts create mode 100644 packages/harness/test/__artifacts__/usage.rollup.json create mode 100644 packages/harness/test/usage.test.ts diff --git a/packages/harness/src/db/migrations.ts b/packages/harness/src/db/migrations.ts index c82f383a..f776955c 100644 --- a/packages/harness/src/db/migrations.ts +++ b/packages/harness/src/db/migrations.ts @@ -1,6 +1,7 @@ import { Effect, Schema } from "effect"; import { SqlClient } from "effect/sql"; import { uuidv7 } from "uuidv7"; +import * as Usage from "../usage/usage.ts"; // Migrations ship as code (Migrator.fromRecord) rather than .sql files on // disk, so they survive bundling and need no runtime path resolution. @@ -315,6 +316,68 @@ export const migrations = { } } }), + + // The usage rollup (usage/usage.ts), backfilled from the entries already finalised. + "202611140001_usage": Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* sql` + CREATE TABLE usage ( + hour INTEGER NOT NULL, + thread_id TEXT NOT NULL, + provider TEXT NOT NULL, + model TEXT NOT NULL, + thinking TEXT NOT NULL, + session_id TEXT NOT NULL, + project_id TEXT NOT NULL, + requests INTEGER NOT NULL, + input INTEGER NOT NULL, + output INTEGER NOT NULL, + cache_read INTEGER NOT NULL, + cache_write INTEGER NOT NULL, + reasoning INTEGER NOT NULL, + cost REAL NOT NULL, + cost_input REAL NOT NULL, + cost_output REAL NOT NULL, + cost_cache_read REAL NOT NULL, + cost_cache_write REAL NOT NULL, + duration REAL NOT NULL, + timed INTEGER NOT NULL, + durations TEXT NOT NULL, + rev INTEGER NOT NULL, + PRIMARY KEY (hour, thread_id, provider, model, thinking) + ) + `; + yield* sql` + CREATE TABLE tally ( + hour INTEGER NOT NULL, + thread_id TEXT NOT NULL, + kind TEXT NOT NULL, + name TEXT NOT NULL, + session_id TEXT NOT NULL, + project_id TEXT NOT NULL, + count INTEGER NOT NULL, + rev INTEGER NOT NULL, + PRIMARY KEY (hour, thread_id, kind, name) + ) + `; + yield* sql`CREATE INDEX usage_rev_idx ON usage (rev)`; + yield* sql`CREATE INDEX tally_rev_idx ON tally (rev)`; + for (const table of ["usage", "tally"]) { + for (const event of ["insert", "update"]) { + yield* sql.unsafe(` + CREATE TRIGGER ${table}_rev_${event} BEFORE ${event.toUpperCase()} ON ${table} + WHEN NEW.rev <> (SELECT rev FROM store) + BEGIN SELECT RAISE(ABORT, 'rev is not the current revision'); END + `); + } + } + yield* Usage.backfill( + sql<{ readonly rev: number }>`UPDATE store SET rev = rev + 1 RETURNING rev`.pipe( + Effect.map(([row]) => row!.rev), + Effect.orDie, + ), + ); + }), }; /** diff --git a/packages/harness/src/db/schema.sql.ts b/packages/harness/src/db/schema.sql.ts index 008d960a..736eb8ec 100644 --- a/packages/harness/src/db/schema.sql.ts +++ b/packages/harness/src/db/schema.sql.ts @@ -9,6 +9,7 @@ import * as DocumentSchema from "../document/schema.ts"; import * as SessionSchema from "../session/schema.ts"; import * as ThreadSchema from "../thread/schema.ts"; import { SpaceSchema } from "../space/schema.ts"; +import * as UsageSchema from "../usage/schema.ts"; // Column names derive from field names via the client's camelToSnake @@ -222,6 +223,48 @@ export class EntryPartRow extends Model.Class("EntryPartRow")({ ...Timestamps, }) {} +// Usage of one thread's finalised assistant entries per UTC hour, provider, model and thinking level. +// No foreign keys: the rows outlive their session, so its session and project are copied in. +export class UsageRow extends Model.Class("UsageRow")({ + hour: Schema.Int, + threadId: ThreadSchema.IDFromDb, + provider: Schema.String, + model: Schema.String, + thinking: Schema.String, + sessionId: SessionSchema.IDFromDb, + projectId: Schema.String, + requests: NonNegativeInt, + input: NonNegativeInt, + output: NonNegativeInt, + cacheRead: NonNegativeInt, + cacheWrite: NonNegativeInt, + reasoning: NonNegativeInt, + cost: Schema.Finite, + costInput: Schema.Finite, + costOutput: Schema.Finite, + costCacheRead: Schema.Finite, + costCacheWrite: Schema.Finite, + // `timed` counts the committed requests with a usable duration; aborted ones are billed, never timed. + // `duration` sums theirs in ms; `durations` counts them per `UsageSchema.edges` bucket. + duration: Schema.Finite, + timed: NonNegativeInt, + durations: UsageSchema.Durations, + rev: Rev, +}) {} + +// Small counters of one thread per UTC hour: committed prompts, and tool calls that ran (completed or +// error) by tool name. +export class TallyRow extends Model.Class("TallyRow")({ + hour: Schema.Int, + threadId: ThreadSchema.IDFromDb, + kind: Schema.Literals(UsageSchema.tallyKinds), + name: Schema.String, + sessionId: SessionSchema.IDFromDb, + projectId: Schema.String, + count: NonNegativeInt, + rev: Rev, +}) {} + // JSON state owned by a session (no thread) or a thread. Every change is a revision. export class DocumentRow extends Model.Class("DocumentRow")({ id: DocumentSchema.ID, diff --git a/packages/harness/src/effect.ts b/packages/harness/src/effect.ts index 94d6d3e6..4887a5cb 100644 --- a/packages/harness/src/effect.ts +++ b/packages/harness/src/effect.ts @@ -15,6 +15,8 @@ export * as PromptSchema from "./session/prompt/schema.ts"; export * as SessionStore from "./session/session.ts"; export * as ThreadStore from "./thread/thread.ts"; export * as ThreadSchema from "./thread/schema.ts"; +export * as Usage from "./usage/usage.ts"; +export * as UsageSchema from "./usage/schema.ts"; export * as DocumentSchema from "./document/schema.ts"; export * as Runner from "./runner/run.ts"; export * as Git from "./git/git.ts"; diff --git a/packages/harness/src/effect/sync.ts b/packages/harness/src/effect/sync.ts index a7457d1b..57d77824 100644 --- a/packages/harness/src/effect/sync.ts +++ b/packages/harness/src/effect/sync.ts @@ -1,6 +1,6 @@ import { DateTime, Effect, Option, Schema } from "effect"; import { SqlClient, SqlSchema, type Statement } from "effect/sql"; -import { InboxRow, ProjectRow, SpaceRow } from "../db/schema.sql.ts"; +import { InboxRow, ProjectRow, SpaceRow, TallyRow, UsageRow } from "../db/schema.sql.ts"; import * as Inbox from "../inbox/inbox.ts"; import type { Admitted } from "../inbox/schema.ts"; import type { ProjectSchema } from "../project/schema.ts"; @@ -171,6 +171,47 @@ export const index = Effect.fn("Sync.index")(function* (from: Cursor, upto: numb .pipe(Effect.orDie); }); +/** A row of the usage rollup at the revision that last wrote it: running totals, not a delta. */ +export type UsageChange = + | { readonly _tag: "Usage"; readonly usage: UsageRow } + | { readonly _tag: "Tally"; readonly tally: TallyRow }; + +/** One page of the `usage` and `tally` rows past `from`, up to revision `upto`, read as one snapshot. */ +export const usage = Effect.fn("Sync.usage")(function* (from: Cursor, upto: number, limit: number) { + const sql = yield* SqlClient.SqlClient; + const findUsage = SqlSchema.findOne({ + Request: Schema.String, + Result: UsageRow, + execute: (id) => sql`SELECT * FROM usage WHERE rowid = ${Number(id)}`, + }); + const findTally = SqlSchema.findOne({ + Request: Schema.String, + Result: TallyRow, + execute: (id) => sql`SELECT * FROM tally WHERE rowid = ${Number(id)}`, + }); + // The rows are keyed by several columns, so their rowid names them in the cursor: an upsert keeps + // it, nothing deletes them, and nothing vacuums the database. + const select = (range: Statement.Fragment) => sql` + SELECT 'tally' AS kind, CAST(rowid AS TEXT) AS id, rev FROM tally WHERE ${range} + UNION ALL SELECT 'usage', CAST(rowid AS TEXT), rev FROM usage WHERE ${range} + `; + const change = Effect.fnUntraced(function* ({ kind, id }: Row) { + if (kind === "usage") return { _tag: "Usage", usage: yield* findUsage(id) } satisfies UsageChange; + return { _tag: "Tally", tally: yield* findTally(id) } satisfies UsageChange; + }); + return yield* sql + .withTransaction( + Effect.gen(function* () { + const { items, next } = yield* page(select, from, upto, limit); + return { + items: yield* Effect.forEach(items, change), + ...(next === undefined ? {} : { next }), + } satisfies Page; + }), + ) + .pipe(Effect.orDie); +}); + export type InboxState = "pending" | "promoted" | "cancelled"; /** A row of one thread's detail at the revision that last wrote it. */ diff --git a/packages/harness/src/session/session.ts b/packages/harness/src/session/session.ts index 27b4b310..adee0eb9 100644 --- a/packages/harness/src/session/session.ts +++ b/packages/harness/src/session/session.ts @@ -27,6 +27,7 @@ import { SpaceSchema } from "../space/schema.ts"; import * as Space from "../space/space.ts"; import * as ThreadSchema from "../thread/schema.ts"; import * as Thread from "../thread/thread.ts"; +import * as Usage from "../usage/usage.ts"; import { posix } from "../util/posix.ts"; import * as Slug from "../util/slug.ts"; import * as SessionSchema from "./schema.ts"; @@ -486,4 +487,8 @@ export const layer = Layer.effect( ); /** The session store with the thread and document stores it is built on. */ -export const storeLayer = layer.pipe(Layer.provideMerge(Thread.layer), Layer.provideMerge(Document.layer)); +export const storeLayer = layer.pipe( + Layer.provideMerge(Thread.layer), + Layer.provideMerge(Usage.layer), + Layer.provideMerge(Document.layer), +); diff --git a/packages/harness/src/thread/thread.ts b/packages/harness/src/thread/thread.ts index b8935867..6d6f8beb 100644 --- a/packages/harness/src/thread/thread.ts +++ b/packages/harness/src/thread/thread.ts @@ -18,6 +18,7 @@ import * as DocumentSchema from "../document/schema.ts"; import * as Event from "../event/event.ts"; import * as EventList from "../event/list.ts"; import * as SessionSchema from "../session/schema.ts"; +import * as Usage from "../usage/usage.ts"; import * as ThreadSchema from "./schema.ts"; /** A host interruption landed on a run the user had asked to stop; see `projectRun`. */ @@ -295,6 +296,7 @@ export const layer = Layer.effect( const store = yield* Store.Service; const events = yield* Event.Service; const documents = yield* Document.Service; + const usage = yield* Usage.Service; const findThread = SqlSchema.findOneOption({ Request: Schema.String, @@ -628,13 +630,16 @@ export const layer = Layer.effect( // The envelope is authoritative; aggregate deltas are never accepted // independently of the data persisted by this transaction. - const usage = + const billed = input.type === "assistant" ? yield* decodeEnvelopeUsage(input.data).pipe( Effect.map((envelope) => envelope.usage), Effect.mapError((error) => invalid(error.message)), ) : undefined; + // What the rollup reads once the entry is finalised, rejected at every write rather than inside + // the commit or abort, where a bad draft would make its turn impossible to end. + yield* Usage.validate(input.type, input.data).pipe(Effect.mapError((error) => invalid(error.message))); const result: AppendTxResult = yield* store .write( @@ -675,16 +680,17 @@ export const layer = Layer.effect( yield* writeParts(input.id, input.threadId, parts); yield* addUsage( thread.value, - usage === undefined + billed === undefined ? undefined : { - cost: usage.cost.total, - input: usage.input, - output: usage.output, - cacheRead: usage.cacheRead, - cacheWrite: usage.cacheWrite, + cost: billed.cost.total, + input: billed.input, + output: billed.output, + cacheRead: billed.cacheRead, + cacheWrite: billed.cacheWrite, }, ); + yield* usage.record(input.id); const stored = yield* findEntry(input.id); if (Option.isNone(stored)) { @@ -733,6 +739,7 @@ export const layer = Layer.effect( }); const replaceAssistant = Effect.fn("Thread.replaceAssistant")(function* (input: ReplaceAssistant) { + yield* Usage.validate("assistant", input.data); yield* store.write( Effect.gen(function* () { const { entry, thread } = yield* requireDraft(input.id, input.threadId, "replaceAssistant"); @@ -823,12 +830,14 @@ export const layer = Layer.effect( RETURNING id `; if (rows.length !== 1) return yield* Effect.die(`commit requires a fully-settled draft ${input.id}`); + yield* usage.record(input.id); }, store.write, Effect.orDie, ); const abort = Effect.fn("Thread.abort")(function* (input: ReplaceAssistant & EndAssistant) { + yield* Usage.validate("assistant", input.data); yield* store.write( Effect.gen(function* () { const { entry, thread } = yield* requireDraft(input.id, input.threadId, "abort"); @@ -843,6 +852,7 @@ export const layer = Layer.effect( WHERE id = ${input.id} AND state = 'draft' `; yield* addUsage(thread, delta); + yield* usage.record(input.id); }), ); }, Effect.orDie); diff --git a/packages/harness/src/usage/schema.ts b/packages/harness/src/usage/schema.ts new file mode 100644 index 00000000..f7ebd3e8 --- /dev/null +++ b/packages/harness/src/usage/schema.ts @@ -0,0 +1,33 @@ +import { Schema } from "effect"; +import { NonNegativeInt, optional } from "../schema.ts"; +import * as SessionSchema from "../session/schema.ts"; + +/** What the rollup reads of a finalised assistant envelope. `reasoning` is a subset of `output`. */ +export const Assistant = Schema.fromJsonString( + Schema.Struct({ + provider: Schema.Struct({ id: Schema.String }), + model: Schema.String, + thinkingLevel: Schema.String, + usage: Schema.Struct({ ...SessionSchema.Usage.fields, reasoning: optional(NonNegativeInt) }), + time: Schema.Struct({ created: Schema.Finite, completed: optional(Schema.Finite) }), + }), +); + +/** What the rollup reads of a committed user envelope. */ +export const User = Schema.fromJsonString(Schema.Struct({ time: Schema.Struct({ created: Schema.Finite }) })); + +/** + * Upper bounds in ms of the request duration histogram, log-spaced from 250 ms to 30 min: bucket `i` + * counts durations below `edges[i]` and at or above `edges[i - 1]`; one more bucket past the last edge + * is open-ended. Medians and percentiles read from it are accurate to a bucket. Persisted counts are + * positional, so the edges are frozen. + */ +export const edges = [ + 250, 374, 561, 839, 1257, 1882, 2818, 4220, 6318, 9461, 14167, 21213, 31764, 47563, 71221, 106645, 159688, 239115, + 358046, 536133, 802797, 1202096, 1800000, +] as const; + +export const Durations = Schema.fromJsonString(Schema.Array(NonNegativeInt)); + +export const tallyKinds = ["prompt", "tool"] as const; +export type TallyKind = (typeof tallyKinds)[number]; diff --git a/packages/harness/src/usage/usage.ts b/packages/harness/src/usage/usage.ts new file mode 100644 index 00000000..9e1456d0 --- /dev/null +++ b/packages/harness/src/usage/usage.ts @@ -0,0 +1,244 @@ +import { Context, Effect, Layer, Schema } from "effect"; +import { SqlClient } from "effect/sql"; +import type { ToolStatus } from "../db/schema.sql.ts"; +import * as Store from "../db/store.ts"; +import * as UsageSchema from "./schema.ts"; + +const HOUR = 3_600_000; +const hourOf = (time: number) => Math.floor(time / HOUR) * HOUR; + +/** The histogram bucket of a duration in ms; see `UsageSchema.edges`. */ +const bucketOf = (duration: number) => { + const index = UsageSchema.edges.findIndex((edge) => duration < edge); + return index === -1 ? UsageSchema.edges.length : index; +}; + +/** What the rollup reads of an entry. */ +export interface Entry { + readonly type: string; + readonly state: string; + readonly data: string; +} + +export interface UsageDelta { + readonly hour: number; + readonly provider: string; + readonly model: string; + readonly thinking: string; + readonly input: number; + readonly output: number; + readonly cacheRead: number; + readonly cacheWrite: number; + readonly reasoning: number; + readonly cost: number; + readonly costInput: number; + readonly costOutput: number; + readonly costCacheRead: number; + readonly costCacheWrite: number; + /** Absent unless the request committed with a usable duration. */ + readonly duration?: number; +} + +export interface TallyDelta { + readonly hour: number; + readonly kind: UsageSchema.TallyKind; + readonly name: string; + readonly count: number; +} + +/** A toolCall part of an entry. */ +export interface Call { + readonly name: string; + readonly status: ToolStatus; +} + +/** A call that ran and gave the model a result; skipped and aborted ones may never have started. */ +const ran = (call: Call) => call.status === "completed" || call.status === "error"; + +/** The rows one finalised entry adds to; empty for any other entry. */ +export interface Delta { + readonly usage?: UsageDelta; + readonly tallies: ReadonlyArray; +} + +const decodeAssistant = Schema.decodeUnknownEffect(UsageSchema.Assistant); +const decodeUser = Schema.decodeUnknownEffect(UsageSchema.User); +const decodeDurations = Schema.decodeUnknownEffect(UsageSchema.Durations); +const encodeDurations = Schema.encodeEffect(UsageSchema.Durations); + +/** Rejects an envelope the rollup could not read once its entry is finalised. */ +export const validate = Effect.fn("Usage.validate")(function* (type: string, data: string) { + if (type === "assistant") yield* decodeAssistant(data); + if (type === "user") yield* decodeUser(data); +}); + +/** + * The deltas of an entry with its toolCall parts. Only terminal assistant entries (committed or + * aborted: an aborted request is billed too) and committed user entries count, so an entry counts + * once, at the transition that finalises it. Only a committed request is timed: an aborted one's + * `completed` is when it was interrupted, failed or healed after a crash, not when it answered. + */ +export const delta = Effect.fn("Usage.delta")(function* (entry: Entry, calls: ReadonlyArray) { + const none: Delta = { tallies: [] }; + if (entry.type === "user" && entry.state === "committed") { + const envelope = yield* decodeUser(entry.data); + const prompt: Delta = { tallies: [{ hour: hourOf(envelope.time.created), kind: "prompt", name: "", count: 1 }] }; + return prompt; + } + if (entry.type !== "assistant" || (entry.state !== "committed" && entry.state !== "aborted")) { + return none; + } + const { provider, model, thinkingLevel, usage, time } = yield* decodeAssistant(entry.data); + const hour = hourOf(time.created); + const elapsed = entry.state !== "committed" || time.completed === undefined ? -1 : time.completed - time.created; + const counts = new Map(); + for (const call of calls.filter(ran)) counts.set(call.name, (counts.get(call.name) ?? 0) + 1); + const rolled: Delta = { + usage: { + hour, + provider: provider.id, + model, + thinking: thinkingLevel, + input: usage.input, + output: usage.output, + cacheRead: usage.cacheRead, + cacheWrite: usage.cacheWrite, + reasoning: usage.reasoning ?? 0, + cost: usage.cost.total, + costInput: usage.cost.input, + costOutput: usage.cost.output, + costCacheRead: usage.cost.cacheRead, + costCacheWrite: usage.cost.cacheWrite, + ...(elapsed >= 0 ? { duration: elapsed } : {}), + }, + tallies: [...counts].map(([name, count]): TallyDelta => ({ hour, kind: "tool", name, count })), + }; + return rolled; +}); + +/** Adds `change` to the rows of `threadId`, stamped `rev`. */ +const apply = Effect.fnUntraced(function* (threadId: string, rev: number, change: Delta) { + const sql = yield* SqlClient.SqlClient; + if (change.usage === undefined && change.tallies.length === 0) return; + const [owner] = yield* sql<{ readonly sessionId: string; readonly projectId: string }>` + SELECT t.session_id, sp.project_id FROM thread t + JOIN session s ON s.id = t.session_id + JOIN space sp ON sp.id = s.space_id + WHERE t.id = ${threadId} + `; + if (owner === undefined) return yield* Effect.die(new Error(`usage: thread ${threadId} is gone`)); + const usage = change.usage; + if (usage !== undefined) { + const key = sql`hour = ${usage.hour} AND thread_id = ${threadId} AND provider = ${usage.provider} + AND model = ${usage.model} AND thinking = ${usage.thinking}`; + const [stored] = yield* sql<{ readonly durations: string }>`SELECT durations FROM usage WHERE ${key}`; + const durations = [ + ...(stored === undefined + ? Array.from({ length: UsageSchema.edges.length + 1 }, () => 0) + : yield* decodeDurations(stored.durations)), + ]; + const timed = usage.duration === undefined ? 0 : 1; + if (usage.duration !== undefined) durations[bucketOf(usage.duration)]! += 1; + // Session and project are the row's first write's: a later relink never moves earlier usage. + yield* sql` + INSERT INTO usage ( + hour, thread_id, provider, model, thinking, session_id, project_id, requests, input, output, + cache_read, cache_write, reasoning, cost, cost_input, cost_output, cost_cache_read, cost_cache_write, + duration, timed, durations, rev + ) VALUES ( + ${usage.hour}, ${threadId}, ${usage.provider}, ${usage.model}, ${usage.thinking}, ${owner.sessionId}, + ${owner.projectId}, 1, ${usage.input}, ${usage.output}, ${usage.cacheRead}, ${usage.cacheWrite}, + ${usage.reasoning}, ${usage.cost}, ${usage.costInput}, ${usage.costOutput}, ${usage.costCacheRead}, + ${usage.costCacheWrite}, ${usage.duration ?? 0}, ${timed}, ${yield* encodeDurations(durations)}, ${rev} + ) + ON CONFLICT (hour, thread_id, provider, model, thinking) DO UPDATE SET + requests = requests + excluded.requests, + input = input + excluded.input, + output = output + excluded.output, + cache_read = cache_read + excluded.cache_read, + cache_write = cache_write + excluded.cache_write, + reasoning = reasoning + excluded.reasoning, + cost = cost + excluded.cost, + cost_input = cost_input + excluded.cost_input, + cost_output = cost_output + excluded.cost_output, + cost_cache_read = cost_cache_read + excluded.cost_cache_read, + cost_cache_write = cost_cache_write + excluded.cost_cache_write, + duration = duration + excluded.duration, + timed = timed + excluded.timed, + durations = excluded.durations, + rev = excluded.rev + `; + } + for (const tally of change.tallies) { + yield* sql` + INSERT INTO tally (hour, thread_id, kind, name, session_id, project_id, count, rev) + VALUES (${tally.hour}, ${threadId}, ${tally.kind}, ${tally.name}, ${owner.sessionId}, ${owner.projectId}, ${tally.count}, ${rev}) + ON CONFLICT (hour, thread_id, kind, name) DO UPDATE SET + count = count + excluded.count, + rev = excluded.rev + `; + } +}); + +/** + * Rolls the stored entry `entryId` and its toolCall parts into the rows, stamped `rev`. Fails only when + * its envelope does not decode. + */ +const rollup = Effect.fnUntraced(function* (entryId: string, rev: number) { + const sql = yield* SqlClient.SqlClient; + const [entry] = yield* sql` + SELECT thread_id, type, state, data FROM entry WHERE id = ${entryId} + `.pipe(Effect.orDie); + if (entry === undefined) return yield* Effect.die(new Error(`usage: entry ${entryId} is gone`)); + const calls = yield* sql` + SELECT tool_name AS name, status FROM part WHERE entry_id = ${entryId} AND type = 'toolCall' ORDER BY part_index + `.pipe(Effect.orDie); + const change = yield* delta(entry, calls); + yield* apply(entry.threadId, rev, change).pipe(Effect.orDie); +}); + +/** + * Rolls every finalised entry into the rows, the way the write path does, at one revision taken from + * `rev` only when there is something to roll up. Expects the rows empty. An entry whose envelope does + * not decode is skipped, so one bad historical row cannot keep the database from opening. + */ +export const backfill = Effect.fnUntraced(function* (rev: Effect.Effect) { + const sql = yield* SqlClient.SqlClient; + const entries = yield* sql<{ readonly id: string }>` + SELECT id FROM entry + WHERE (type = 'assistant' AND state IN ('committed', 'aborted')) OR (type = 'user' AND state = 'committed') + ORDER BY seq, id + `.pipe(Effect.orDie); + if (entries.length === 0) return; + const taken = yield* rev; + let skipped = 0; + for (const entry of entries) { + yield* rollup(entry.id, taken).pipe(Effect.catchTag("SchemaError", () => Effect.sync(() => skipped++))); + } + if (skipped > 0) + yield* Effect.logWarning(`usage backfill skipped ${skipped} entries whose envelope does not decode`); +}); + +export interface Interface { + /** + * Rolls a stored entry into the rows inside the enclosing `Store.write`. The write validated its + * envelope (`validate`), so one that does not decode is a defect. + */ + readonly record: (entryId: string) => Effect.Effect; +} + +export class Service extends Context.Service()("@codeworksh/harness/usage/usage/Service") {} + +export const layer = Layer.effect( + Service, + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const store = yield* Store.Service; + + const record = Effect.fn("Usage.record")(function* (entryId: string) { + yield* rollup(entryId, yield* store.rev).pipe(Effect.orDie, Effect.provideService(SqlClient.SqlClient, sql)); + }); + + return Service.of({ record }); + }), +); diff --git a/packages/harness/test/__artifacts__/usage.rollup.json b/packages/harness/test/__artifacts__/usage.rollup.json new file mode 100644 index 00000000..6141443a --- /dev/null +++ b/packages/harness/test/__artifacts__/usage.rollup.json @@ -0,0 +1,519 @@ +{ + "usage": [ + { + "hour": "2026-11-14T09:00:00.000Z", + "threadId": "main", + "provider": "openai", + "model": "gpt-5.6-luna", + "thinking": "medium", + "sessionId": "S1", + "projectId": "P1", + "requests": 1, + "input": 100, + "output": 40, + "cacheRead": 20, + "cacheWrite": 5, + "reasoning": 10, + "cost": 0.5, + "costInput": 0.25, + "costOutput": 0.125, + "costCacheRead": 0.0625, + "costCacheWrite": 0.0625, + "duration": 2000, + "timed": 1, + "durations": [ + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + }, + { + "hour": "2026-11-14T10:00:00.000Z", + "threadId": "main", + "provider": "openai", + "model": "gpt-5.6-luna", + "thinking": "medium", + "sessionId": "S1", + "projectId": "P1", + "requests": 1, + "input": 7, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0, + "reasoning": 0, + "cost": 0, + "costInput": 0, + "costOutput": 0, + "costCacheRead": 0, + "costCacheWrite": 0, + "duration": 0, + "timed": 0, + "durations": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + }, + { + "hour": "2026-11-14T10:00:00.000Z", + "threadId": "main", + "provider": "openai", + "model": "other-model", + "thinking": "high", + "sessionId": "S1", + "projectId": "P1", + "requests": 1, + "input": 300, + "output": 60, + "cacheRead": 0, + "cacheWrite": 0, + "reasoning": 0, + "cost": 0.75, + "costInput": 0.5, + "costOutput": 0.25, + "costCacheRead": 0, + "costCacheWrite": 0, + "duration": 0, + "timed": 0, + "durations": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + }, + { + "hour": "2026-11-14T10:00:00.000Z", + "threadId": "subagent", + "provider": "openai", + "model": "gpt-5.6-luna", + "thinking": "low", + "sessionId": "S1", + "projectId": "P1", + "requests": 1, + "input": 1, + "output": 1, + "cacheRead": 0, + "cacheWrite": 0, + "reasoning": 0, + "cost": 0, + "costInput": 0, + "costOutput": 0, + "costCacheRead": 0, + "costCacheWrite": 0, + "duration": 250, + "timed": 1, + "durations": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + }, + { + "hour": "2026-11-14T11:00:00.000Z", + "threadId": "fork", + "provider": "openai", + "model": "gpt-5.6-luna", + "thinking": "medium", + "sessionId": "S1", + "projectId": "P1", + "requests": 1, + "input": 50, + "output": 5, + "cacheRead": 0, + "cacheWrite": 0, + "reasoning": 0, + "cost": 0.25, + "costInput": 0.125, + "costOutput": 0.125, + "costCacheRead": 0, + "costCacheWrite": 0, + "duration": 90000, + "timed": 1, + "durations": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + }, + { + "hour": "2026-11-14T12:00:00.000Z", + "threadId": "healed", + "provider": "openai", + "model": "gpt-5.6-luna", + "thinking": "medium", + "sessionId": "S1", + "projectId": "P1", + "requests": 1, + "input": 11, + "output": 3, + "cacheRead": 0, + "cacheWrite": 0, + "reasoning": 0, + "cost": 0, + "costInput": 0, + "costOutput": 0, + "costCacheRead": 0, + "costCacheWrite": 0, + "duration": 0, + "timed": 0, + "durations": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + }, + { + "hour": "2026-11-14T13:00:00.000Z", + "threadId": "recovered", + "provider": "openai", + "model": "gpt-5.6-luna", + "thinking": "medium", + "sessionId": "S1", + "projectId": "P1", + "requests": 1, + "input": 1, + "output": 1, + "cacheRead": 0, + "cacheWrite": 0, + "reasoning": 0, + "cost": 0, + "costInput": 0, + "costOutput": 0, + "costCacheRead": 0, + "costCacheWrite": 0, + "duration": 2000, + "timed": 1, + "durations": [ + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + }, + { + "hour": "2026-11-14T20:00:00.000Z", + "threadId": "healed", + "provider": "openai", + "model": "gpt-5.6-luna", + "thinking": "medium", + "sessionId": "S1", + "projectId": "P1", + "requests": 1, + "input": 1, + "output": 1, + "cacheRead": 0, + "cacheWrite": 0, + "reasoning": 0, + "cost": 0, + "costInput": 0, + "costOutput": 0, + "costCacheRead": 0, + "costCacheWrite": 0, + "duration": 2000, + "timed": 1, + "durations": [ + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + }, + { + "hour": "2026-11-14T21:00:00.000Z", + "threadId": "recovered", + "provider": "openai", + "model": "gpt-5.6-luna", + "thinking": "medium", + "sessionId": "S1", + "projectId": "P1", + "requests": 1, + "input": 1, + "output": 1, + "cacheRead": 0, + "cacheWrite": 0, + "reasoning": 0, + "cost": 0, + "costInput": 0, + "costOutput": 0, + "costCacheRead": 0, + "costCacheWrite": 0, + "duration": 500, + "timed": 1, + "durations": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + } + ], + "tally": [ + { + "hour": "2026-11-14T09:00:00.000Z", + "threadId": "main", + "kind": "prompt", + "name": "", + "sessionId": "S1", + "projectId": "P1", + "count": 1 + }, + { + "hour": "2026-11-14T09:00:00.000Z", + "threadId": "main", + "kind": "tool", + "name": "echo", + "sessionId": "S1", + "projectId": "P1", + "count": 2 + }, + { + "hour": "2026-11-14T09:00:00.000Z", + "threadId": "main", + "kind": "tool", + "name": "shout", + "sessionId": "S1", + "projectId": "P1", + "count": 1 + }, + { + "hour": "2026-11-14T10:00:00.000Z", + "threadId": "main", + "kind": "prompt", + "name": "", + "sessionId": "S1", + "projectId": "P1", + "count": 1 + }, + { + "hour": "2026-11-14T10:00:00.000Z", + "threadId": "subagent", + "kind": "prompt", + "name": "", + "sessionId": "S1", + "projectId": "P1", + "count": 1 + }, + { + "hour": "2026-11-14T11:00:00.000Z", + "threadId": "fork", + "kind": "prompt", + "name": "", + "sessionId": "S1", + "projectId": "P1", + "count": 1 + }, + { + "hour": "2026-11-14T13:00:00.000Z", + "threadId": "recovered", + "kind": "tool", + "name": "echo", + "sessionId": "S1", + "projectId": "P1", + "count": 1 + }, + { + "hour": "2026-11-14T20:00:00.000Z", + "threadId": "healed", + "kind": "prompt", + "name": "", + "sessionId": "S1", + "projectId": "P1", + "count": 1 + }, + { + "hour": "2026-11-14T21:00:00.000Z", + "threadId": "recovered", + "kind": "prompt", + "name": "", + "sessionId": "S1", + "projectId": "P1", + "count": 1 + } + ] +} diff --git a/packages/harness/test/db/db.test.ts b/packages/harness/test/db/db.test.ts index b7d52e2d..4650bb32 100644 --- a/packages/harness/test/db/db.test.ts +++ b/packages/harness/test/db/db.test.ts @@ -85,13 +85,16 @@ const registerInstances = (...ids: ReadonlyArray) => describe("Database", () => { describe("migrations", () => { memoryIt( - "applies the one threads migration and creates its tables", + "applies the migrations and creates their tables", Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; const applied = yield* sql<{ migrationId: number; name: string }>` SELECT migration_id, name FROM effect_sql_migrations ORDER BY migration_id `; - expect(applied.map((row) => `${row.migrationId}_${row.name}`)).toEqual(["202610290001_init"]); + expect(applied.map((row) => `${row.migrationId}_${row.name}`)).toEqual([ + "202610290001_init", + "202611140001_usage", + ]); const tables = yield* sql<{ name: string }>` SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' AND name <> 'effect_sql_migrations' @@ -111,7 +114,9 @@ describe("Database", () => { "session", "space", "store", + "tally", "thread", + "usage", ]); }), ); diff --git a/packages/harness/test/session.test.ts b/packages/harness/test/session.test.ts index 771c690d..4a466558 100644 --- a/packages/harness/test/session.test.ts +++ b/packages/harness/test/session.test.ts @@ -75,8 +75,12 @@ const assistantEntry = ( data: JSON.stringify({ messageId: id, role: "assistant", + provider: { id: "test" }, + model: "test", + thinkingLevel: "off", stopReason: "stop", usage: options?.usage ?? usage(), + time: { created: 1, completed: 1 }, }), parts: [{ type: "text", data: JSON.stringify({ type: "text", text: "ok" }) }], }); diff --git a/packages/harness/test/usage.test.ts b/packages/harness/test/usage.test.ts new file mode 100644 index 00000000..ffa41096 --- /dev/null +++ b/packages/harness/test/usage.test.ts @@ -0,0 +1,526 @@ +import "./utils/env.ts"; +import { createAssistantMessageEventStream, type Message } from "@codeworksh/aikit"; +import { DateTime, Effect, Layer, Schema } from "effect"; +import { SqlClient } from "effect/sql"; +import * as TestClock from "effect/testing/TestClock"; +import { describe, expect } from "vite-plus/test"; +import * as Context from "../src/context/context.ts"; +import * as Control from "../src/control.ts"; +import * as Database from "../src/db/db.ts"; +import { migrations } from "../src/db/migrations.ts"; +import type { TallyRow, UsageRow } from "../src/db/schema.sql.ts"; +import * as Store from "../src/db/store.ts"; +import * as Sync from "../src/effect/sync.ts"; +import * as Event from "../src/event/event.ts"; +import * as EventList from "../src/event/list.ts"; +import * as EventRegistry from "../src/event/registry.ts"; +import * as Global from "../src/global.ts"; +import { builtins } from "../src/plugin/builtin.ts"; +import type { Plugin } from "../src/plugin/plugin.ts"; +import * as RunnerExecute from "../src/runner/execute.ts"; +import * as Hooks from "../src/runner/hooks.ts"; +import * as LLM from "../src/runner/llm.ts"; +import * as Loop from "../src/runner/loop.ts"; +import * as SandboxController from "../src/sandbox/control.ts"; +import * as SandboxDriverRegistry from "../src/sandbox/registry.ts"; +import * as SessionMessageSchema from "../src/session/message/schema.ts"; +import * as PromptSchema from "../src/session/prompt/schema.ts"; +import * as SessionProjector from "../src/session/projector.ts"; +import * as SessionRuntime from "../src/session/runtime.ts"; +import * as SessionSchema from "../src/session/schema.ts"; +import * as Session from "../src/session/session.ts"; +import * as Settings from "../src/settings/settings.ts"; +import * as State from "../src/state/state.ts"; +import * as ThreadSchema from "../src/thread/schema.ts"; +import * as Thread from "../src/thread/thread.ts"; +import * as Tool from "../src/tool/tool.ts"; +import * as Usage from "../src/usage/usage.ts"; +import * as Mcp from "../src/mcp/mcp.ts"; +import * as Process from "../src/sandbox/utils/process.ts"; +import { assistant } from "./fixtures/llm.ts"; +import { credentials, named } from "./fixtures/model.ts"; +import { pooled } from "./fixtures/pool.ts"; +import { seedSpace } from "./fixtures/space.ts"; +import { testEffect } from "./utils/effect.ts"; + +/** One model request: its times, usage, what it calls, and whether the provider fails it. */ +interface Step { + readonly created: number; + readonly completed: number; + readonly tools?: ReadonlyArray; + readonly model?: string; + readonly thinking?: Message.AssistantMessage["thinkingLevel"]; + readonly usage?: Partial; + readonly fail?: boolean; +} + +const HOUR = 3_600_000; +// 2026-11-14T10:00:00Z: the scenario straddles it. +const H = Date.UTC(2026, 10, 14, 10); + +const script = new Map>(); +let responses = 0; +/** The last request's input, which a crashed draft is built from. */ +let requested: LLM.Input | undefined; + +const open: LLM.Open = (input) => + Effect.sync(() => { + requested = input; + const step = script.get(input.threadId)?.shift(); + if (step === undefined) throw new Error(`no step scripted for ${input.threadId}`); + responses += 1; + const base = assistant(input, responses).usage; + const message = assistant(input, responses, { + ...(step.model === undefined ? {} : { model: step.model }), + ...(step.thinking === undefined ? {} : { thinkingLevel: step.thinking }), + usage: { ...base, ...step.usage }, + time: { created: step.created, completed: step.completed }, + ...(step.fail === true + ? { stopReason: "error", errorMessage: "provider failed" } + : step.tools === undefined + ? {} + : { + stopReason: "toolUse", + parts: step.tools.map((name, index) => ({ + type: "toolCall", + callID: `call_${responses}_${index}`, + name, + arguments: { value: name }, + status: "pending", + time: { start: step.created, end: step.created }, + })), + }), + }); + const stream = createAssistantMessageEventStream(); + stream.push({ type: "start", partial: message }); + if (step.fail === true) { + const failure = { _tag: "Authentication", reason: "missing", message: "provider failed", retryable: false }; + stream.push({ type: "error", reason: "error", error: { ...message, failure } as Message.AssistantMessage }); + } else stream.push({ type: "done", reason: step.tools === undefined ? "stop" : "toolUse", message }); + return stream; + }); + +const echoTools: Plugin = { + id: "test.tool.echo", + kind: "tool", + setup: (ctx) => { + for (const name of ["echo", "shout"]) { + ctx.plugin.tools.add( + Tool.register( + Tool.make({ + name, + description: `${name}es its value`, + parameters: Schema.Struct({ value: Schema.String }), + success: Schema.String, + encodeContent: (value: string) => [{ type: "text", text: value }], + handler: ({ value }) => Effect.succeed(value), + }), + ), + ); + } + }, +}; + +const layer = () => { + const database = Database.layer(":memory:"); + const sandbox = SandboxController.layer({ hostCwd: "/" }).pipe( + Layer.provideMerge(SandboxDriverRegistry.layer()), + Layer.provideMerge(database), + ); + const seeded = pooled([...builtins, echoTools]); + return Control.layer.pipe( + Layer.provideMerge( + RunnerExecute.layer.pipe( + Layer.provide(Loop.layer({ request: LLM.make(open) }).pipe(Layer.provide(Hooks.layer))), + ), + ), + Layer.provideMerge(State.layer({}, seeded.ref, seeded.references, seeded.follow, seeded.rebuild)), + Layer.provideMerge(credentials), + Layer.provideMerge(Mcp.layer({ hostCwd: "/", home: "/unused/home" }).pipe(Layer.provide(Process.host))), + Layer.provideMerge(SessionRuntime.layer), + Layer.provideMerge( + Layer.succeed(Settings.Service, { + load: () => Effect.succeed(named), + locate: () => Effect.die("unused"), + }), + ), + Layer.provideMerge(sandbox), + Layer.provideMerge(Context.layer), + Layer.provideMerge(SessionProjector.layer), + Layer.provideMerge(Session.storeLayer), + Layer.provideMerge(Event.layer), + Layer.provideMerge(EventRegistry.layer()), + Layer.provideMerge(database), + Layer.provideMerge(Layer.succeed(Global.Service, Global.make({ home: "/unused/home" }))), + ); +}; + +const { effect: it } = testEffect(layer()); + +const artifact = (name: string, value: unknown) => + Effect.promise(() => + expect(`${JSON.stringify(value, null, "\t")}\n`).toMatchFileSnapshot(`./__artifacts__/${name}`), + ); + +/** Prompts `threadId` at clock time `at` and waits for the run, which may fail. */ +const run = (sessionId: SessionSchema.ID, threadId: ThreadSchema.ID, at: number, text: string) => + Effect.gen(function* () { + const control = yield* Control.Service; + yield* TestClock.setTime(at); + yield* control + .run({ sessionId, threadId, prompt: PromptSchema.Prompt.fromText(text), delivery: "followUp" }) + .pipe(Effect.exit); + yield* control.wait(threadId); + }); + +/** + * What a host that died mid-turn leaves for the next run to heal: the draft it was writing, ended when + * its response finished, with the calls in `ran` settled as completed, and the run interrupted. + */ +const crashed = Effect.fnUntraced(function* ( + sessionId: SessionSchema.ID, + threadId: ThreadSchema.ID, + draft: Message.AssistantMessage, + ran: ReadonlyArray, +) { + const events = yield* Event.Service; + const ids = { sessionId, threadId, messageId: SessionMessageSchema.ID.from(draft.messageId) }; + const at = DateTime.makeUnsafe(draft.time.completed); + yield* events.publish(EventList.LLMStarted, { timestamp: at, ...ids, message: draft }); + if (draft.stopReason === "toolUse") { + yield* events.publish(EventList.LLMEnded, { timestamp: at, ...ids, reason: "toolUse", message: draft }); + } + for (const part of draft.parts) { + if (part.type !== "toolCall" || part.status !== "pending" || !ran.includes(part.callID)) continue; + yield* events.publish(EventList.ToolSettled, { + timestamp: at, + ...ids, + callID: part.callID, + part: { ...part, status: "completed", result: { content: [{ type: "text", text: "ran" }], isError: false } }, + }); + } + yield* events.publish(EventList.ExecutionInterrupted, { timestamp: at, sessionId, threadId, reason: "crash" }); +}); + +/** + * One session: main runs a tool turn across the 10:00 boundary (two tools, two requests on different + * models and thinking levels) and a request the provider fails; a fork and a subagent thread run one + * request each. Two more threads heal what a crash left: `healed` a request in flight at 12:00, aborted + * when the host is back at 20:00; `recovered` a response that ended at 13:00 with two calls, one of which + * ran, committed at 21:00 with the other aborted unrun. + */ +const scenario = Effect.gen(function* () { + script.clear(); + responses = 0; + const sessions = yield* Session.Service; + const threads = yield* Thread.Service; + const { spaceId, location } = yield* seedSpace({ location: process.cwd(), projectId: "p" }); + const created = yield* sessions.create({ spaceId, directory: location }); + const sessionId = SessionSchema.ID.make(created.session.id); + const main = ThreadSchema.ID.make(created.thread.id); + + const cost = (input: number, output: number, cacheRead: number, cacheWrite: number) => ({ + input, + output, + cacheRead, + cacheWrite, + total: input + output + cacheRead + cacheWrite, + }); + script.set(main, [ + { + created: H - 30_000, + completed: H - 28_000, + tools: ["echo", "shout", "echo"], + usage: { + input: 100, + output: 40, + cacheRead: 20, + cacheWrite: 5, + reasoning: 10, + totalTokens: 165, + cost: cost(0.25, 0.125, 0.0625, 0.0625), + }, + }, + // Completed before it was created: billed, but no duration. + { + created: H + 1_000, + completed: H + 400, + model: "other-model", + thinking: "high", + usage: { input: 300, output: 60, totalTokens: 360, cost: cost(0.5, 0.25, 0, 0) }, + }, + { created: H + 130_000, completed: H + 135_000, fail: true, usage: { input: 7, output: 0, totalTokens: 7 } }, + ]); + yield* run(sessionId, main, H - 60_000, "first"); + yield* run(sessionId, main, H + 120_000, "second"); + + const line = yield* threads.line(main); + const second = line.filter(({ entry }) => entry.type === "assistant" && entry.state === "committed")[1]!; + const fork = ThreadSchema.ID.make((yield* threads.fork({ threadId: main, entryId: second.entry.id })).thread.id); + script.set(fork, [ + { + created: H + HOUR + 100_000, + completed: H + HOUR + 190_000, + usage: { input: 50, output: 5, totalTokens: 55, cost: cost(0.125, 0.125, 0, 0) }, + }, + ]); + yield* run(sessionId, fork, H + HOUR + 90_000, "fork"); + + const subagent = ThreadSchema.ID.create(); + yield* threads.insert({ + id: subagent, + sessionId, + origin: { kind: "subagent", threadId: main, callId: "call_1_0" }, + seq: 1, + }); + // Exactly on the first edge: the second bucket. + script.set(subagent, [{ created: H + 5_000, completed: H + 5_250, thinking: "low" }]); + yield* run(sessionId, subagent, H + 4_000, "subagent"); + + const thread = Effect.fnUntraced(function* (callId: string) { + const id = ThreadSchema.ID.create(); + yield* threads.insert({ id, sessionId, origin: { kind: "subagent", threadId: main, callId }, seq: 1 }); + return id; + }); + const input = requested!; + const healed = yield* thread("call_1_1"); + yield* crashed( + sessionId, + healed, + assistant(input, 101, { + usage: { ...assistant(input, 101).usage, input: 11, output: 3, totalTokens: 14 }, + time: { created: H + 2 * HOUR, completed: H + 2 * HOUR }, + }), + [], + ); + script.set(healed, [{ created: H + 10 * HOUR + 1_000, completed: H + 10 * HOUR + 3_000 }]); + yield* run(sessionId, healed, H + 10 * HOUR, "healed"); + + const recovered = yield* thread("call_1_2"); + const call = (callID: string, name: string): Message.ToolCallPendingPart => ({ + type: "toolCall", + callID, + name, + arguments: { value: name }, + status: "pending", + time: { start: H + 3 * HOUR, end: H + 3 * HOUR }, + }); + yield* crashed( + sessionId, + recovered, + assistant(input, 102, { + stopReason: "toolUse", + parts: [call("call_ran", "echo"), call("call_unrun", "shout")], + time: { created: H + 3 * HOUR, completed: H + 3 * HOUR + 2_000 }, + }), + ["call_ran"], + ); + script.set(recovered, [{ created: H + 11 * HOUR + 1_000, completed: H + 11 * HOUR + 1_500 }]); + yield* run(sessionId, recovered, H + 11 * HOUR, "recovered"); + + const sql = yield* SqlClient.SqlClient; + const [owner] = yield* sql<{ projectId: string }>` + SELECT sp.project_id FROM session s JOIN space sp ON sp.id = s.space_id WHERE s.id = ${sessionId} + `; + return { sessionId, projectId: owner!.projectId, threads: { main, fork, subagent, healed, recovered } }; +}); + +type Plain = Omit; +const plain = (rows: ReadonlyArray): Array> => + rows.map(({ rev: _rev, ...rest }) => rest); +const byKey = (rows: ReadonlyArray) => + [...rows].sort((left, right) => JSON.stringify(left).localeCompare(JSON.stringify(right))); + +/** Every row, read the way the sync reads them: one page wide enough for all of them. */ +const rollup = Effect.gen(function* () { + const store = yield* Store.Service; + const { items, next } = yield* Sync.usage({ rev: 0 }, (yield* store.head).rev, 10_000); + expect(next).toBeUndefined(); + return { + usage: items.flatMap((item) => (item._tag === "Usage" ? [item.usage] : [])), + tally: items.flatMap((item) => (item._tag === "Tally" ? [item.tally] : [])), + }; +}); + +/** The invariants the rollup holds against the thread rows and entries, per thread. */ +const verify = Effect.fnUntraced(function* ( + sessionId: SessionSchema.ID, + rows: { readonly usage: ReadonlyArray; readonly tally: ReadonlyArray }, +) { + const sql = yield* SqlClient.SqlClient; + const threads = yield* sql<{ + id: string; + cost: number; + tokensInput: number; + tokensOutput: number; + tokensCacheRead: number; + tokensCacheWrite: number; + }>`SELECT * FROM thread WHERE session_id = ${sessionId}`; + for (const thread of threads) { + const own = rows.usage.filter((row) => row.threadId === thread.id); + const sum = (pick: (row: UsageRow) => number) => own.reduce((total, row) => total + pick(row), 0); + expect({ + cost: sum((row) => row.cost), + input: sum((row) => row.input), + output: sum((row) => row.output), + cacheRead: sum((row) => row.cacheRead), + cacheWrite: sum((row) => row.cacheWrite), + }).toEqual({ + cost: thread.cost, + input: thread.tokensInput, + output: thread.tokensOutput, + cacheRead: thread.tokensCacheRead, + cacheWrite: thread.tokensCacheWrite, + }); + const [counts] = yield* sql<{ prompts: number; requests: number; timed: number }>` + SELECT + count(*) FILTER (WHERE type = 'user' AND state = 'committed') AS prompts, + count(*) FILTER (WHERE type = 'assistant' AND state IN ('committed', 'aborted')) AS requests, + count(*) FILTER ( + WHERE type = 'assistant' AND state = 'committed' + AND data ->> '$.time.completed' >= data ->> '$.time.created' + ) AS timed + FROM entry WHERE thread_id = ${thread.id} + `; + const tallied = (kind: string) => rows.tally.filter((row) => row.threadId === thread.id && row.kind === kind); + expect(tallied("prompt").reduce((total, row) => total + row.count, 0)).toBe(counts!.prompts); + expect(sum((row) => row.requests)).toBe(counts!.requests); + // Only committed requests are timed. + expect(sum((row) => row.timed)).toBe(counts!.timed); + const calls = yield* sql<{ toolName: string; calls: number }>` + SELECT p.tool_name, count(*) AS calls FROM part p JOIN entry e ON e.id = p.entry_id + WHERE e.thread_id = ${thread.id} AND p.type = 'toolCall' AND e.state IN ('committed', 'aborted') + AND p.status IN ('completed', 'error') + GROUP BY p.tool_name + `; + const tools = new Map(); + for (const row of tallied("tool")) tools.set(row.name, (tools.get(row.name) ?? 0) + row.count); + expect(Object.fromEntries(tools)).toEqual(Object.fromEntries(calls.map((row) => [row.toolName, row.calls]))); + for (const row of own) { + expect(row.timed).toBeLessThanOrEqual(row.requests); + expect(row.durations.reduce((total, count) => total + count, 0)).toBe(row.timed); + } + } +}); + +describe("usage rollup", () => { + it( + "rolls finalised entries into hourly rows that match the thread totals", + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const { sessionId, projectId, threads } = yield* scenario; + const rows = yield* rollup; + yield* verify(sessionId, rows); + expect( + yield* sql`SELECT state FROM entry WHERE session_id = ${sessionId} AND type = 'assistant' ORDER BY seq`, + ).toEqual( + ["committed", "committed", "aborted", "committed", "committed"] + .concat(["aborted", "committed", "committed", "committed"]) + .map((state) => ({ state })), + ); + const at = (threadId: string, hour: number) => + rows.usage.filter((row) => row.threadId === threadId && row.hour === hour); + const untimed = { timed: 0, duration: 0, durations: Array.from({ length: 24 }, () => 0) }; + // The provider failed it, and the crash healed hours later: billed, never timed. + expect(at(threads.main, H).find((row) => row.model === "gpt-5.6-luna")).toMatchObject({ + requests: 1, + input: 7, + ...untimed, + }); + expect(at(threads.healed, H + 2 * HOUR)).toMatchObject([{ requests: 1, input: 11, output: 3, ...untimed }]); + // Its response ended before the crash: timed as it streamed. + expect(at(threads.recovered, H + 3 * HOUR)).toMatchObject([{ requests: 1, timed: 1, duration: 2_000 }]); + expect( + rows.tally + .filter((row) => row.threadId === threads.recovered && row.kind === "tool") + .map((row) => [row.name, row.count]), + ).toEqual([["echo", 1]]); + const name = new Map([ + [threads.main, "main"], + [threads.fork, "fork"], + [threads.subagent, "subagent"], + [threads.healed, "healed"], + [threads.recovered, "recovered"], + [sessionId, "S1"], + [projectId, "P1"], + ]); + const normal = < + A extends { hour: number; threadId: string; sessionId: string; projectId: string; rev: number }, + >( + row: A, + ) => { + const { rev: _rev, ...rest } = row; + return { + ...rest, + hour: new Date(row.hour).toISOString(), + threadId: name.get(row.threadId), + sessionId: name.get(row.sessionId), + projectId: name.get(row.projectId), + }; + }; + // Revisions move with unrelated writes; the order they give is checked, not their values. + expect(rows.usage.map((row) => row.rev)).toEqual(rows.usage.map((row) => row.rev).sort((a, b) => a - b)); + yield* artifact("usage.rollup.json", { + usage: byKey(rows.usage.map(normal)), + tally: byKey(rows.tally.map(normal)), + }); + }), + ); + + it( + "keeps a deleted session's rows", + Effect.gen(function* () { + const control = yield* Control.Service; + const sql = yield* SqlClient.SqlClient; + const { sessionId } = yield* scenario; + const before = yield* rollup; + expect(yield* control.deleteSession(sessionId)).toBe(true); + expect(yield* sql`SELECT id FROM thread WHERE session_id = ${sessionId}`).toEqual([]); + const after = yield* rollup; + expect(plain(after.usage)).toEqual(plain(before.usage)); + expect(plain(after.tally)).toEqual(plain(before.tally)); + expect(after.usage.every((row) => row.sessionId === sessionId)).toBe(true); + }), + ); + + it( + "backfills the same rows from the entries", + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const store = yield* Store.Service; + const { sessionId } = yield* scenario; + const live = yield* rollup; + yield* sql`DELETE FROM usage`; + yield* sql`DELETE FROM tally`; + const head = (yield* store.head).rev; + yield* store.write(Usage.backfill(store.rev)); + const filled = yield* rollup; + expect(byKey(plain(filled.usage))).toEqual(byKey(plain(live.usage))); + expect(byKey(plain(filled.tally))).toEqual(byKey(plain(live.tally))); + // One revision for everything it wrote. + expect(new Set([...filled.usage, ...filled.tally].map((row) => row.rev))).toEqual(new Set([head + 1])); + yield* verify(sessionId, filled); + }), + ); + + it( + "migrates entries written before the rollup, skipping an envelope it cannot read", + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const { sessionId, threads } = yield* scenario; + const live = yield* rollup; + yield* sql`DROP TABLE usage`; + yield* sql`DROP TABLE tally`; + yield* sql`UPDATE store SET rev = rev + 1`; + yield* sql` + INSERT INTO entry (id, session_id, thread_id, seq, committed_seq, type, state, data, created_at, updated_at, rev) + VALUES ('legacy', ${sessionId}, ${threads.main}, 1000000, 1000000, 'assistant', 'committed', '{}', 0, 0, + (SELECT rev FROM store)) + `; + yield* sql.withTransaction(migrations["202611140001_usage"]); + const migrated = yield* rollup; + expect(byKey(plain(migrated.usage))).toEqual(byKey(plain(live.usage))); + expect(byKey(plain(migrated.tally))).toEqual(byKey(plain(live.tally))); + }), + ); +}); From f3ce5b7eb182f8610ceb678e3951272926b519dc Mon Sep 17 00:00:00 2001 From: Sanchit Rk <0xsanchit@gmail.com> Date: Sat, 14 Nov 2026 14:16:44 +0530 Subject: [PATCH 2/8] feat(codework): serve usage.sync and report the host's name usage.sync streams the usage rollup in revision order, then live, as running totals a client replaces by key. It is a stream of its own so a client that predates it is unaffected. server.info gains an optional host name: the macOS computer name, else the hostname. Co-Authored-By: Claude Fable 5.2 --- packages/codework/src/server/contract.ts | 76 ++++++ packages/codework/src/server/handlers.ts | 52 +++- packages/codework/src/server/identity.ts | 30 ++- packages/codework/src/server/server.ts | 10 +- .../test/__artifacts__/sync.usage.json | 241 ++++++++++++++++++ packages/codework/test/server.test.ts | 13 +- packages/codework/test/usage.test.ts | 226 ++++++++++++++++ 7 files changed, 642 insertions(+), 6 deletions(-) create mode 100644 packages/codework/test/__artifacts__/sync.usage.json create mode 100644 packages/codework/test/usage.test.ts diff --git a/packages/codework/src/server/contract.ts b/packages/codework/src/server/contract.ts index 6d71d2f4..d672ea0c 100644 --- a/packages/codework/src/server/contract.ts +++ b/packages/codework/src/server/contract.ts @@ -18,6 +18,7 @@ import { SettingsFile, ThreadSchema, ThreadStore, + UsageSchema, } from "@codeworksh/harness/effect"; import { Schema } from "effect"; import { Rpc, RpcGroup } from "effect/rpc"; @@ -177,6 +178,67 @@ export const IndexItem = Schema.Union([ ]); export type IndexItem = typeof IndexItem.Type; +/** + * Upper bounds in ms of the request duration histogram: bucket `i` of `durations` counts durations + * below `edges[i]` and at or above `edges[i - 1]`; one more bucket past the last edge is open-ended. + */ +export const edges = UsageSchema.edges; + +/** Where a `usage.sync` row belongs: a thread's UTC hour. */ +const UsageOwner = { + /** Start of the UTC hour, epoch milliseconds. */ + hour: Schema.Int, + threadId: ThreadSchema.ID, + /** The thread's session and project when the row was first written. */ + sessionId: Session.SessionSchema.ID, + projectId: Schema.String, +}; + +/** A `usage.sync` item: a rollup row with the revision that last wrote it, or progress. */ +export const UsageItem = Schema.Union([ + /** Requests of one thread in one hour to one provider, model and thinking level, aborted ones included. */ + Schema.TaggedStruct("Usage", { + ...UsageOwner, + provider: Schema.String, + model: Schema.String, + thinking: Schema.String, + requests: Schema.Int, + /** `reasoning` is part of `output`. */ + tokens: Schema.Struct({ + input: Schema.Int, + output: Schema.Int, + cacheRead: Schema.Int, + cacheWrite: Schema.Int, + reasoning: Schema.Int, + }), + /** US dollars, from the catalog's prices. */ + cost: Schema.Struct({ + total: Schema.Finite, + input: Schema.Finite, + output: Schema.Finite, + cacheRead: Schema.Finite, + cacheWrite: Schema.Finite, + }), + /** Total ms of the `timed` requests: the committed ones; aborted requests are never timed. */ + duration: Schema.Finite, + timed: Schema.Int, + /** The `timed` requests per bucket of `edges`, one more than there are edges. */ + durations: Schema.Array(Schema.Int), + rev: Schema.Int, + }), + /** Committed prompts (`name` is empty), or tool calls that ran by tool name, of one thread in one hour. */ + Schema.TaggedStruct("Tally", { + ...UsageOwner, + kind: Schema.Literals(UsageSchema.tallyKinds), + name: Schema.String, + count: Schema.Int, + rev: Schema.Int, + }), + /** Every row with a revision up to `rev` has been sent. */ + Schema.TaggedStruct("Synced", { storeId: Schema.String, rev: Schema.Int }), +]); +export type UsageItem = typeof UsageItem.Type; + /** A thread of a session: `threadId`, or the session's `main` thread when absent. */ const ThreadRef = { sessionId: Session.SessionSchema.ID, @@ -444,6 +506,8 @@ export const Api = RpcGroup.make( storeId: Schema.String, /** The store's head: the last revision taken. */ rev: Schema.Int, + /** The machine it runs on, by the name its user knows it; absent when the OS names it nothing. */ + host: optional(Schema.Struct({ name: Schema.String })), }), }), /** @@ -457,6 +521,18 @@ export const Api = RpcGroup.make( success: IndexItem, stream: true, }), + /** + * The usage rollup of this machine: every `usage` and `tally` row whose revision is past `after`, + * in revision order, then `Synced` at the head read when the call began; then the same live after + * each commit. `after: 0` reads every row. A row written again moves to a later revision and is + * sent again in full; the latest wins. Rows carry running totals, not deltas: a client replaces a + * row by its key, never adds to it. A deleted session's rows stay. + */ + Rpc.make("usage.sync", { + payload: { after: Schema.Int }, + success: UsageItem, + stream: true, + }), Rpc.make("auth.save", { payload: { credentials: OAuthCredentials }, success: OAuthInfo, diff --git a/packages/codework/src/server/handlers.ts b/packages/codework/src/server/handlers.ts index 2db276f3..87dd6f38 100644 --- a/packages/codework/src/server/handlers.ts +++ b/packages/codework/src/server/handlers.ts @@ -36,6 +36,7 @@ import { type ThreadInfo, type ThreadItem, type ThreadSummary, + type UsageItem, } from "./contract.ts"; import * as Envelope from "./envelope.ts"; import { type EncodingError, type EventEnvelope } from "./envelope.ts"; @@ -112,6 +113,43 @@ const toIndexItem = (change: Sync.IndexChange): IndexItem => { } }; +const toUsageItem = (change: Sync.UsageChange): UsageItem => { + if (change._tag === "Tally") { + const { hour, threadId, sessionId, projectId, kind, name, count, rev } = change.tally; + return { _tag: "Tally", hour, threadId, sessionId, projectId, kind, name, count, rev }; + } + const row = change.usage; + return { + _tag: "Usage", + hour: row.hour, + threadId: row.threadId, + sessionId: row.sessionId, + projectId: row.projectId, + provider: row.provider, + model: row.model, + thinking: row.thinking, + requests: row.requests, + tokens: { + input: row.input, + output: row.output, + cacheRead: row.cacheRead, + cacheWrite: row.cacheWrite, + reasoning: row.reasoning, + }, + cost: { + total: row.cost, + input: row.costInput, + output: row.costOutput, + cacheRead: row.costCacheRead, + cacheWrite: row.costCacheWrite, + }, + duration: row.duration, + timed: row.timed, + durations: row.durations, + rev: row.rev, + }; +}; + // Wire configs name only the keys they set, so `undefined` never reaches the harness under // exactOptionalPropertyTypes. Creating takes choices only; `null` clears nothing there. const runtimeInput = (runtime: RuntimeConfig | undefined): Session.RuntimeInput => ({ @@ -252,7 +290,7 @@ export const layer = Contract.Api.toLayer( return Contract.Api.of({ "server.info": Effect.fnUntraced(function* () { const head = yield* store.head; - return { pid: identity.pid, version: identity.version, storeId: head.id, rev: head.rev }; + return { ...identity, storeId: head.id, rev: head.rev }; }), "index.sync": ({ after }) => Rows.follow( @@ -266,6 +304,18 @@ export const layer = Contract.Api.toLayer( ), (head): IndexItem => ({ _tag: "Synced", storeId: head.id, rev: head.rev }), ), + "usage.sync": ({ after }) => + Rows.follow( + after, + (from, upto) => + Sync.usage(from, upto, Rows.batch).pipe( + Effect.map(({ items, next }) => ({ + items: items.map(toUsageItem), + ...(next === undefined ? {} : { next }), + })), + ), + (head): UsageItem => ({ _tag: "Synced", storeId: head.id, rev: head.rev }), + ), "auth.save": ({ credentials }) => oauth.save(credentials), "auth.status": ({ provider }) => oauth.status(provider), "auth.refresh": ({ provider }) => oauth.refresh(provider), diff --git a/packages/codework/src/server/identity.ts b/packages/codework/src/server/identity.ts index 833abcec..b547a5c5 100644 --- a/packages/codework/src/server/identity.ts +++ b/packages/codework/src/server/identity.ts @@ -1,4 +1,8 @@ -import { Context, Layer } from "effect"; +/* @effect-diagnostics nodeBuiltinImport:off -- the host's name is the OS's hostname. */ +import os from "node:os"; +import { NodeServices } from "@effect/platform-node"; +import { Context, Effect, Layer } from "effect"; +import { ChildProcess, ChildProcessSpawner } from "effect/process"; import pkg from "../../package.json" with { type: "json" }; /** @@ -11,6 +15,8 @@ import pkg from "../../package.json" with { type: "json" }; export interface Interface { readonly pid: number; readonly version: string; + /** The machine it runs on, by the name its user knows it; none when the OS names it nothing. */ + readonly host?: { readonly name: string }; } export class Service extends Context.Service()("@codeworksh/cli/server/identity/Service") {} @@ -18,4 +24,24 @@ export class Service extends Context.Service()("@codeworksh/ /** This process: a server answers with it, a client compares a server's version against it. */ export const current: Interface = { pid: process.pid, version: pkg.version }; -export const layer = Layer.succeed(Service, Service.of(current)); +// macOS's computer name is the one Finder and sharing show; its hostname is a mangled copy of it. +const hostName = Effect.gen(function* () { + if (process.platform !== "darwin") return os.hostname().trim(); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const computer = yield* spawner + .string(ChildProcess.make("scutil", ["--get", "ComputerName"])) + .pipe(Effect.orElseSucceed(() => "")); + return ( + computer.trim() || + os + .hostname() + .replace(/\.local$/, "") + .trim() + ); +}); + +/** The host's name is read once, when the server starts. */ +export const layer = Layer.effect( + Service, + hostName.pipe(Effect.map((name) => Service.of(name === "" ? current : { ...current, host: { name } }))), +).pipe(Layer.provide(NodeServices.layer)); diff --git a/packages/codework/src/server/server.ts b/packages/codework/src/server/server.ts index ed07a9e4..4f69c207 100644 --- a/packages/codework/src/server/server.ts +++ b/packages/codework/src/server/server.ts @@ -89,7 +89,13 @@ const announce = Layer.unwrap( }), ); -export const layer = (options: { host: string; port: number; harness: Harness.Options }) => { +export const layer = (options: { + host: string; + port: number; + harness: Harness.Options; + /** Who the server says it is; tests name its host. */ + identity?: Layer.Layer; +}) => { const rpc = RpcServer.layerHttp({ group: Contract.Api, path: Contract.path, protocol: "websocket" }).pipe( Layer.provide(Handlers.layer.pipe(Layer.provide(EventFeed.layer))), ); @@ -104,7 +110,7 @@ export const layer = (options: { host: string; port: number; harness: Harness.Op Layer.provideMerge(announce), Layer.provideMerge(NodeHttpServer.layer(createServer, { host: options.host, port: options.port })), Layer.provide(RpcSerialization.layerNdjson), - Layer.provide(Identity.layer), + Layer.provide(options.identity ?? Identity.layer), // Built after the harness and before everything above it, the listener included. Layer.provide(sweep), Layer.provide(Harness.layer(options.harness)), diff --git a/packages/codework/test/__artifacts__/sync.usage.json b/packages/codework/test/__artifacts__/sync.usage.json new file mode 100644 index 00000000..34256a31 --- /dev/null +++ b/packages/codework/test/__artifacts__/sync.usage.json @@ -0,0 +1,241 @@ +{ + "edges": 23, + "first": [ + { + "_tag": "Tally", + "hour": "", + "threadId": "
", + "sessionId": "", + "projectId": "", + "kind": "prompt", + "name": "", + "count": 1, + "rev": "" + }, + { + "_tag": "Tally", + "hour": "", + "threadId": "
", + "sessionId": "", + "projectId": "", + "kind": "tool", + "name": "probe", + "count": 1, + "rev": "" + }, + { + "_tag": "Usage", + "hour": "", + "threadId": "
", + "sessionId": "", + "projectId": "", + "provider": "openai", + "model": "gpt-5.6-luna", + "thinking": "medium", + "requests": 2, + "tokens": { + "input": 300, + "output": 30, + "cacheRead": 3, + "cacheWrite": 0, + "reasoning": 0 + }, + "cost": { + "total": 0.75, + "input": 0.375, + "output": 0.375, + "cacheRead": 0, + "cacheWrite": 0 + }, + "duration": 900, + "timed": 2, + "durations": [ + 0, + 1, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "rev": "" + }, + { + "_tag": "Synced", + "storeId": "", + "rev": "" + } + ], + "second": [ + { + "_tag": "Tally", + "hour": "", + "threadId": "
", + "sessionId": "", + "projectId": "", + "kind": "prompt", + "name": "", + "count": 2, + "rev": "" + }, + { + "_tag": "Usage", + "hour": "", + "threadId": "
", + "sessionId": "", + "projectId": "", + "provider": "openai", + "model": "gpt-5.6-luna", + "thinking": "medium", + "requests": 3, + "tokens": { + "input": 600, + "output": 60, + "cacheRead": 6, + "cacheWrite": 0, + "reasoning": 0 + }, + "cost": { + "total": 1.5, + "input": 0.75, + "output": 0.75, + "cacheRead": 0, + "cacheWrite": 0 + }, + "duration": 1800, + "timed": 3, + "durations": [ + 0, + 1, + 0, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "rev": "" + }, + { + "_tag": "Synced", + "storeId": "", + "rev": "" + } + ], + "resumed": [ + { + "_tag": "Tally", + "hour": "", + "threadId": "
", + "sessionId": "", + "projectId": "", + "kind": "prompt", + "name": "", + "count": 2, + "rev": "" + }, + { + "_tag": "Usage", + "hour": "", + "threadId": "
", + "sessionId": "", + "projectId": "", + "provider": "openai", + "model": "gpt-5.6-luna", + "thinking": "medium", + "requests": 3, + "tokens": { + "input": 600, + "output": 60, + "cacheRead": 6, + "cacheWrite": 0, + "reasoning": 0 + }, + "cost": { + "total": 1.5, + "input": 0.75, + "output": 0.75, + "cacheRead": 0, + "cacheWrite": 0 + }, + "duration": 1800, + "timed": 3, + "durations": [ + 0, + 1, + 0, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "rev": "" + }, + { + "_tag": "Synced", + "storeId": "", + "rev": "" + } + ], + "idle": [ + { + "_tag": "Synced", + "storeId": "", + "rev": "" + } + ], + "paged": { + "tallies": 600, + "rows": 603 + } +} diff --git a/packages/codework/test/server.test.ts b/packages/codework/test/server.test.ts index 0369eeb6..536963cc 100644 --- a/packages/codework/test/server.test.ts +++ b/packages/codework/test/server.test.ts @@ -9,7 +9,7 @@ import { HttpServer } from "effect/http"; import { RpcTest } from "effect/rpc"; import { execFile } from "node:child_process"; import { mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs"; -import { tmpdir } from "node:os"; +import { hostname, tmpdir } from "node:os"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; @@ -76,6 +76,17 @@ afterAll(() => { }); describe("server", () => { + it("server.info names the host as its user sees it", async () => { + const name = + process.platform === "darwin" + ? (await exec("scutil", ["--get", "ComputerName"])).stdout.trim() + : hostname().trim(); + await Effect.gen(function* () { + const rpc = yield* RpcTest.makeClient(Contract.Api); + expect((yield* rpc["server.info"]({})).host).toEqual({ name }); + }).pipe(Effect.scoped, Effect.provide(layer()), Effect.runPromise); + }); + it("sandbox.drivers returns the core drivers", () => Effect.gen(function* () { const rpc = yield* RpcTest.makeClient(Contract.Api); diff --git a/packages/codework/test/usage.test.ts b/packages/codework/test/usage.test.ts new file mode 100644 index 00000000..593b31ca --- /dev/null +++ b/packages/codework/test/usage.test.ts @@ -0,0 +1,226 @@ +/* @effect-diagnostics nodeBuiltinImport:off -- starts the real server in a temporary home. */ +import { createAssistantMessageEventStream, type Message } from "@codeworksh/aikit"; +import { Effect, Layer, Queue, Stream } from "effect"; +import { HttpServer } from "effect/http"; +import { mkdtempSync, realpathSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, it } from "vite-plus/test"; +import type * as LLM from "../../harness/src/runner/llm.ts"; +import { assistant } from "../../harness/test/fixtures/llm.ts"; +import * as Client from "../src/server/client.ts"; +import { edges, type UsageItem } from "../src/server/contract.ts"; +import * as Server from "../src/server/server.ts"; +import { testModel } from "./utils/model.ts"; + +type Synced = Extract; +type Row = Exclude; + +// 2020-01-01T00:00:00Z: every request is made in this hour; prompts are tallied in the real one. +const H = Date.UTC(2020, 0, 1); +// More tool names in one reply than one page holds (`Rows.batch`), so its revision spans two pages. +const wide = 600; + +/** Request `n` of the server: 1 calls a tool, 4 calls `wide` tools at once, every other one stops. */ +const scripted: LLM.Open = (() => { + let count = 0; + return (input) => + Effect.sync(() => { + count += 1; + const index = count; + const calls = index === 1 ? ["probe"] : index === 4 ? Array.from({ length: wide }, (_, i) => `tool_${i}`) : []; + const message = assistant(input, index, { + usage: { + input: 100 * index, + output: 10 * index, + cacheRead: index, + cacheWrite: 0, + totalTokens: 111 * index, + cost: { input: 0.125 * index, output: 0.125 * index, cacheRead: 0, cacheWrite: 0, total: 0.25 * index }, + }, + time: { created: H + index * 1000, completed: H + index * 1000 + 300 * index }, + ...(calls.length === 0 + ? {} + : { + stopReason: "toolUse", + parts: calls.map((name, i): Message.ToolCallPendingPart => ({ + type: "toolCall", + callID: `call_${index}_${i}`, + name, + arguments: {}, + status: "pending", + time: { start: H, end: H }, + })), + }), + }); + const events = createAssistantMessageEventStream(); + events.push({ type: "start", partial: message }); + events.push({ type: "done", reason: calls.length === 0 ? "stop" : "toolUse", message }); + return events; + }); +})(); + +const keyOf = (row: Row) => + row._tag === "Usage" + ? `usage|${row.hour}|${row.threadId}|${row.provider}|${row.model}|${row.thinking}` + : `tally|${row.hour}|${row.threadId}|${row.kind}|${row.name}`; +const rowsOf = (items: ReadonlyArray) => + items.flatMap((item): Array => (item._tag === "Synced" ? [] : [item])); +const isUsage = (item: UsageItem): item is Extract => item._tag === "Usage"; + +it("usage.sync sends the rollup in revision order, resends rows with running totals, resumes and pages", async () => { + const home = realpathSync(mkdtempSync(join(tmpdir(), "codework-sync-usage-"))); + writeFileSync(join(home, "settings.jsonc"), JSON.stringify({ model: testModel })); + try { + const artifact = await Effect.gen(function* () { + const listener = yield* HttpServer.HttpServer; + if (listener.address._tag === "UnixPathAddress") return yield* Effect.die("Expected TCP listener"); + const context = yield* Layer.build(Client.layer(`ws://127.0.0.1:${listener.address.port}/rpc/v1`)); + const rpc = yield* Client.make.pipe(Effect.provideContext(context)); + + // Up to and including the first `Synced`: one catch-up. + const catchUp = (after: number) => + rpc["usage.sync"]({ after }).pipe( + Stream.takeUntil((item) => item._tag === "Synced"), + Stream.runCollect, + ); + + const live = yield* Queue.unbounded(); + yield* rpc["usage.sync"]({ after: 0 }).pipe( + Stream.runForEach((item) => Queue.offer(live, item)), + Effect.forkScoped, + ); + /** Live items up to and including `Synced` at the current head. */ + const settle = Effect.gen(function* () { + const head = (yield* rpc["server.info"]({})).rev; + const items: Array = []; + while (true) { + const item = yield* Queue.take(live); + items.push(item); + if (item._tag === "Synced" && item.rev >= head) return items; + } + }); + + // (1) A fresh server: `Synced` at revision 0 and nothing before it. + const empty = yield* Queue.take(live); + + const created = yield* rpc["session.create"]({ directory: home, project: { name: "Alpha" } }); + const sessionId = created.session.id; + const projectId = created.session.project?.id; + if (projectId === undefined) return yield* Effect.die("session has no project"); + const turn = (text: string) => + rpc["thread.prompt"]({ sessionId, text }).pipe(Effect.andThen(rpc["thread.wait"]({ sessionId }))); + + // Requests 1 and 2: a tool call, then the answer. + yield* turn("First prompt"); + const first = yield* settle; + const firstSynced = first.at(-1) as Synced; + const info = yield* rpc["thread.info"]({ sessionId }); + + // (2) Request 3, in the same hour: the same `Usage` key again, with larger running totals. + yield* turn("Second prompt"); + const second = yield* settle; + + // (3) Resume from either `Synced`. + const resumed = yield* catchUp(firstSynced.rev); + const latest = second.at(-1) as Synced; + const idle = yield* catchUp(latest.rev); + + // (4) Request 4 calls `wide` tools: their tallies share one revision across two pages. + yield* turn("Wide prompt"); + const paged = yield* settle; + const whole = yield* catchUp(0); + + // (5) Deleting the session retracts nothing. + const removed = yield* rpc["session.delete"]({ sessionId }); + const afterDelete = yield* settle; + const reread = yield* catchUp(0); + + const usageOf = (items: ReadonlyArray) => items.filter(isUsage); + const [usage1] = usageOf(first); + const [usage2] = usageOf(second); + if (usage1 === undefined || usage2 === undefined) return yield* Effect.die("no usage row was sent"); + const wideTallies = rowsOf(paged).filter((row) => row._tag === "Tally" && row.name.startsWith("tool_")); + const wholeRows = rowsOf(whole); + const revs = (items: ReadonlyArray) => items.map((item) => item.rev); + + expect(empty).toEqual({ _tag: "Synced", storeId: firstSynced.storeId, rev: 0 }); + expect(first.filter((item) => item._tag === "Usage")).toHaveLength(1); + expect(usage1.requests).toBe(2); + expect({ + cost: usage1.cost.total, + input: usage1.tokens.input, + output: usage1.tokens.output, + cacheRead: usage1.tokens.cacheRead, + cacheWrite: usage1.tokens.cacheWrite, + }).toEqual(info.usage); + expect(rowsOf(first).some((row) => row._tag === "Tally" && row.kind === "tool" && row.name === "probe")).toBe( + true, + ); + expect(keyOf(usage2)).toBe(keyOf(usage1)); + expect(usage2.rev).toBeGreaterThan(usage1.rev); + expect(usage2.requests).toBe(3); + expect(usage2.tokens.input).toBe(usage1.tokens.input + 300); + expect(usage2.cost.total).toBe(usage1.cost.total + 0.75); + expect(rowsOf(resumed).map(keyOf).sort()).toEqual(rowsOf(second).map(keyOf).sort()); + expect(rowsOf(resumed).every((row) => row.rev > firstSynced.rev)).toBe(true); + expect(idle).toEqual([latest]); + expect(wideTallies).toHaveLength(wide); + expect(new Set(wideTallies.map((row) => row.rev)).size).toBe(1); + expect(new Set(wideTallies.map(keyOf)).size).toBe(wide); + expect(revs(paged)).toEqual(revs(paged).toSorted((a, b) => a - b)); + expect(wholeRows.length).toBeGreaterThan(500); + expect(new Set(wholeRows.map(keyOf)).size).toBe(wholeRows.length); + expect(revs(whole)).toEqual(revs(whole).toSorted((a, b) => a - b)); + expect(removed.deleted).toBe(true); + expect(rowsOf(afterDelete)).toEqual([]); + expect(rowsOf(reread)).toEqual(wholeRows); + + const labels = new Map([ + [firstSynced.storeId, ""], + [sessionId, ""], + [projectId, ""], + [created.id, "
"], + ]); + // Revisions by rank within the artifact: their order is the contract, their values are not. + // How commits batch into live rounds varies, so a live segment shows its rows and its last `Synced`. + const settled = (items: ReadonlyArray) => [...rowsOf(items), items.at(-1)!]; + const shown = [...settled(first), ...settled(second), ...resumed, ...idle]; + const ranks = [...new Set(revs(shown))].sort((a, b) => a - b); + const normalize = (value: unknown, key?: string): unknown => { + if (typeof value === "string") return labels.get(value) ?? value; + if (typeof value === "number") { + if (key === "rev") return ranks.includes(value) ? `` : ""; + if (key === "hour") return value === H ? "" : ""; + return value; + } + if (Array.isArray(value)) return value.map((item) => normalize(item)); + if (typeof value === "object" && value !== null) + return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, normalize(v, k)])); + return value; + }; + return normalize({ + edges: edges.length, + first: settled(first), + second: settled(second), + resumed, + idle, + paged: { tallies: wideTallies.length, rows: wholeRows.length }, + }); + }).pipe( + Effect.scoped, + Effect.provide( + Server.layer({ + host: "127.0.0.1", + port: 0, + harness: { home, hostCwd: home, plugins: [], llm: scripted }, + }), + ), + Effect.timeout("60 seconds"), + Effect.runPromise, + ); + await expect(`${JSON.stringify(artifact, null, "\t")}\n`).toMatchFileSnapshot("./__artifacts__/sync.usage.json"); + } finally { + rmSync(home, { recursive: true, force: true }); + } +}, 80_000); From 6ae4e6fc0f5892782a17621b40ebee08b019c6a8 Mon Sep 17 00:00:00 2001 From: Sanchit Rk <0xsanchit@gmail.com> Date: Sat, 14 Nov 2026 14:16:44 +0530 Subject: [PATCH 3/8] feat(server): mirror usage per machine, name machines by their host, serve metrics The app server follows usage.sync for each machine into its own usage and tally tables, keyed by machine and store so history survives a recreated store, and keeps working against a server without the stream. A machine's name is now the one its server reports; the name typed in Settings is an optional override, and the built-in server is no longer seeded as CodeWork. metrics.usage and metrics.hours read the mirror by calendar day and hour in a given time zone, with machine and project filters. Co-Authored-By: Claude Fable 5.2 --- apps/contract/src/contract.ts | 155 ++++- apps/server/src/cli.ts | 2 + apps/server/src/db.ts | 66 +++ apps/server/src/handlers.ts | 3 + apps/server/src/health.ts | 10 + apps/server/src/metrics.ts | 336 +++++++++++ apps/server/src/mirror.ts | 46 ++ apps/server/src/reads.ts | 77 ++- apps/server/src/servers.ts | 74 ++- apps/server/src/sync.ts | 98 +++- .../test/__artifacts__/host.migrated.json | 30 + .../server/test/__artifacts__/host.names.json | 178 ++++++ .../test/__artifacts__/metrics.filters.json | 415 +++++++++++++ .../test/__artifacts__/metrics.stores.json | 223 +++++++ .../test/__artifacts__/metrics.usage.json | 271 +++++++++ .../test/__artifacts__/metrics.zones.json | 410 +++++++++++++ .../test/__artifacts__/reads.machines.json | 4 +- apps/server/test/__artifacts__/schema.json | 46 +- .../__artifacts__/usage.machines.rollup.json | 550 ++++++++++++++++++ .../test/__artifacts__/usage.mirror.json | 228 ++++++++ .../test/__artifacts__/usage.older.json | 9 + .../test/__artifacts__/usage.restored.json | 63 ++ .../test/__artifacts__/usage.store.json | 106 ++++ apps/server/test/fixtures/apps.ts | 58 +- apps/server/test/fixtures/codework.ts | 44 +- apps/server/test/host.test.ts | 158 +++++ apps/server/test/metrics.test.ts | 486 ++++++++++++++++ apps/server/test/reads.test.ts | 19 +- apps/server/test/schema.test.ts | 12 +- apps/server/test/usage.machines.live.test.ts | 537 +++++++++++++++++ apps/server/test/usage.test.ts | 335 +++++++++++ apps/widget/src/codework.ts | 78 +++ 32 files changed, 5025 insertions(+), 102 deletions(-) create mode 100644 apps/server/src/metrics.ts create mode 100644 apps/server/test/__artifacts__/host.migrated.json create mode 100644 apps/server/test/__artifacts__/host.names.json create mode 100644 apps/server/test/__artifacts__/metrics.filters.json create mode 100644 apps/server/test/__artifacts__/metrics.stores.json create mode 100644 apps/server/test/__artifacts__/metrics.usage.json create mode 100644 apps/server/test/__artifacts__/metrics.zones.json create mode 100644 apps/server/test/__artifacts__/usage.machines.rollup.json create mode 100644 apps/server/test/__artifacts__/usage.mirror.json create mode 100644 apps/server/test/__artifacts__/usage.older.json create mode 100644 apps/server/test/__artifacts__/usage.restored.json create mode 100644 apps/server/test/__artifacts__/usage.store.json create mode 100644 apps/server/test/host.test.ts create mode 100644 apps/server/test/metrics.test.ts create mode 100644 apps/server/test/usage.machines.live.test.ts create mode 100644 apps/server/test/usage.test.ts diff --git a/apps/contract/src/contract.ts b/apps/contract/src/contract.ts index f0198831..b8faf95e 100644 --- a/apps/contract/src/contract.ts +++ b/apps/contract/src/contract.ts @@ -17,6 +17,7 @@ import { Repository, RunState, SpaceInfo, + UsageItem, } from "@codeworksh/widget/codework"; import { Event, EventTimeout, Publish, Request, Synced } from "@codeworksh/widget/event"; @@ -373,11 +374,21 @@ export const ServerKind = Schema.Literals(["local", "remote"]); /** The server the app always has, which cannot be removed. */ export const LOCAL_SERVER = "local"; +/** How a server is named: the user's `override`, else the `host` name its server reports, else where it is. */ +const ServerNaming = { + /** As shown. */ + name: Schema.String, + /** As its server last reported it; none from a server that reports no name or was never reached. */ + host: Schema.NullOr(Schema.String), + /** As typed in Settings; none when the user named it nothing. */ + override: Schema.NullOr(Schema.String), +}; + /** A local server and how it is set up, with its executable and process as found when listed. */ export const LocalServerStatus = Schema.Struct({ id: Schema.String, kind: Schema.Literal("local"), - name: Schema.String, + ...ServerNaming, /** The one the app connects to. */ isDefault: Schema.Boolean, /** @@ -398,7 +409,7 @@ export type LocalServerStatus = typeof LocalServerStatus.Type; export const RemoteServerStatus = Schema.Struct({ id: Schema.String, kind: Schema.Literal("remote"), - name: Schema.String, + ...ServerNaming, isDefault: Schema.Boolean, url: Schema.String, }); @@ -421,23 +432,23 @@ export type CliAction = typeof CliAction.Type; /** * A new server. A local one needs a home of its own, since a home names one server; a remote one - * needs its WebSocket URL. + * needs its WebSocket URL. A blank or absent `name` is no override. */ export const ServerAdd = Schema.Union([ Schema.Struct({ kind: Schema.Literal("local"), - name: Schema.String, + name: Schema.optionalKey(Schema.String), executable: Schema.NullOr(Schema.String), home: Schema.String, args: Schema.String, }), - Schema.Struct({ kind: Schema.Literal("remote"), name: Schema.String, url: Schema.String }), + Schema.Struct({ kind: Schema.Literal("remote"), name: Schema.optionalKey(Schema.String), url: Schema.String }), ]); export type ServerAdd = typeof ServerAdd.Type; /** * Changes to a server; an absent field is left as it is. A `null` path goes back to the one found; - * `env` replaces every variable. `url` is for a remote server only, the rest but `name` for a local one. + * a blank `name` removes the override; `env` replaces every variable. `url` is for a remote server only, the rest but `name` for a local one. */ export const ServerUpdate = Schema.Struct({ id: Schema.String, @@ -468,6 +479,8 @@ export const syncTopics = { tracking: "sync/tracking", /** `SyncState` whenever a space daemon's mirrored state changed (git status, skills); stored. */ state: "sync/state", + /** `SyncUsage` after each applied usage batch that carried rows; stored. */ + usage: "sync/usage", }; /** @@ -499,6 +512,14 @@ export const SyncState = Schema.Struct({ }); export type SyncState = typeof SyncState.Type; +/** A machine's usage rows from one of its stores were mirrored anew; `items` counts them. */ +export const SyncUsage = Schema.Struct({ + machineId: Schema.String, + storeId: Schema.String, + items: Schema.Int, +}); +export type SyncUsage = typeof SyncUsage.Type; + /** A thread's status moved; `at` is the app server's clock. */ export const SyncStatus = Schema.Struct({ threadId: Schema.String, @@ -603,13 +624,123 @@ export type AllThreads = typeof AllThreads.Type; * `sync/delta/` streams its draft between checkpoints. * - `threads.list`: `sync/changed` naming a thread or a removed session; every `sync/status` and * `sync/tracking`. + * - `metrics.usage` and `metrics.hours`: every `sync/usage`; the titles and names they carry are + * read again with it. */ export const readTopics = { sidebar: [syncTopics.changed, syncTopics.status, syncTopics.tracking], threads: [syncTopics.changed, syncTopics.status, syncTopics.tracking], line: (threadId: string) => [syncTopics.changed, syncTopics.delta(threadId)], + metrics: [syncTopics.usage], +}; + +const [UsageWire, TallyWire] = UsageItem.members; + +/** A calendar day in the request's `zone`, `YYYY-MM-DD`. */ +const LocalDay = Schema.String; + +/** + * What the metrics reads cover: every machine and project unless one is named. `zone` is an IANA + * time zone; days and hours are its wall clock. The rollup's resolution is a UTC hour, which counts + * whole in the local hour holding its start: in a zone 30 or 45 minutes off the hour, all of it in + * the earlier one. A day a clock change shortens has 23 hours, one it lengthens 25, the repeated + * hour summed into one. + */ +const MetricsScope = { + zone: Schema.String, + machineId: Schema.optionalKey(Schema.String), + projectId: Schema.optionalKey(Schema.String), }; +/** Requests of one session on one local day to one provider, model and thinking level, summed over its threads. */ +export const MetricsRow = Schema.Struct({ + day: LocalDay, + machineId: Schema.String, + sessionId: Schema.String, + provider: Schema.String, + model: Schema.String, + thinking: Schema.String, + requests: Schema.Int, + tokens: UsageWire.fields.tokens, + cost: UsageWire.fields.cost, + duration: Schema.Finite, + timed: Schema.Int, + /** Over `edges`, as `UsageItem`'s. */ + durations: Schema.Array(Schema.Int), +}); +export type MetricsRow = typeof MetricsRow.Type; + +/** Prompts (`name` empty) or a tool's calls of one session on one local day. */ +export const MetricsTally = Schema.Struct({ + day: LocalDay, + machineId: Schema.String, + sessionId: Schema.String, + kind: TallyWire.fields.kind, + name: Schema.String, + count: Schema.Int, +}); +export type MetricsTally = typeof MetricsTally.Type; + +/** How many of a session's threads had requests or tallies on a local day. */ +export const MetricsThreads = Schema.Struct({ + day: LocalDay, + machineId: Schema.String, + sessionId: Schema.String, + threads: Schema.Int, +}); +export type MetricsThreads = typeof MetricsThreads.Type; + +/** + * A session the rows name. `title` is its main thread's, or what an untitled one is called; null + * once the session is deleted or gone from the mirror, its rows kept. Which sandbox it ran in is + * not mirrored, so it is not here. + */ +export const MetricsSession = Schema.Struct({ + id: Schema.String, + machineId: Schema.String, + /** As its rows recorded it. */ + projectId: Schema.String, + title: Schema.NullOr(Schema.String), + deleted: Schema.Boolean, +}); +export type MetricsSession = typeof MetricsSession.Type; + +/** A period's usage at the grain the Metrics page slices, with the names it shows. */ +export const MetricsUsage = Schema.Struct({ + rows: Schema.Array(MetricsRow), + tallies: Schema.Array(MetricsTally), + threads: Schema.Array(MetricsThreads), + sessions: Schema.Array(MetricsSession), + /** + * A project id is its repo's, so one on several machines is listed once, named as on the machine + * that changed it last; null when no machine has it any more. + */ + projects: Schema.Array(Schema.Struct({ id: Schema.String, name: Schema.NullOr(Schema.String) })), + machines: Schema.Array(Schema.Struct({ id: Schema.String, name: Schema.String })), + /** The upper bounds in ms of `durations`' buckets. */ + edges: Schema.Array(Schema.Int), +}); +export type MetricsUsage = typeof MetricsUsage.Type; + +/** Requests of one session in one local hour (0–23) to one provider and model. */ +export const MetricsHour = Schema.Struct({ + hour: Schema.Int, + machineId: Schema.String, + sessionId: Schema.String, + provider: Schema.String, + model: Schema.String, + requests: Schema.Int, + /** Input, output, cache read and cache write. */ + tokens: Schema.Int, + cost: Schema.Finite, +}); +export type MetricsHour = typeof MetricsHour.Type; + +/** A metrics read refused: an unknown time zone, a malformed day, or a period ending before it starts. */ +export class MetricsError extends Schema.TaggedError()("Server.MetricsError", { + message: Schema.String, +}) {} + /** A desktop edit refused: a value out of range, or a thread, session, group or organization not found. */ export class TrackingError extends Schema.TaggedError()("Server.TrackingError", { message: Schema.String, @@ -668,6 +799,18 @@ export const Api = RpcGroup.make( payload: { organizationId: Schema.String, by: ThreadsBy }, success: AllThreads, }), + /** Usage from `since` to `until`, both local days included, summed over threads and stores. Read from app.db alone. */ + Rpc.make("metrics.usage", { + payload: { ...MetricsScope, since: LocalDay, until: LocalDay }, + success: MetricsUsage, + error: MetricsError, + }), + /** One local day's usage by local hour, for its chart; its sessions and threads are in `metrics.usage`. Read from app.db alone. */ + Rpc.make("metrics.hours", { + payload: { ...MetricsScope, day: LocalDay }, + success: Schema.Array(MetricsHour), + error: MetricsError, + }), /** The models a new session can use; `hostDir` is its folder, whose project settings apply. */ Rpc.make("models.list", { payload: { hostDir: Schema.optionalKey(Schema.String) }, diff --git a/apps/server/src/cli.ts b/apps/server/src/cli.ts index 4da40b21..f6b72522 100644 --- a/apps/server/src/cli.ts +++ b/apps/server/src/cli.ts @@ -171,6 +171,8 @@ export const statuses = Effect.gen(function* () { id: row.id, kind: row.kind, name: row.name, + host: row.host, + override: row.override, isDefault: row.isDefault, home, cli, diff --git a/apps/server/src/db.ts b/apps/server/src/db.ts index 28f0d323..b8914f73 100644 --- a/apps/server/src/db.ts +++ b/apps/server/src/db.ts @@ -274,6 +274,72 @@ const migrations = { // The one organization the UI uses until auth exists. yield* sql`INSERT INTO organization ${sql.insert({ id: "personal", name: "Personal", createdAt: now, updatedAt: now })}`; }), + + // Each machine's usage rollup, as `usage.sync` sends it. Keyed by store too: a machine whose + // store is recreated starts a new one, and the old store's rows are then the only history left. + // No keys to project, session or thread: the rows outlive them. + "202611140001_usage": Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + // `rollup` is the usage cursor, counted in `rollup_store`: it moves apart from `cursor` and `store_id`. + yield* sql`ALTER TABLE machine ADD COLUMN rollup INTEGER NOT NULL DEFAULT 0`; + yield* sql`ALTER TABLE machine ADD COLUMN rollup_store TEXT`; + yield* sql` + CREATE TABLE usage ( + machine_id TEXT NOT NULL REFERENCES machine(id) ON DELETE CASCADE, + store_id TEXT NOT NULL, + hour INTEGER NOT NULL, + thread_id TEXT NOT NULL, + provider TEXT NOT NULL, + model TEXT NOT NULL, + thinking TEXT NOT NULL, + session_id TEXT NOT NULL, + project_id TEXT NOT NULL, + requests INTEGER NOT NULL, + input INTEGER NOT NULL, + output INTEGER NOT NULL, + cache_read INTEGER NOT NULL, + cache_write INTEGER NOT NULL, + reasoning INTEGER NOT NULL, + cost REAL NOT NULL, + cost_input REAL NOT NULL, + cost_output REAL NOT NULL, + cost_cache_read REAL NOT NULL, + cost_cache_write REAL NOT NULL, + duration REAL NOT NULL, + timed INTEGER NOT NULL, + durations TEXT NOT NULL, + rev INTEGER NOT NULL, + PRIMARY KEY (machine_id, store_id, hour, thread_id, provider, model, thinking) + ) + `; + yield* sql` + CREATE TABLE tally ( + machine_id TEXT NOT NULL REFERENCES machine(id) ON DELETE CASCADE, + store_id TEXT NOT NULL, + hour INTEGER NOT NULL, + thread_id TEXT NOT NULL, + kind TEXT NOT NULL, + name TEXT NOT NULL, + session_id TEXT NOT NULL, + project_id TEXT NOT NULL, + count INTEGER NOT NULL, + rev INTEGER NOT NULL, + PRIMARY KEY (machine_id, store_id, hour, thread_id, kind, name) + ) + `; + }), + + // A machine is named by what its server reports (`host`); `name` is only the user's override, so + // it becomes nullable. ADD + DROP + RENAME instead of rebuilding the table, which its foreign keys + // would cascade into. The built-in server's old seed name is no override. + "202611140002_host": Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* sql`ALTER TABLE machine ADD COLUMN host TEXT`; + yield* sql`ALTER TABLE machine ADD COLUMN alias TEXT`; + yield* sql`UPDATE machine SET alias = name WHERE NOT (id = 'local' AND name = 'CodeWork')`; + yield* sql`ALTER TABLE machine DROP COLUMN name`; + yield* sql`ALTER TABLE machine RENAME COLUMN alias TO name`; + }), }; const setup = Effect.gen(function* () { diff --git a/apps/server/src/handlers.ts b/apps/server/src/handlers.ts index 52e24bdb..3c41f6cb 100644 --- a/apps/server/src/handlers.ts +++ b/apps/server/src/handlers.ts @@ -18,6 +18,7 @@ import { Desk } from "./desk.ts"; import { Event } from "./event.ts"; import { Health } from "./health.ts"; import { Kv } from "./kv.ts"; +import { Metrics } from "./metrics.ts"; import { Reads } from "./reads.ts"; import { Skills } from "./skills.ts"; import { Servers } from "./servers.ts"; @@ -106,6 +107,8 @@ export const layer = Contract.Api.toLayer( "changes.diff": (input) => Changes.diff(input, sync.client(input.machineId)), "threads.list": (input) => Reads.threads(input).pipe(Effect.provideService(SqlClient.SqlClient, sql)), "threads.line": (input) => Reads.line(input).pipe(Effect.provideService(SqlClient.SqlClient, sql)), + "metrics.usage": (input) => Metrics.usage(input).pipe(Effect.provideService(SqlClient.SqlClient, sql)), + "metrics.hours": (input) => Metrics.hours(input).pipe(Effect.provideService(SqlClient.SqlClient, sql)), "host.info": Effect.fn("host.info")(function* () { return yield* call((client) => client("host.info", {})); diff --git a/apps/server/src/health.ts b/apps/server/src/health.ts index 7863f51c..1ca12248 100644 --- a/apps/server/src/health.ts +++ b/apps/server/src/health.ts @@ -71,6 +71,16 @@ export const layer = Layer.effect( yield* check.pipe(Effect.forkScoped); + // A name a server reports reaches Settings without waiting for the next check; the app's own + // edits are checked by their handlers. + yield* servers.watch.pipe( + Stream.map((rows) => rows.map((row) => `${row.id}=${row.host}`).join("\n")), + Stream.changes, + Stream.drop(1), + Stream.runForEach(() => check), + Effect.forkScoped, + ); + // Every interval; a new interval starts counting at once, and 0 waits for one. yield* Effect.forever( Effect.gen(function* () { diff --git a/apps/server/src/metrics.ts b/apps/server/src/metrics.ts new file mode 100644 index 00000000..8dbd37e4 --- /dev/null +++ b/apps/server/src/metrics.ts @@ -0,0 +1,336 @@ +import { Effect, Schema } from "effect"; +import { SqlClient } from "effect/sql"; +import { + type MetricsHour, + MetricsError, + type MetricsRow, + type MetricsSession, + type MetricsTally, + type MetricsUsage, +} from "@codeworksh/contract"; +import { edges, UNTITLED } from "@codeworksh/widget/codework"; +import { Reads } from "./reads.ts"; + +/* + * The Metrics page's reads, from the mirrored usage rollup. SQL sums each UTC hour over threads and + * stores; the hours are then bucketed into `zone`'s days and hours here, as SQLite knows no zones. + */ + +const HOUR = 3_600_000; +const DAY = 24 * HOUR; +/** Past the widest UTC offsets (−12 and +14 hours): every UTC hour that can start on a requested local day. */ +const MARGIN = 15 * HOUR; + +interface Scope { + readonly zone: string; + readonly machineId?: string; + readonly projectId?: string; +} + +/** A day's UTC midnight; a malformed or impossible day (`2026-02-30`) is refused. */ +const midnight = (day: string) => { + const parts = /^(\d{4})-(\d{2})-(\d{2})$/.exec(day); + const at = parts === null ? Number.NaN : Date.UTC(Number(parts[1]), Number(parts[2]) - 1, Number(parts[3])); + return Number.isNaN(at) || new Date(at).toISOString().slice(0, 10) !== day + ? Effect.fail(new MetricsError({ message: `"${day}" is not a day: expected YYYY-MM-DD.` })) + : Effect.succeed(at); +}; + +/** Each UTC hour's local day and hour in `zone`; each hour is formatted once. */ +const clock = (zone: string) => + Effect.try({ + try: () => + new Intl.DateTimeFormat("en-US", { + timeZone: zone, + year: "numeric", + month: "2-digit", + day: "2-digit", + hour: "2-digit", + hourCycle: "h23", + }), + catch: () => new MetricsError({ message: `"${zone}" is not a time zone.` }), + }).pipe( + Effect.map((format) => { + const seen = new Map(); + return (hour: number) => { + const known = seen.get(hour); + if (known !== undefined) return known; + const part = Object.fromEntries(format.formatToParts(hour).map(({ type, value }) => [type, value])); + const local = { day: `${part.year}-${part.month}-${part.day}`, hour: Number(part.hour) }; + seen.set(hour, local); + return local; + }; + }), + ); + +/** The local days `since` to `until` in `zone`, and the UTC hours to read for them. */ +const period = Effect.fn(function* (zone: string, since: string, until: string) { + const local = yield* clock(zone); + const from = yield* midnight(since); + const to = yield* midnight(until); + if (to < from) + return yield* new MetricsError({ message: `The period ends (${until}) before it starts (${since}).` }); + return { local, from: from - MARGIN, to: to + DAY + MARGIN }; +}); + +const where = (sql: SqlClient.SqlClient, scope: Scope, from: number, to: number) => + sql.and([ + sql`hour >= ${from}`, + sql`hour < ${to}`, + ...(scope.machineId === undefined ? [] : [sql`machine_id = ${scope.machineId}`]), + ...(scope.projectId === undefined ? [] : [sql`project_id = ${scope.projectId}`]), + ]); + +const decodeHistograms = Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Array(Schema.Array(Schema.Int)))); + +/** Each session's usage per UTC hour, provider, model and thinking level, summed over its threads and stores. */ +const hourly = (sql: SqlClient.SqlClient, scope: Scope, from: number, to: number) => + sql<{ + machineId: string; + hour: number; + sessionId: string; + provider: string; + model: string; + thinking: string; + requests: number; + input: number; + output: number; + cacheRead: number; + cacheWrite: number; + reasoning: number; + cost: number; + costInput: number; + costOutput: number; + costCacheRead: number; + costCacheWrite: number; + duration: number; + timed: number; + durations: string; + }>` + SELECT machine_id, hour, session_id, provider, model, thinking, SUM(requests) AS requests, + SUM(input) AS input, SUM(output) AS output, SUM(cache_read) AS cache_read, SUM(cache_write) AS cache_write, + SUM(reasoning) AS reasoning, SUM(cost) AS cost, SUM(cost_input) AS cost_input, SUM(cost_output) AS cost_output, + SUM(cost_cache_read) AS cost_cache_read, SUM(cost_cache_write) AS cost_cache_write, + SUM(duration) AS duration, SUM(timed) AS timed, json_group_array(json(durations)) AS durations + FROM usage WHERE ${where(sql, scope, from, to)} + GROUP BY machine_id, hour, session_id, provider, model, thinking + `; + +/** Adds `value` into the entry at `key`, starting from `value` itself. */ +const add = (into: Map, key: ReadonlyArray, value: A, plus: (sum: A, value: A) => A) => { + const id = key.join("\u0000"); + const sum = into.get(id); + into.set(id, sum === undefined ? value : plus(sum, value)); +}; +/** Entries by key, so a read answers in the same order each time. */ +const ordered = (map: Map) => + [...map].toSorted(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)).map(([, value]) => value); + +const plusRow = (sum: MetricsRow, row: MetricsRow): MetricsRow => ({ + ...sum, + requests: sum.requests + row.requests, + tokens: { + input: sum.tokens.input + row.tokens.input, + output: sum.tokens.output + row.tokens.output, + cacheRead: sum.tokens.cacheRead + row.tokens.cacheRead, + cacheWrite: sum.tokens.cacheWrite + row.tokens.cacheWrite, + reasoning: sum.tokens.reasoning + row.tokens.reasoning, + }, + cost: { + total: sum.cost.total + row.cost.total, + input: sum.cost.input + row.cost.input, + output: sum.cost.output + row.cost.output, + cacheRead: sum.cost.cacheRead + row.cost.cacheRead, + cacheWrite: sum.cost.cacheWrite + row.cost.cacheWrite, + }, + duration: sum.duration + row.duration, + timed: sum.timed + row.timed, + durations: sum.durations.map((count, bucket) => count + (row.durations[bucket] ?? 0)), +}); + +export const usage = Effect.fn("Metrics.usage")(function* ( + request: Scope & { readonly since: string; readonly until: string }, +) { + const { local, from, to } = yield* period(request.zone, request.since, request.until); + const inPeriod = (hour: number) => { + const { day } = local(hour); + return day >= request.since && day <= request.until ? day : undefined; + }; + return yield* Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const filter = where(sql, request, from, to); + + const rows = new Map(); + for (const row of yield* hourly(sql, request, from, to)) { + const day = inPeriod(row.hour); + if (day === undefined) continue; + const empty = Array.from({ length: edges.length + 1 }, () => 0); + const durations = (yield* decodeHistograms(row.durations)).reduce( + (sum, list) => sum.map((count, bucket) => count + (list[bucket] ?? 0)), + empty, + ); + const key = [day, row.machineId, row.sessionId, row.provider, row.model, row.thinking]; + add( + rows, + key, + { + day, + machineId: row.machineId, + sessionId: row.sessionId, + provider: row.provider, + model: row.model, + thinking: row.thinking, + requests: row.requests, + tokens: { + input: row.input, + output: row.output, + cacheRead: row.cacheRead, + cacheWrite: row.cacheWrite, + reasoning: row.reasoning, + }, + cost: { + total: row.cost, + input: row.costInput, + output: row.costOutput, + cacheRead: row.costCacheRead, + cacheWrite: row.costCacheWrite, + }, + duration: row.duration, + timed: row.timed, + durations, + }, + plusRow, + ); + } + + const tallies = new Map(); + const counted = yield* sql<{ + machineId: string; + hour: number; + sessionId: string; + kind: MetricsTally["kind"]; + name: string; + count: number; + }>` + SELECT machine_id, hour, session_id, kind, name, SUM(count) AS count FROM tally WHERE ${filter} + GROUP BY machine_id, hour, session_id, kind, name + `; + for (const row of counted) { + const day = inPeriod(row.hour); + if (day === undefined) continue; + const { machineId, sessionId, kind, name, count } = row; + const tally = { day, machineId, sessionId, kind, name, count }; + add(tallies, [day, machineId, sessionId, kind, name], tally, (sum, next) => ({ + ...sum, + count: sum.count + next.count, + })); + } + + // Threads are summed away above, so the ones each session worked on a day are counted apart. + const worked = yield* sql<{ + machineId: string; + hour: number; + sessionId: string; + threadId: string; + projectId: string; + }>` + SELECT machine_id, hour, session_id, thread_id, project_id FROM usage WHERE ${filter} + UNION SELECT machine_id, hour, session_id, thread_id, project_id FROM tally WHERE ${filter} + `; + const threads = new Map }>(); + const sessions = new Map(); + for (const row of worked) { + const day = inPeriod(row.hour); + if (day === undefined) continue; + const ids = new Set([row.threadId]); + add( + threads, + [day, row.machineId, row.sessionId], + { day, machineId: row.machineId, sessionId: row.sessionId, ids }, + (sum, next) => { + for (const id of next.ids) sum.ids.add(id); + return sum; + }, + ); + sessions.set(row.sessionId, { id: row.sessionId, machineId: row.machineId, projectId: row.projectId }); + } + + const ids = [...sessions.keys()]; + const projectIds = [...new Set([...sessions.values()].map((session) => session.projectId))]; + const machineIds = [...new Set([...sessions.values()].map((session) => session.machineId))]; + const named = + ids.length === 0 + ? [] + : yield* sql<{ id: string; deletedAt: number | null; title: string | null }>` + SELECT session.id, session.deleted_at, thread.title FROM session + LEFT JOIN thread ON thread.session_id = session.id AND thread.origin_kind = 'main' + WHERE session.id IN ${sql.in(ids)} + `; + const titles = new Map(named.map((row) => [row.id, row])); + const projects = + projectIds.length === 0 + ? [] + : yield* sql<{ id: string; name: string }>` + SELECT id, name FROM project WHERE id IN ${sql.in(projectIds)} ORDER BY updated_at DESC, machine_id + `; + const projectNames = new Map(); + for (const project of projects) if (!projectNames.has(project.id)) projectNames.set(project.id, project.name); + const machines = + machineIds.length === 0 + ? [] + : yield* sql` + SELECT machine.id, ${sql.literal(Reads.MACHINE)} FROM machine WHERE machine.id IN ${sql.in(machineIds)} + ORDER BY machine.id + `; + + return { + rows: ordered(rows), + tallies: ordered(tallies), + threads: ordered(threads).map(({ ids, ...row }) => ({ ...row, threads: ids.size })), + sessions: ordered(sessions).map((owner): MetricsSession => { + const session = titles.get(owner.id); + const deleted = session === undefined || session.deletedAt !== null; + return { ...owner, title: deleted ? null : (session.title ?? UNTITLED), deleted }; + }), + projects: projectIds.toSorted().map((id) => ({ id, name: projectNames.get(id) ?? null })), + machines: machines.map((machine) => ({ id: machine.id, name: Reads.machineName(machine) })), + edges: [...edges], + } satisfies MetricsUsage; + }).pipe(Effect.orDie); +}); + +export const hours = Effect.fn("Metrics.hours")(function* (request: Scope & { readonly day: string }) { + const { local, from, to } = yield* period(request.zone, request.day, request.day); + return yield* Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const hours = new Map(); + for (const row of yield* hourly(sql, request, from, to)) { + const { day, hour } = local(row.hour); + if (day !== request.day) continue; + // A clock change's repeated hour is one wall-clock hour: both of its UTC hours sum into it. + add( + hours, + [String(hour).padStart(2, "0"), row.machineId, row.sessionId, row.provider, row.model], + { + hour, + machineId: row.machineId, + sessionId: row.sessionId, + provider: row.provider, + model: row.model, + requests: row.requests, + tokens: row.input + row.output + row.cacheRead + row.cacheWrite, + cost: row.cost, + }, + (sum, next) => ({ + ...sum, + requests: sum.requests + next.requests, + tokens: sum.tokens + next.tokens, + cost: sum.cost + next.cost, + }), + ); + } + return ordered(hours); + }).pipe(Effect.orDie); +}); + +export * as Metrics from "./metrics.ts"; diff --git a/apps/server/src/mirror.ts b/apps/server/src/mirror.ts index 4a95ee5b..40292a5d 100644 --- a/apps/server/src/mirror.ts +++ b/apps/server/src/mirror.ts @@ -18,6 +18,7 @@ import { type IndexRow = Exclude; type ThreadRow = Exclude; +type UsageRow = Exclude; type Append = (input: Publish) => Effect.Effect; /** The rows a batch wrote, for its `SyncChanged` notice. */ @@ -274,6 +275,51 @@ export const unseen = (machineId: string, seen: Changes, now: number) => `; }); +/** Applies one `usage.sync` batch of a machine's store: each row replaces the one at its key, running totals included. */ +export const usage = (machineId: string, storeId: string, rows: ReadonlyArray) => + Effect.forEach( + rows, + (row) => { + const owner = { + machineId, + storeId, + hour: row.hour, + threadId: row.threadId, + sessionId: row.sessionId, + projectId: row.projectId, + rev: row.rev, + }; + if (row._tag === "Tally") + return upsert("tally", ["machine_id", "store_id", "hour", "thread_id", "kind", "name"], { + ...owner, + kind: row.kind, + name: row.name, + count: row.count, + }); + return upsert("usage", ["machine_id", "store_id", "hour", "thread_id", "provider", "model", "thinking"], { + ...owner, + provider: row.provider, + model: row.model, + thinking: row.thinking, + requests: row.requests, + input: row.tokens.input, + output: row.tokens.output, + cacheRead: row.tokens.cacheRead, + cacheWrite: row.tokens.cacheWrite, + reasoning: row.tokens.reasoning, + cost: row.cost.total, + costInput: row.cost.input, + costOutput: row.cost.output, + costCacheRead: row.cost.cacheRead, + costCacheWrite: row.cost.cacheWrite, + duration: row.duration, + timed: row.timed, + durations: JSON.stringify(row.durations), + }); + }, + { discard: true }, + ); + /** Stores entries of a thread's line, each under the thread that wrote it, with its parts. */ export const entries = (list: ReadonlyArray, changes: Changes) => Effect.gen(function* () { diff --git a/apps/server/src/reads.ts b/apps/server/src/reads.ts index b411a0cc..2b4f68f1 100644 --- a/apps/server/src/reads.ts +++ b/apps/server/src/reads.ts @@ -17,6 +17,7 @@ import { type ThreadStatus, } from "@codeworksh/contract"; import { HistoryEntry, Part, QueuedPrompt, RunError, UNTITLED } from "@codeworksh/widget/codework"; +import { Servers } from "./servers.ts"; /* * What the UI reads from app.db: the sidebar and a thread's line. Neither calls CodeWork, so both @@ -35,6 +36,18 @@ const decodeThread = Schema.decodeUnknownEffect(LineThread); const model = (provider: string | null, id: string | null) => provider === null || id === null ? {} : { model: { provider, id } }; +/** The machine columns a read selects to name it by {@link Servers.nameOf}. */ +export const MACHINE = + "machine.kind AS machine_kind, machine.name AS machine_override, machine.host AS machine_host, machine.url AS machine_url"; +export interface MachineColumns { + readonly machineKind: string; + readonly machineOverride: string | null; + readonly machineHost: string | null; + readonly machineUrl: string | null; +} +export const machineName = (row: MachineColumns) => + Servers.nameOf({ kind: row.machineKind, override: row.machineOverride, host: row.machineHost, url: row.machineUrl }); + /** The machine filter of `sidebar.list`. */ const onMachines = (sql: SqlClient.SqlClient, machines: "default" | "all") => machines === "all" ? sql.literal("") : sql`AND machine.is_default = 1`; @@ -47,16 +60,17 @@ export const sidebar = Effect.fn("Reads.sidebar")(function* ({ readonly machines: "default" | "all"; }) { const sql = yield* SqlClient.SqlClient; - const projects = yield* sql<{ - machineId: string; - machineName: string; - id: string; - name: string; - location: string | null; - createdAt: number; - updatedAt: number; - }>` - SELECT project.machine_id, machine.name AS machine_name, project.id, project.name, project.created_at, project.updated_at, + const projects = yield* sql< + MachineColumns & { + machineId: string; + id: string; + name: string; + location: string | null; + createdAt: number; + updatedAt: number; + } + >` + SELECT project.machine_id, ${sql.literal(MACHINE)}, project.id, project.name, project.created_at, project.updated_at, (SELECT location FROM space WHERE space.machine_id = project.machine_id AND space.project_id = project.id ORDER BY space.kind = 'primary' DESC, space.created_at LIMIT 1) AS location FROM project JOIN machine ON machine.id = project.machine_id @@ -122,7 +136,7 @@ export const sidebar = Effect.fn("Reads.sidebar")(function* ({ .map((project): SidebarProject => { const list = sessionsOf(project); return { - machine: { id: project.machineId, name: project.machineName }, + machine: { id: project.machineId, name: machineName(project) }, id: project.id, name: project.name, // A worktree session's folder is not the project's, so new threads start in the primary space. @@ -177,26 +191,27 @@ export const threads = Effect.fn("Reads.threads")(function* ({ readonly by: ThreadsBy; }) { const sql = yield* SqlClient.SqlClient; - const rows = yield* sql<{ - id: string; - sessionId: string; - originKind: ListedThread["originKind"]; - originThreadId: string | null; - title: string | null; - machineId: string; - machineName: string; - projectId: string; - projectName: string; - runState: ListedThread["runState"]; - status: ThreadStatus; - priority: Priority; - groupingId: string | null; - ordering: number; - color: string | null; - updatedAt: number; - }>` + const rows = yield* sql< + MachineColumns & { + id: string; + sessionId: string; + originKind: ListedThread["originKind"]; + originThreadId: string | null; + title: string | null; + machineId: string; + projectId: string; + projectName: string; + runState: ListedThread["runState"]; + status: ThreadStatus; + priority: Priority; + groupingId: string | null; + ordering: number; + color: string | null; + updatedAt: number; + } + >` SELECT thread.id, thread.session_id, thread.origin_kind, thread.origin_thread_id, thread.title, - machine.id AS machine_id, machine.name AS machine_name, project.id AS project_id, project.name AS project_name, thread.run_state, thread.status, thread.priority, + machine.id AS machine_id, ${sql.literal(MACHINE)}, project.id AS project_id, project.name AS project_name, thread.run_state, thread.status, thread.priority, thread.grouping_id, thread.ordering, COALESCE(thread.color, session.color) AS color, thread.updated_at FROM thread JOIN session ON session.id = thread.session_id @@ -217,7 +232,7 @@ export const threads = Effect.fn("Reads.threads")(function* ({ originKind: row.originKind, originThreadId: row.originThreadId, title: titleOf(row) ?? UNTITLED, - machine: { id: row.machineId, name: row.machineName }, + machine: { id: row.machineId, name: machineName(row) }, project: { id: row.projectId, name: row.projectName }, runState: row.runState, status: row.status, diff --git a/apps/server/src/servers.ts b/apps/server/src/servers.ts index a644122e..78e0cdc2 100644 --- a/apps/server/src/servers.ts +++ b/apps/server/src/servers.ts @@ -20,12 +20,37 @@ export type StoredEnv = typeof StoredEnv.Type; export const splitArgs = (input: string) => [...input.matchAll(/"([^"]*)"|'([^']*)'|(\S+)/g)].map((match) => match[1] ?? match[2] ?? match[3] ?? ""); +/** + * What a machine is called: the user's override, else the name its server reports, else where it is. + * The one rule every machine name shown goes through. + */ +export const nameOf = (machine: { + readonly kind: string; + readonly override: string | null; + readonly host: string | null; + readonly url: string | null; +}) => + machine.override ?? + machine.host ?? + (machine.kind === "remote" && machine.url !== null && URL.canParse(machine.url) + ? new URL(machine.url).host + : "This Machine"); + +/** How a machine is named, as stored and as shown. */ +interface Naming { + /** As shown: {@link nameOf}. */ + readonly name: string; + /** As its server last reported it. */ + readonly host: string | null; + /** As the user named it in Settings. */ + readonly override: string | null; +} + /** A local server as stored. A `null` executable or home is the one found. An executable is a path, or a * command whose first word is one (`node --conditions=development …/index.ts`, running from source). */ -export interface LocalRow { +export interface LocalRow extends Naming { readonly id: string; readonly kind: "local"; - readonly name: string; readonly executable: string | null; readonly home: string | null; readonly args: string; @@ -34,10 +59,9 @@ export interface LocalRow { } /** A remote server as stored: where the app reaches it. */ -export interface RemoteRow { +export interface RemoteRow extends Naming { readonly id: string; readonly kind: "remote"; - readonly name: string; readonly url: string; readonly isDefault: boolean; } @@ -136,6 +160,8 @@ export class Service extends Context.Service< readonly homeChanges: Stream.Stream; /** Every server now, then again after each change to any of them. */ readonly watch: Stream.Stream>; + /** Stores the name a server reports, when it is not the one stored. */ + readonly setHost: (id: string, host: string | null) => Effect.Effect; readonly add: (input: ServerAdd) => Effect.Effect; readonly update: (input: ServerUpdate) => Effect.Effect; readonly remove: (id: string) => Effect.Effect; @@ -151,23 +177,30 @@ export const layer = Layer.effect( const fallback = yield* cliHome.pipe(Effect.orDie); const list = sql< - Omit & { + Omit & { kind: string; env: string; url: string | null; isDefault: number; } >` - SELECT id, kind, name, executable, home, args, env, url, is_default FROM machine ORDER BY pos + SELECT id, kind, name AS override, host, executable, home, args, env, url, is_default FROM machine ORDER BY pos `.pipe( Effect.flatMap((rows) => - Effect.forEach(rows, ({ kind, url, env, isDefault, ...row }) => - kind === "remote" && url !== null - ? Effect.succeed({ id: row.id, kind, name: row.name, url, isDefault: isDefault === 1 }) + Effect.forEach(rows, ({ kind, url, env, isDefault, ...row }) => { + const naming = { name: nameOf({ kind, url, ...row }), host: row.host, override: row.override }; + return kind === "remote" && url !== null + ? Effect.succeed({ id: row.id, kind, ...naming, url, isDefault: isDefault === 1 }) : decodeEnv(env).pipe( - Effect.map((env): Row => ({ ...row, kind: "local", env, isDefault: isDefault === 1 })), - ), - ), + Effect.map((env): Row => ({ + ...row, + ...naming, + kind: "local", + env, + isDefault: isDefault === 1, + })), + ); + }), ), Effect.orDie, ); @@ -218,9 +251,10 @@ export const layer = Layer.effect( }), ); - const name = (input: string) => { - const trimmed = input.trim(); - return trimmed === "" ? Effect.fail(fail("A server needs a name.")) : Effect.succeed(trimmed); + /** A name typed in Settings; blank is no override. */ + const override = (input: string | undefined) => { + const trimmed = input?.trim() ?? ""; + return trimmed === "" ? null : trimmed; }; const transaction = (effect: Effect.Effect) => @@ -234,6 +268,12 @@ export const layer = Layer.effect( defaultHome, homeChanges: SubscriptionRef.changes(current).pipe(Stream.drop(1)), watch: SubscriptionRef.changes(machines), + setHost: Effect.fn("Servers.setHost")(function* (id, host) { + const changed = yield* sql` + UPDATE machine SET host = ${host} WHERE id = ${id} AND host IS NOT ${host} RETURNING id + `.pipe(Effect.orDie); + if (changed.length > 0) yield* announce; + }), add: Effect.fn("Servers.add")(function* (input) { const id = yield* Effect.sync(() => crypto.randomUUID().slice(0, 8)); @@ -250,7 +290,7 @@ export const layer = Layer.effect( yield* sql`INSERT INTO machine ${sql.insert({ id, kind: input.kind, - name: yield* name(input.name), + name: override(input.name), ...route, namespace: "codework", pos: (last?.pos ?? 0) + 1, @@ -267,7 +307,7 @@ export const layer = Layer.effect( Effect.gen(function* () { const row = yield* get(id); const fields: Record = {}; - if (patch.name !== undefined) fields.name = yield* name(patch.name); + if (patch.name !== undefined) fields.name = override(patch.name); if (row.kind === "remote") { if ([patch.executable, patch.home, patch.args, patch.env].some((value) => value !== undefined)) return yield* fail("A remote server has no executable, home, arguments or variables."); diff --git a/apps/server/src/sync.ts b/apps/server/src/sync.ts index eb12c276..7b02ad05 100644 --- a/apps/server/src/sync.ts +++ b/apps/server/src/sync.ts @@ -34,6 +34,12 @@ type IndexSynced = Extract; type IndexRow = Exclude; type ThreadSynced = Extract; type ThreadRow = Exclude; +type UsageSynced = Extract; +type UsageRow = Exclude; + +/** The RPC server's answer to a call it has no handler for: a server older than `usage.sync`. */ +const unserved = (cause: Cause.Cause) => + cause.reasons.some((reason) => Cause.isDieReason(reason) && reason.defect === "Unknown request tag: usage.sync"); /** A machine whose index is current: its connection and the threads whose detail it syncs. */ interface Live { @@ -62,7 +68,7 @@ export interface Interface { /** * Keeps app.db a copy of every machine's CodeWork: each machine over its own connection, its index - * through `index.sync` from `machine.cursor`, and the detail of opened threads through `thread.sync` + * through `index.sync` from `machine.cursor`, its usage rollup through `usage.sync` from `machine.rollup`, and the detail of opened threads through `thread.sync` * from `thread.cursor`. A new or restored store is read again from zero. */ export class Service extends Context.Service()("@codeworksh/server/sync/Service") {} @@ -303,23 +309,29 @@ export const layer = Layer.effect( Effect.forever, ); + /** Waits forever when another machine already syncs `storeId`: the same server added twice. */ + const unclaimed = (machineId: string, storeId: string) => + Effect.gen(function* () { + const [owner] = yield* sql<{ id: string }>` + SELECT id FROM machine WHERE store_id = ${storeId} AND id <> ${machineId} + `; + if (owner === undefined) return; + yield* Effect.logWarning( + `Not syncing machine ${machineId}: its store is already synced as machine ${owner.id}`, + ); + return yield* Effect.never; + }); + /** One connection's sync: decides between resume and full sync, then follows the index. */ const connected = (machineId: string, client: Connection) => Effect.gen(function* () { const info = yield* client("server.info", {}); + yield* servers.setHost(machineId, info.host?.name ?? null); const [machine] = yield* sql<{ storeId: string | null; cursor: number; namespace: string }>` SELECT store_id, cursor, namespace FROM machine WHERE id = ${machineId} `; if (machine === undefined) return yield* Effect.never; - const [owner] = yield* sql<{ id: string }>` - SELECT id FROM machine WHERE store_id = ${info.storeId} AND id <> ${machineId} - `; - if (owner !== undefined) { - yield* Effect.logWarning( - `Not syncing machine ${machineId}: its store is already synced as machine ${owner.id}`, - ); - return yield* Effect.never; - } + yield* unclaimed(machineId, info.storeId); let full = info.storeId !== machine.storeId || info.rev < machine.cursor; const live: Live = { client, threads: yield* FiberMap.make() }; yield* Effect.addFinalizer(() => @@ -372,6 +384,71 @@ export const layer = Layer.effect( return yield* Effect.fail("The index stream ended"); }).pipe(Effect.scoped); + /** + * One connection's usage rollup: from `machine.rollup` while the store is the one it counts in, + * else from zero. A store that went back (a restored backup) replaces its own rows, in the + * transaction of its first batch so a failed re-read leaves the old ones. + */ + const syncUsage = (machineId: string, client: Connection) => + Effect.gen(function* () { + const info = yield* client("server.info", {}); + const [machine] = yield* sql<{ rollup: number; rollupStore: string | null }>` + SELECT rollup, rollup_store FROM machine WHERE id = ${machineId} + `; + if (machine === undefined) return yield* Effect.never; + yield* unclaimed(machineId, info.storeId); + const same = info.storeId === machine.rollupStore; + let restored = same && info.rev < machine.rollup; + yield* batches( + client("usage.sync", { after: same && !restored ? machine.rollup : 0 }), + (item) => item._tag === "Synced", + () => Effect.succeed(true), + (rows, synced) => + Effect.gen(function* () { + if (synced.storeId !== info.storeId) return yield* Effect.fail("The usage stream changed store"); + yield* transact((append) => + Effect.gen(function* () { + if (restored) { + yield* sql`DELETE FROM usage WHERE machine_id = ${machineId} AND store_id = ${synced.storeId}`; + yield* sql`DELETE FROM tally WHERE machine_id = ${machineId} AND store_id = ${synced.storeId}`; + } + yield* Mirror.usage(machineId, synced.storeId, rows); + yield* sql` + UPDATE machine SET rollup = ${synced.rev}, rollup_store = ${synced.storeId} WHERE id = ${machineId} + `; + if (rows.length > 0 || restored) + yield* append({ + topic: syncTopics.usage, + source: "sync", + payload: { machineId, storeId: synced.storeId, items: rows.length }, + }); + }), + ); + restored = false; + }), + ); + return yield* Effect.fail("The usage stream ended"); + }); + + /** + * {@link syncUsage} for as long as the connection lasts, apart from the index: either may fail + * and retry without the other. A server older than `usage.sync` is asked again only after a while. + */ + const followUsage = (machineId: string, client: Connection) => + syncUsage(machineId, client).pipe( + Effect.catchCause((cause) => + Cause.hasInterruptsOnly(cause) + ? Effect.interrupt + : unserved(cause) + ? Effect.logInfo(`Machine ${machineId} does not serve usage.sync`).pipe( + Effect.andThen(Effect.sleep("1 minute")), + ) + : Effect.logWarning(`Usage sync of machine ${machineId} stopped`, cause), + ), + Effect.andThen(Effect.sleep("1 second")), + Effect.forever, + ); + /** A machine for as long as it is configured: connected, and reconnected after any drop. */ const follow = (row: Servers.Row) => Effect.gen(function* () { @@ -382,6 +459,7 @@ export const layer = Layer.effect( const client = Context.get(context, Client); clients.set(row.id, client); yield* Effect.addFinalizer(() => Effect.sync(() => clients.delete(row.id))); + yield* Effect.forkScoped(followUsage(row.id, client)); yield* connected(row.id, client).pipe( Effect.catchCause((cause) => Effect.logWarning("Machine sync stopped; reconnecting", cause).pipe( diff --git a/apps/server/test/__artifacts__/host.migrated.json b/apps/server/test/__artifacts__/host.migrated.json new file mode 100644 index 00000000..c937d927 --- /dev/null +++ b/apps/server/test/__artifacts__/host.migrated.json @@ -0,0 +1,30 @@ +{ + "stored": [ + { + "id": "", + "kind": "local", + "name": null, + "host": null + }, + { + "id": "", + "kind": "remote", + "name": "Box", + "host": null + } + ], + "servers": [ + { + "id": "", + "name": "This Machine", + "host": null, + "override": null + }, + { + "id": "", + "name": "Box", + "host": null, + "override": "Box" + } + ] +} diff --git a/apps/server/test/__artifacts__/host.names.json b/apps/server/test/__artifacts__/host.names.json new file mode 100644 index 00000000..56b7283e --- /dev/null +++ b/apps/server/test/__artifacts__/host.names.json @@ -0,0 +1,178 @@ +{ + "reported": [ + { + "stored": { + "id": "", + "kind": "local", + "name": null, + "host": "Studio Mac" + }, + "id": "", + "name": "Studio Mac", + "host": "Studio Mac", + "override": null, + "sidebar": "Studio Mac", + "allThreads": "Studio Mac" + }, + { + "stored": { + "id": "", + "kind": "remote", + "name": null, + "host": "ubuntu" + }, + "id": "", + "name": "ubuntu", + "host": "ubuntu", + "override": null, + "sidebar": "ubuntu", + "allThreads": "ubuntu" + }, + { + "stored": { + "id": "", + "kind": "remote", + "name": null, + "host": null + }, + "id": "", + "name": "127.0.0.1:", + "host": null, + "override": null, + "sidebar": "127.0.0.1:", + "allThreads": "127.0.0.1:" + } + ], + "overridden": [ + { + "stored": { + "id": "", + "kind": "local", + "name": null, + "host": "Studio Mac" + }, + "id": "", + "name": "Studio Mac", + "host": "Studio Mac", + "override": null, + "sidebar": "Studio Mac", + "allThreads": "Studio Mac" + }, + { + "stored": { + "id": "", + "kind": "remote", + "name": "Build Box", + "host": "ubuntu" + }, + "id": "", + "name": "Build Box", + "host": "ubuntu", + "override": "Build Box", + "sidebar": "Build Box", + "allThreads": "Build Box" + }, + { + "stored": { + "id": "", + "kind": "remote", + "name": null, + "host": null + }, + "id": "", + "name": "127.0.0.1:", + "host": null, + "override": null, + "sidebar": "127.0.0.1:", + "allThreads": "127.0.0.1:" + } + ], + "cleared": [ + { + "stored": { + "id": "", + "kind": "local", + "name": null, + "host": "Studio Mac" + }, + "id": "", + "name": "Studio Mac", + "host": "Studio Mac", + "override": null, + "sidebar": "Studio Mac", + "allThreads": "Studio Mac" + }, + { + "stored": { + "id": "", + "kind": "remote", + "name": null, + "host": "ubuntu" + }, + "id": "", + "name": "ubuntu", + "host": "ubuntu", + "override": null, + "sidebar": "ubuntu", + "allThreads": "ubuntu" + }, + { + "stored": { + "id": "", + "kind": "remote", + "name": null, + "host": null + }, + "id": "", + "name": "127.0.0.1:", + "host": null, + "override": null, + "sidebar": "127.0.0.1:", + "allThreads": "127.0.0.1:" + } + ], + "renamed": [ + { + "stored": { + "id": "", + "kind": "local", + "name": null, + "host": "Studio Mac" + }, + "id": "", + "name": "Studio Mac", + "host": "Studio Mac", + "override": null, + "sidebar": "Studio Mac", + "allThreads": "Studio Mac" + }, + { + "stored": { + "id": "", + "kind": "remote", + "name": null, + "host": "ubuntu-2" + }, + "id": "", + "name": "ubuntu-2", + "host": "ubuntu-2", + "override": null, + "sidebar": "ubuntu-2", + "allThreads": "ubuntu-2" + }, + { + "stored": { + "id": "", + "kind": "remote", + "name": null, + "host": null + }, + "id": "", + "name": "127.0.0.1:", + "host": null, + "override": null, + "sidebar": "127.0.0.1:", + "allThreads": "127.0.0.1:" + } + ] +} diff --git a/apps/server/test/__artifacts__/metrics.filters.json b/apps/server/test/__artifacts__/metrics.filters.json new file mode 100644 index 00000000..10ca22f6 --- /dev/null +++ b/apps/server/test/__artifacts__/metrics.filters.json @@ -0,0 +1,415 @@ +{ + "Host A": { + "rows": [ + { + "day": "1970-01-01", + "machineId": "", + "sessionId": "", + "provider": "openai", + "model": "gpt-5.6-luna", + "thinking": "medium", + "requests": 2, + "tokens": { + "input": 2, + "output": 2, + "cacheRead": 0, + "cacheWrite": 0, + "reasoning": 0 + }, + "cost": { + "total": 0, + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "duration": 0, + "timed": 2, + "durations": [ + 2, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + }, + { + "day": "1970-01-01", + "machineId": "", + "sessionId": "", + "provider": "openai", + "model": "gpt-5.6-luna", + "thinking": "medium", + "requests": 2, + "tokens": { + "input": 2, + "output": 2, + "cacheRead": 0, + "cacheWrite": 0, + "reasoning": 0 + }, + "cost": { + "total": 0, + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "duration": 0, + "timed": 2, + "durations": [ + 2, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + } + ], + "tallies": [ + { + "day": "1970-01-01", + "machineId": "", + "sessionId": "", + "kind": "tool", + "name": "bash", + "count": 1 + }, + { + "day": "1970-01-01", + "machineId": "", + "sessionId": "", + "kind": "tool", + "name": "bash", + "count": 1 + }, + { + "day": "", + "machineId": "", + "sessionId": "", + "kind": "prompt", + "name": "", + "count": 1 + }, + { + "day": "", + "machineId": "", + "sessionId": "", + "kind": "prompt", + "name": "", + "count": 1 + } + ], + "threads": [ + { + "day": "1970-01-01", + "machineId": "", + "sessionId": "", + "threads": 1 + }, + { + "day": "1970-01-01", + "machineId": "", + "sessionId": "", + "threads": 1 + }, + { + "day": "", + "machineId": "", + "sessionId": "", + "threads": 1 + }, + { + "day": "", + "machineId": "", + "sessionId": "", + "threads": 1 + } + ], + "sessions": [ + { + "id": "", + "machineId": "", + "projectId": "", + "title": "bash echo alpha", + "deleted": false + }, + { + "id": "", + "machineId": "", + "projectId": "", + "title": "bash echo beta", + "deleted": false + } + ], + "projects": [ + { + "id": "", + "name": "Alpha" + }, + { + "id": "", + "name": "Beta" + } + ], + "machines": [ + { + "id": "", + "name": "Host A" + } + ], + "edges": [ + 250, + 374, + 561, + 839, + 1257, + 1882, + 2818, + 4220, + 6318, + 9461, + 14167, + 21213, + 31764, + 47563, + 71221, + 106645, + 159688, + 239115, + 358046, + 536133, + 802797, + 1202096, + 1800000 + ] + }, + "Host B": { + "rows": [ + { + "day": "1970-01-01", + "machineId": "", + "sessionId": "", + "provider": "openai", + "model": "gpt-5.6-luna", + "thinking": "medium", + "requests": 2, + "tokens": { + "input": 2, + "output": 2, + "cacheRead": 0, + "cacheWrite": 0, + "reasoning": 0 + }, + "cost": { + "total": 0, + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "duration": 0, + "timed": 2, + "durations": [ + 2, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + } + ], + "tallies": [ + { + "day": "1970-01-01", + "machineId": "", + "sessionId": "", + "kind": "tool", + "name": "bash", + "count": 1 + }, + { + "day": "", + "machineId": "", + "sessionId": "", + "kind": "prompt", + "name": "", + "count": 1 + } + ], + "threads": [ + { + "day": "1970-01-01", + "machineId": "", + "sessionId": "", + "threads": 1 + }, + { + "day": "", + "machineId": "", + "sessionId": "", + "threads": 1 + } + ], + "sessions": [ + { + "id": "", + "machineId": "", + "projectId": "", + "title": "bash echo gamma", + "deleted": false + } + ], + "projects": [ + { + "id": "", + "name": "Gamma" + } + ], + "machines": [ + { + "id": "", + "name": "Host B" + } + ], + "edges": [ + 250, + 374, + 561, + 839, + 1257, + 1882, + 2818, + 4220, + 6318, + 9461, + 14167, + 21213, + 31764, + 47563, + 71221, + 106645, + 159688, + 239115, + 358046, + 536133, + 802797, + 1202096, + 1800000 + ] + }, + "all": { + "rows": 3, + "machines": 2, + "projects": [ + { + "id": "", + "name": "Alpha" + }, + { + "id": "", + "name": "Beta" + }, + { + "id": "", + "name": "Gamma" + } + ] + }, + "alpha": { + "sessions": [ + { + "id": "", + "machineId": "", + "projectId": "", + "title": "bash echo alpha", + "deleted": false + } + ], + "projects": [ + { + "id": "", + "name": "Alpha" + } + ] + }, + "hoursA": [ + { + "hour": 0, + "machineId": "", + "sessionId": "", + "provider": "openai", + "model": "gpt-5.6-luna", + "requests": 2, + "tokens": 4, + "cost": 0 + }, + { + "hour": 0, + "machineId": "", + "sessionId": "", + "provider": "openai", + "model": "gpt-5.6-luna", + "requests": 2, + "tokens": 4, + "cost": 0 + } + ] +} diff --git a/apps/server/test/__artifacts__/metrics.stores.json b/apps/server/test/__artifacts__/metrics.stores.json new file mode 100644 index 00000000..eeb50460 --- /dev/null +++ b/apps/server/test/__artifacts__/metrics.stores.json @@ -0,0 +1,223 @@ +{ + "rows": [ + { + "day": "1970-01-01", + "machineId": "", + "sessionId": "", + "provider": "openai", + "model": "gpt-5.6-luna", + "thinking": "medium", + "requests": 2, + "tokens": { + "input": 2, + "output": 2, + "cacheRead": 0, + "cacheWrite": 0, + "reasoning": 0 + }, + "cost": { + "total": 0, + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "duration": 0, + "timed": 2, + "durations": [ + 2, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + }, + { + "day": "1970-01-01", + "machineId": "", + "sessionId": "", + "provider": "openai", + "model": "gpt-5.6-luna", + "thinking": "medium", + "requests": 2, + "tokens": { + "input": 2, + "output": 2, + "cacheRead": 0, + "cacheWrite": 0, + "reasoning": 0 + }, + "cost": { + "total": 0, + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "duration": 0, + "timed": 2, + "durations": [ + 2, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + } + ], + "tallies": [ + { + "day": "1970-01-01", + "machineId": "", + "sessionId": "", + "kind": "tool", + "name": "bash", + "count": 1 + }, + { + "day": "1970-01-01", + "machineId": "", + "sessionId": "", + "kind": "tool", + "name": "bash", + "count": 1 + }, + { + "day": "", + "machineId": "", + "sessionId": "", + "kind": "prompt", + "name": "", + "count": 1 + }, + { + "day": "", + "machineId": "", + "sessionId": "", + "kind": "prompt", + "name": "", + "count": 1 + } + ], + "threads": [ + { + "day": "1970-01-01", + "machineId": "", + "sessionId": "", + "threads": 1 + }, + { + "day": "1970-01-01", + "machineId": "", + "sessionId": "", + "threads": 1 + }, + { + "day": "", + "machineId": "", + "sessionId": "", + "threads": 1 + }, + { + "day": "", + "machineId": "", + "sessionId": "", + "threads": 1 + } + ], + "sessions": [ + { + "id": "", + "machineId": "", + "projectId": "", + "title": null, + "deleted": true + }, + { + "id": "", + "machineId": "", + "projectId": "", + "title": "bash echo new", + "deleted": false + } + ], + "projects": [ + { + "id": "", + "name": "New" + }, + { + "id": "", + "name": "Old" + } + ], + "machines": [ + { + "id": "", + "name": "Fixture" + } + ], + "edges": [ + 250, + 374, + 561, + 839, + 1257, + 1882, + 2818, + 4220, + 6318, + 9461, + 14167, + 21213, + 31764, + 47563, + 71221, + 106645, + 159688, + 239115, + 358046, + 536133, + 802797, + 1202096, + 1800000 + ] +} diff --git a/apps/server/test/__artifacts__/metrics.usage.json b/apps/server/test/__artifacts__/metrics.usage.json new file mode 100644 index 00000000..6ed95683 --- /dev/null +++ b/apps/server/test/__artifacts__/metrics.usage.json @@ -0,0 +1,271 @@ +{ + "notice": { + "machineId": "", + "storeId": "", + "items": 1 + }, + "reported": [ + { + "cost": 0, + "input": 2, + "output": 2, + "cacheRead": 0, + "cacheWrite": 0 + }, + { + "cost": 0, + "input": 2, + "output": 2, + "cacheRead": 0, + "cacheWrite": 0 + }, + { + "cost": 0, + "input": 2, + "output": 2, + "cacheRead": 0, + "cacheWrite": 0 + } + ], + "after": { + "rows": [ + { + "day": "1970-01-01", + "machineId": "", + "sessionId": "", + "provider": "openai", + "model": "gpt-5.6-luna", + "thinking": "medium", + "requests": 6, + "tokens": { + "input": 6, + "output": 6, + "cacheRead": 0, + "cacheWrite": 0, + "reasoning": 0 + }, + "cost": { + "total": 0, + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "duration": 0, + "timed": 6, + "durations": [ + 6, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + }, + { + "day": "1970-01-01", + "machineId": "", + "sessionId": "", + "provider": "openai", + "model": "gpt-5.6-luna", + "thinking": "medium", + "requests": 2, + "tokens": { + "input": 2, + "output": 2, + "cacheRead": 0, + "cacheWrite": 0, + "reasoning": 0 + }, + "cost": { + "total": 0, + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "duration": 0, + "timed": 2, + "durations": [ + 2, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + } + ], + "tallies": [ + { + "day": "1970-01-01", + "machineId": "", + "sessionId": "", + "kind": "tool", + "name": "bash", + "count": 2 + }, + { + "day": "1970-01-01", + "machineId": "", + "sessionId": "", + "kind": "tool", + "name": "bash", + "count": 1 + }, + { + "day": "", + "machineId": "", + "sessionId": "", + "kind": "prompt", + "name": "", + "count": 2 + }, + { + "day": "", + "machineId": "", + "sessionId": "", + "kind": "prompt", + "name": "", + "count": 1 + } + ], + "threads": [ + { + "day": "1970-01-01", + "machineId": "", + "sessionId": "", + "threads": 3 + }, + { + "day": "1970-01-01", + "machineId": "", + "sessionId": "", + "threads": 1 + }, + { + "day": "", + "machineId": "", + "sessionId": "", + "threads": 2 + }, + { + "day": "", + "machineId": "", + "sessionId": "", + "threads": 1 + } + ], + "sessions": [ + { + "id": "", + "machineId": "", + "projectId": "", + "title": "bash echo one", + "deleted": false + }, + { + "id": "", + "machineId": "", + "projectId": "", + "title": "bash echo two", + "deleted": false + } + ], + "projects": [ + { + "id": "", + "name": "Alpha" + }, + { + "id": "", + "name": "Beta" + } + ], + "machines": [ + { + "id": "", + "name": "Metrics Host" + } + ], + "edges": [ + 250, + 374, + 561, + 839, + 1257, + 1882, + 2818, + 4220, + 6318, + 9461, + 14167, + 21213, + 31764, + 47563, + 71221, + 106645, + 159688, + 239115, + 358046, + 536133, + 802797, + 1202096, + 1800000 + ] + }, + "deleted": { + "sessions": [ + { + "id": "", + "machineId": "", + "projectId": "", + "title": "bash echo one", + "deleted": false + }, + { + "id": "", + "machineId": "", + "projectId": "", + "title": null, + "deleted": true + } + ] + } +} diff --git a/apps/server/test/__artifacts__/metrics.zones.json b/apps/server/test/__artifacts__/metrics.zones.json new file mode 100644 index 00000000..9aff028f --- /dev/null +++ b/apps/server/test/__artifacts__/metrics.zones.json @@ -0,0 +1,410 @@ +{ + "kolkata": { + "both": { + "2026-03-09": 1, + "2026-03-10": 2 + }, + "first": { + "2026-03-09": 1 + }, + "second": { + "2026-03-10": 2 + }, + "tallies": [ + { + "day": "2026-03-10", + "machineId": "local", + "sessionId": "india", + "kind": "prompt", + "name": "", + "count": 1 + } + ], + "hoursFirst": [ + [ + 23, + 1 + ] + ], + "hoursSecond": [ + [ + 0, + 2 + ] + ] + }, + "kathmandu": { + "both": { + "2026-03-09": 1, + "2026-03-10": 2 + }, + "hours": [ + [ + 0, + 2 + ] + ] + }, + "utc": { + "2026-03-09": 3 + }, + "losAngeles": { + "both": { + "2026-03-09": 1, + "2026-03-10": 2 + }, + "second": { + "2026-03-10": 2 + }, + "hoursFirst": [ + [ + 23, + 1 + ] + ], + "hoursSecond": [ + [ + 0, + 2 + ] + ] + }, + "fallBack": { + "days": { + "2026-10-31": 1, + "2026-11-01": 25, + "2026-11-02": 1 + }, + "hours": [ + [ + 0, + 1 + ], + [ + 1, + 2 + ], + [ + 2, + 1 + ], + [ + 3, + 1 + ], + [ + 4, + 1 + ], + [ + 5, + 1 + ], + [ + 6, + 1 + ], + [ + 7, + 1 + ], + [ + 8, + 1 + ], + [ + 9, + 1 + ], + [ + 10, + 1 + ], + [ + 11, + 1 + ], + [ + 12, + 1 + ], + [ + 13, + 1 + ], + [ + 14, + 1 + ], + [ + 15, + 1 + ], + [ + 16, + 1 + ], + [ + 17, + 1 + ], + [ + 18, + 1 + ], + [ + 19, + 1 + ], + [ + 20, + 1 + ], + [ + 21, + 1 + ], + [ + 22, + 1 + ], + [ + 23, + 1 + ] + ] + }, + "springForward": { + "days": { + "2026-03-07": 1, + "2026-03-08": 23, + "2026-03-09": 1 + }, + "hours": [ + [ + 0, + 1 + ], + [ + 1, + 1 + ], + [ + 3, + 1 + ], + [ + 4, + 1 + ], + [ + 5, + 1 + ], + [ + 6, + 1 + ], + [ + 7, + 1 + ], + [ + 8, + 1 + ], + [ + 9, + 1 + ], + [ + 10, + 1 + ], + [ + 11, + 1 + ], + [ + 12, + 1 + ], + [ + 13, + 1 + ], + [ + 14, + 1 + ], + [ + 15, + 1 + ], + [ + 16, + 1 + ], + [ + 17, + 1 + ], + [ + 18, + 1 + ], + [ + 19, + 1 + ], + [ + 20, + 1 + ], + [ + 21, + 1 + ], + [ + 22, + 1 + ], + [ + 23, + 1 + ] + ] + }, + "stores": { + "rows": [ + { + "day": "2026-05-01", + "machineId": "local", + "sessionId": "stores", + "provider": "scripted", + "model": "model", + "thinking": "off", + "requests": 7, + "tokens": { + "input": 70, + "output": 35, + "cacheRead": 0, + "cacheWrite": 0, + "reasoning": 0 + }, + "cost": { + "total": 3.5, + "input": 1.75, + "output": 1.75, + "cacheRead": 0, + "cacheWrite": 0 + }, + "duration": 2100, + "timed": 7, + "durations": [ + 0, + 7, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + } + ], + "tallies": [], + "threads": [ + { + "day": "2026-05-01", + "machineId": "local", + "sessionId": "stores", + "threads": 2 + } + ], + "sessions": [ + { + "id": "stores", + "machineId": "local", + "projectId": "prj_stores", + "title": null, + "deleted": true + } + ], + "projects": [ + { + "id": "prj_stores", + "name": null + } + ], + "machines": [ + { + "id": "local", + "name": "This Machine" + } + ], + "edges": [ + 250, + 374, + 561, + 839, + 1257, + 1882, + 2818, + 4220, + 6318, + 9461, + 14167, + 21213, + 31764, + 47563, + 71221, + 106645, + 159688, + 239115, + 358046, + 536133, + 802797, + 1202096, + 1800000 + ] + }, + "refusals": { + "zone": { + "tag": "Server.MetricsError", + "message": "\"Mars/Olympus\" is not a time zone." + }, + "backwards": { + "tag": "Server.MetricsError", + "message": "The period ends (2026-03-09) before it starts (2026-03-10)." + }, + "malformed": { + "tag": "Server.MetricsError", + "message": "\"2026-3-9\" is not a day: expected YYYY-MM-DD." + }, + "impossible": { + "tag": "Server.MetricsError", + "message": "\"2026-02-30\" is not a day: expected YYYY-MM-DD." + }, + "hoursZone": { + "tag": "Server.MetricsError", + "message": "\"\" is not a time zone." + } + } +} diff --git a/apps/server/test/__artifacts__/reads.machines.json b/apps/server/test/__artifacts__/reads.machines.json index 87b8ffa4..112e8e85 100644 --- a/apps/server/test/__artifacts__/reads.machines.json +++ b/apps/server/test/__artifacts__/reads.machines.json @@ -3,7 +3,7 @@ { "machine": { "id": "", - "name": "CodeWork" + "name": "Fixture" }, "id": "", "name": "Local", @@ -30,7 +30,7 @@ { "machine": { "id": "", - "name": "CodeWork" + "name": "Fixture" }, "id": "", "name": "Local", diff --git a/apps/server/test/__artifacts__/schema.json b/apps/server/test/__artifacts__/schema.json index f5da9b3b..6fbeef63 100644 --- a/apps/server/test/__artifacts__/schema.json +++ b/apps/server/test/__artifacts__/schema.json @@ -70,7 +70,7 @@ "type": "table", "name": "machine", "tbl_name": "machine", - "sql": "CREATE TABLE machine ( id TEXT PRIMARY KEY, kind TEXT NOT NULL, name TEXT NOT NULL, executable TEXT, home TEXT, args TEXT NOT NULL DEFAULT '', env TEXT NOT NULL DEFAULT '[]', url TEXT, namespace TEXT NOT NULL, is_default INTEGER NOT NULL DEFAULT 0, pos INTEGER NOT NULL, store_id TEXT UNIQUE, cursor INTEGER NOT NULL DEFAULT 0, created_at INTEGER NOT NULL )" + "sql": "CREATE TABLE machine ( id TEXT PRIMARY KEY, kind TEXT NOT NULL, executable TEXT, home TEXT, args TEXT NOT NULL DEFAULT '', env TEXT NOT NULL DEFAULT '[]', url TEXT, namespace TEXT NOT NULL, is_default INTEGER NOT NULL DEFAULT 0, pos INTEGER NOT NULL, store_id TEXT UNIQUE, cursor INTEGER NOT NULL DEFAULT 0, created_at INTEGER NOT NULL , rollup INTEGER NOT NULL DEFAULT 0, rollup_store TEXT, host TEXT, name TEXT)" }, { "type": "table", @@ -150,6 +150,12 @@ "tbl_name": "sqlite_sequence", "sql": "CREATE TABLE sqlite_sequence(name,seq)" }, + { + "type": "table", + "name": "tally", + "tbl_name": "tally", + "sql": "CREATE TABLE tally ( machine_id TEXT NOT NULL REFERENCES machine(id) ON DELETE CASCADE, store_id TEXT NOT NULL, hour INTEGER NOT NULL, thread_id TEXT NOT NULL, kind TEXT NOT NULL, name TEXT NOT NULL, session_id TEXT NOT NULL, project_id TEXT NOT NULL, count INTEGER NOT NULL, rev INTEGER NOT NULL, PRIMARY KEY (machine_id, store_id, hour, thread_id, kind, name) )" + }, { "type": "table", "name": "thread", @@ -192,6 +198,12 @@ "tbl_name": "thread", "sql": "CREATE INDEX thread_status_idx ON thread (organization_id, status, ordering) WHERE deleted_at IS NULL" }, + { + "type": "table", + "name": "usage", + "tbl_name": "usage", + "sql": "CREATE TABLE usage ( machine_id TEXT NOT NULL REFERENCES machine(id) ON DELETE CASCADE, store_id TEXT NOT NULL, hour INTEGER NOT NULL, thread_id TEXT NOT NULL, provider TEXT NOT NULL, model TEXT NOT NULL, thinking TEXT NOT NULL, session_id TEXT NOT NULL, project_id TEXT NOT NULL, requests INTEGER NOT NULL, input INTEGER NOT NULL, output INTEGER NOT NULL, cache_read INTEGER NOT NULL, cache_write INTEGER NOT NULL, reasoning INTEGER NOT NULL, cost REAL NOT NULL, cost_input REAL NOT NULL, cost_output REAL NOT NULL, cost_cache_read REAL NOT NULL, cost_cache_write REAL NOT NULL, duration REAL NOT NULL, timed INTEGER NOT NULL, durations TEXT NOT NULL, rev INTEGER NOT NULL, PRIMARY KEY (machine_id, store_id, hour, thread_id, provider, model, thinking) )" + }, { "type": "table", "name": "widget", @@ -210,7 +222,6 @@ { "id": "local", "kind": "local", - "name": "CodeWork", "executable": null, "home": null, "args": "", @@ -221,7 +232,11 @@ "pos": 0, "store_id": null, "cursor": 0, - "created_at": "( throw new Error(`timed out waiting for ${what}`); }; -/** A CodeWork server of the fixture in `home` on `port`, with its scripted model. */ -export const codework = async (home: string, port: number) => { - const proc = spawn(process.execPath, ["--conditions=development", fixture, home, String(port)], { +/** + * A CodeWork server of the fixture in `home` on `port`, with its scripted model; `older` lacks + * `usage.sync` and a host name. `host` is the host name it reports (`null` for none), else `Fixture`. + */ +export const codework = async ( + home: string, + port: number, + options: { readonly older?: boolean; readonly host?: string | null } = {}, +) => { + const args = [fixture, home, String(port), ...(options.older === true ? ["127.0.0.1", "older"] : [])]; + const proc = spawn(process.execPath, ["--conditions=development", ...args], { + env: options.host === undefined ? process.env : { ...process.env, CODEWORK_FIXTURE_HOST: options.host ?? "" }, stdio: ["pipe", "pipe", "inherit"], }); processes.add(proc); @@ -108,6 +117,9 @@ export const app = async (home: string) => { stdio: ["ignore", "pipe", "inherit"], }); processes.add(proc); + // Its stdout, the log included, line by line. + const lines: Array = []; + createInterface({ input: proc.stdout! }).on("line", (line) => lines.push(line)); const { port } = await ready(proc); const protocol = RpcClient.layerProtocolSocket().pipe( Layer.provide(Socket.layerWebSocket(`ws://127.0.0.1:${port}/rpc`)), @@ -116,6 +128,7 @@ export const app = async (home: string) => { ); return { proc, + lines, run: (use: (rpc: AppRpc) => Effect.Effect) => Effect.gen(function* () { return yield* use(yield* appClient); @@ -143,7 +156,8 @@ export const query = (home: string, statement: string, ...params: Array) => { - const db = new DatabaseSync(join(home, "app.db")); + // Waits out the app's own writes, as a startup's. + const db = new DatabaseSync(join(home, "app.db"), { timeout: 5_000 }); try { db.prepare(statement).run(...params); } finally { @@ -174,6 +188,40 @@ export const synced = async (home: string, machineId: string, server: Codework) return { storeId, rev }; }; +export const cursorOf = (home: string, machineId: string) => + query(home, "SELECT rollup, rollup_store FROM machine WHERE id = ?", machineId)[0] as { + rollup: number; + rollup_store: string | null; + }; + +/** Waits until the app's usage cursor for `machineId` reached the server's head. */ +export const rolled = async (home: string, machineId: string, server: Codework) => { + const { storeId, rev } = await server.run((rpc) => rpc["server.info"]({})); + await waitFor(() => { + const row = cursorOf(home, machineId); + return row.rollup_store === storeId && row.rollup >= rev ? row : undefined; + }, `usage of machine ${machineId} at rev ${rev}`); + return { storeId, rev }; +}; + +/** Events on `topics` from the moment it resolves, collected until `stop`. */ +export const follow = async (desk: Awaited>, topics: ReadonlyArray) => { + const events: Array<{ topic: string; payload: Row }> = []; + let subscribed = false; + const background = desk.background((rpc) => + rpc["event.subscribe"]({ topics }).pipe( + Stream.runForEach((item) => + Effect.sync(() => { + if ("_tag" in item) subscribed = true; + else events.push({ topic: item.topic, payload: item.payload as Row }); + }), + ), + ), + ); + await waitFor(() => (subscribed ? true : undefined), `a subscription to ${topics.join(", ")}`); + return { events, stop: background.stop }; +}; + /** The payloads of the sync's bus events after `after` (an event seq). */ export const notices = (home: string, topic: string, after = 0) => query(home, "SELECT payload FROM event WHERE topic = ? AND seq > ? ORDER BY seq", topic, after).map( diff --git a/apps/server/test/fixtures/codework.ts b/apps/server/test/fixtures/codework.ts index 8d81816f..6a332cae 100644 --- a/apps/server/test/fixtures/codework.ts +++ b/apps/server/test/fixtures/codework.ts @@ -1,13 +1,23 @@ /* @effect-diagnostics nodeBuiltinImport:off -- a CodeWork server run as its own process by the sync suite. */ import { createAssistantMessageEventStream, type Message } from "@codeworksh/aikit"; -import { NodeRuntime } from "@effect/platform-node"; +import { NodeHttpServer, NodeRuntime } from "@effect/platform-node"; import { Effect, Layer } from "effect"; +import { HttpRouter } from "effect/http"; +import { RpcSerialization, RpcServer } from "effect/rpc"; import { existsSync, mkdirSync, writeFileSync } from "node:fs"; +import { createServer } from "node:http"; import { join } from "node:path"; import { createInterface } from "node:readline"; import { assistant } from "../../../../packages/harness/test/fixtures/llm.ts"; import { testModel } from "../../../../packages/harness/test/fixtures/model.ts"; import type * as LLM from "../../../../packages/harness/src/runner/llm.ts"; +import * as Clones from "../../../../packages/codework/src/server/clones.ts"; +import * as Contract from "../../../../packages/codework/src/server/contract.ts"; +import * as EventFeed from "../../../../packages/codework/src/server/feed.ts"; +import * as Handlers from "../../../../packages/codework/src/server/handlers.ts"; +import * as Host from "../../../../packages/codework/src/server/host.ts"; +import * as Identity from "../../../../packages/codework/src/server/identity.ts"; +import * as OAuth from "../../../../packages/codework/src/server/oauth.ts"; import * as Server from "../../../../packages/codework/src/server/server.ts"; import { Control, Harness } from "../../../../packages/harness/src/effect.ts"; import { bashPlugin } from "../../../../packages/harness/src/plugin/builtin/tool/bash.ts"; @@ -25,6 +35,9 @@ import * as DaytonaDriver from "../../../../packages/harness/src/sandboxes/dayto * Each line `release` on stdin lets the oldest held reply take its next step. * With `sweep`, it only ends the runs the last server left open, as a start does before it listens, * prints `swept` and exits: the store can then be read as no client has seen it. + * With `older`, it serves every call but `usage.sync`, and names no host, as a server built before it did. + * It reports its host as `CODEWORK_FIXTURE_HOST` (none when blank), else `Fixture`: never this computer's name, + * unless `CODEWORK_FIXTURE_HOST_OS=1`, which a live machine sets to report its own name as a real server does. */ const [home, port, host = "127.0.0.1", mode] = process.argv.slice(2); @@ -147,6 +160,31 @@ const sandboxes = process.env.DAYTONA_API_KEY?.trim() ? [DaytonaDriver.make()] : // Only the bash tool: the scripted model's `bash ` prompts call it, and nothing else varies the prompt. const harness = { home, hostCwd: home, plugins: [bashPlugin], llm: scripted, sandboxes }; +const named = process.env.CODEWORK_FIXTURE_HOST ?? "Fixture"; +const identity = + process.env.CODEWORK_FIXTURE_HOST_OS === "1" + ? Identity.layer + : Layer.succeed( + Identity.Service, + Identity.Service.of(named === "" ? Identity.current : { ...Identity.current, host: { name: named } }), + ); + +/** The RPC server alone, without the calls a server older than this build lacks. */ +const older = HttpRouter.serve( + RpcServer.layerHttp({ group: Contract.Api.omit("usage.sync"), path: Contract.path, protocol: "websocket" }).pipe( + Layer.provide(Handlers.layer.pipe(Layer.provide(EventFeed.layer))), + ), + { disableListenLog: true }, +).pipe( + Layer.provide(OAuth.layer({ home })), + Layer.provide(Host.layer), + Layer.provide(Clones.layer), + Layer.provide(NodeHttpServer.layer(createServer, { host, port: Number(port) })), + Layer.provide(RpcSerialization.layerNdjson), + Layer.provide(Layer.succeed(Identity.Service, Identity.current)), + Layer.provide(Harness.layer(harness)), +); + NodeRuntime.runMain( mode === "sweep" ? Effect.gen(function* () { @@ -156,7 +194,9 @@ NodeRuntime.runMain( yield* Effect.sync(() => process.stdout.write(`${JSON.stringify({ type: "swept" })}\n`)); }) : Effect.gen(function* () { - yield* Layer.build(Server.layer({ host, port: Number(port), harness })); + yield* Layer.build( + mode === "older" ? older : Server.layer({ host, port: Number(port), harness, identity }), + ); yield* Effect.sync(() => process.stdout.write(`${JSON.stringify({ type: "ready" })}\n`)); return yield* Effect.never; }).pipe(Effect.scoped), diff --git a/apps/server/test/host.test.ts b/apps/server/test/host.test.ts new file mode 100644 index 00000000..849b6179 --- /dev/null +++ b/apps/server/test/host.test.ts @@ -0,0 +1,158 @@ +/* @effect-diagnostics nodeBuiltinImport:off -- the suite registers CodeWork servers in temp dirs. */ +import { Effect, Option, Stream } from "effect"; +import { mkdirSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { afterEach, expect, it } from "vite-plus/test"; +import { Contract } from "@codeworksh/contract"; +import { + type AppRpc, + app, + cleanup, + type Codework, + codework, + freePort, + normalizer, + query, + snapshot as match, + stop, + synced, + tempDir, + waitFor, + write, +} from "./fixtures/apps.ts"; + +afterEach(cleanup); + +const snapshot = (name: string, artifact: unknown) => match(`host.${name}`, artifact); + +/** Registers `server` in `home` as its default local server would be, before the app starts. */ +const registerLocal = (appHome: string, server: Codework, port: number) => { + const cli = join(appHome, "cli"); + mkdirSync(cli, { recursive: true }); + writeFileSync( + join(cli, "service.json"), + JSON.stringify({ pid: server.proc.pid, version: "0.0.0", url: server.url, port }), + ); +}; + +const project = (server: Codework, directory: string, name: string) => + server.run((rpc) => rpc["session.create"]({ directory, project: { name } })); + +/** What each read that names a machine shows, per server as Settings lists them. */ +const names = (rpc: AppRpc) => + Effect.gen(function* () { + const sidebar = yield* rpc["sidebar.list"]({ organizationId: Contract.PERSONAL_ORGANIZATION, machines: "all" }); + const threads = yield* rpc["threads.list"]({ organizationId: Contract.PERSONAL_ORGANIZATION, by: "status" }); + yield* rpc["servers.check"](undefined); + const statuses = yield* rpc["servers.watch"](undefined).pipe(Stream.runHead, Effect.map(Option.getOrThrow)); + const listed = threads.lists.flatMap((list) => list.threads); + return statuses.map(({ id, name, host, override }) => ({ + id, + name, + host, + override, + sidebar: sidebar.find((project) => project.machine.id === id)?.machine.name, + allThreads: listed.find((thread) => thread.machine.id === id)?.machine.name, + })); + }); + +const stored = (appHome: string) => query(appHome, "SELECT id, kind, name, host FROM machine ORDER BY pos"); + +/** The stored row and every read of each machine. */ +const read = async (appHome: string, desk: Awaited>) => { + const rows = stored(appHome); + const reads = await desk.run(names); + return reads.map((machine) => ({ stored: rows.find((row) => row.id === machine.id), ...machine })); +}; +const shown = (state: Awaited>, id: string) => { + const machine = state.find((machine) => machine.id === id); + return [machine?.name, machine?.sidebar, machine?.allThreads]; +}; + +it("a machine is named by its server's host unless the user names it, else by where it is", async () => { + const localHome = tempDir("host-local"); + const remoteHome = tempDir("host-remote"); + const olderHome = tempDir("host-older"); + const appHome = tempDir("host-app"); + const localPort = await freePort(); + const remotePort = await freePort(); + const olderPort = await freePort(); + const local = await codework(localHome, localPort, { host: "Studio Mac" }); + let remote = await codework(remoteHome, remotePort, { host: "ubuntu" }); + const older = await codework(olderHome, olderPort, { older: true }); + await project(local, localHome, "Local"); + await project(remote, remoteHome, "Remote"); + await project(older, olderHome, "Older"); + + registerLocal(appHome, local, localPort); + const desk = await app(appHome); + const remoteId = await desk.run((rpc) => rpc["servers.add"]({ kind: "remote", url: remote.url })); + const olderId = await desk.run((rpc) => rpc["servers.add"]({ kind: "remote", name: " ", url: older.url })); + await synced(appHome, Contract.LOCAL_SERVER, local); + await synced(appHome, remoteId, remote); + await synced(appHome, olderId, older); + + const reported = await read(appHome, desk); + expect(shown(reported, Contract.LOCAL_SERVER)).toEqual(Array(3).fill("Studio Mac")); + expect(shown(reported, remoteId)).toEqual(Array(3).fill("ubuntu")); + expect(shown(reported, olderId)).toEqual(Array(3).fill(`127.0.0.1:${olderPort}`)); + + await desk.run((rpc) => rpc["servers.update"]({ id: remoteId, name: " Build Box " })); + const overridden = await read(appHome, desk); + expect(shown(overridden, remoteId)).toEqual(Array(3).fill("Build Box")); + + await desk.run((rpc) => rpc["servers.update"]({ id: remoteId, name: "" })); + const cleared = await read(appHome, desk); + expect(shown(cleared, remoteId)).toEqual(Array(3).fill("ubuntu")); + + // The same server comes back under a new name. + await stop(remote.proc); + remote = await codework(remoteHome, remotePort, { host: "ubuntu-2" }); + await waitFor( + () => (stored(appHome).find((row) => row.id === remoteId)?.host === "ubuntu-2" ? true : undefined), + "the new host name", + ); + const renamed = await read(appHome, desk); + expect(shown(renamed, remoteId)).toEqual(Array(3).fill("ubuntu-2")); + + const steps = { reported, overridden, cleared, renamed }; + expect(JSON.stringify(steps)).not.toContain("CodeWork"); + await snapshot( + "names", + normalizer( + [localHome, remoteHome, olderHome, appHome], + [ + [remoteId, ""], + [olderId, ""], + [`127.0.0.1:${olderPort}`, "127.0.0.1:"], + ], + )(steps), + ); +}, 90_000); + +it("the built-in server's old seed name is no override after the migration", async () => { + const appHome = tempDir("host-migrate"); + const first = await app(appHome); + const boxId = await first.run((rpc) => + rpc["servers.add"]({ kind: "remote", name: "Box", url: "wss://box.example.test/rpc" }), + ); + await stop(first.proc); + // app.db as it was before `host`: the seed's name on the built-in server, a name on every row. + write(appHome, "ALTER TABLE machine DROP COLUMN host"); + write(appHome, "UPDATE machine SET name = 'CodeWork' WHERE id = 'local'"); + write(appHome, "DELETE FROM effect_sql_migrations WHERE migration_id = 202611140002"); + + const second = await app(appHome); + const servers = await second.run((rpc) => + rpc["servers.watch"](undefined).pipe(Stream.runHead, Effect.map(Option.getOrThrow)), + ); + const migrated = { + stored: stored(appHome), + servers: servers.map(({ id, name, host, override }) => ({ id, name, host, override })), + }; + expect(migrated.stored).toEqual([ + { id: Contract.LOCAL_SERVER, kind: "local", name: null, host: null }, + { id: boxId, kind: "remote", name: "Box", host: null }, + ]); + await snapshot("migrated", normalizer([appHome], [[boxId, ""]])(migrated)); +}, 60_000); diff --git a/apps/server/test/metrics.test.ts b/apps/server/test/metrics.test.ts new file mode 100644 index 00000000..2d93518b --- /dev/null +++ b/apps/server/test/metrics.test.ts @@ -0,0 +1,486 @@ +/* @effect-diagnostics nodeBuiltinImport:off -- the suite makes project folders and removes CodeWork stores in temp dirs. */ +import { Effect } from "effect"; +import { mkdirSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { afterEach, expect, it } from "vite-plus/test"; +import { Contract, type MetricsHour, type MetricsRow, type MetricsUsage } from "@codeworksh/contract"; +import { edges } from "@codeworksh/widget/codework"; +import type * as Client from "../../../packages/codework/src/server/client.ts"; +import { + app, + type AppRpc, + cleanup, + type Codework, + codework, + follow, + freePort, + normalizer, + query, + rolled, + snapshot as match, + stop, + synced, + tempDir, + waitFor, + write, +} from "./fixtures/apps.ts"; + +afterEach(cleanup); + +const snapshot = (name: string, artifact: unknown) => match(`metrics.${name}`, artifact); + +type Desk = Awaited>; +type Scope = Omit[0], "since" | "until">; + +// The scripted model's requests land in the first hour of 1970 and prompts in the real one, so a +// period covering every row runs from then to tomorrow. +const DAY = 86_400_000; +const today = new Date().toISOString().slice(0, 10); +const everything = { zone: "UTC", since: "1970-01-01", until: new Date(Date.now() + DAY).toISOString().slice(0, 10) }; + +const read = (desk: Desk, request: Parameters[0]) => + desk.run((rpc) => rpc["metrics.usage"](request)); +const hoursOf = (desk: Desk, request: Parameters[0]) => + desk.run((rpc) => rpc["metrics.hours"](request)); + +/** What the rows add up to. */ +const sum = (rows: ReadonlyArray) => + rows.reduce( + (total, row) => ({ + requests: total.requests + row.requests, + input: total.input + row.tokens.input, + output: total.output + row.tokens.output, + cacheRead: total.cacheRead + row.tokens.cacheRead, + cacheWrite: total.cacheWrite + row.tokens.cacheWrite, + cost: total.cost + row.cost.total, + }), + { requests: 0, input: 0, output: 0, cacheRead: 0, cacheWrite: 0, cost: 0 }, + ); +const counts = (metrics: MetricsUsage) => { + const by = new Map(); + for (const tally of metrics.tallies) + by.set(`${tally.kind}:${tally.name}`, (by.get(`${tally.kind}:${tally.name}`) ?? 0) + tally.count); + return Object.fromEntries(by); +}; + +/** Projects by name: their ids are their folders' hashes, which differ each run. */ +const named = (metrics: MetricsUsage) => ({ + ...metrics, + projects: metrics.projects.toSorted((a, b) => (a.name ?? "").localeCompare(b.name ?? "")), +}); + +/** A session in `directory` of project `name` whose prompt runs one `bash` call. */ +const session = async (server: Codework, directory: string, name: string, command: string) => { + mkdirSync(directory, { recursive: true }); + const created = await server.run((rpc) => rpc["session.create"]({ directory, project: { name } })); + await turn(server, { sessionId: created.session.id }, command); + return created; +}; +const turn = (server: Codework, ref: Parameters[0], command: string) => + server.run((rpc) => + rpc["thread.prompt"]({ ...ref, text: `bash ${command}` }).pipe(Effect.andThen(rpc["thread.wait"](ref))), + ); + +/** Adds `server` with no name of its own: it is named by the host it reports. */ +const addServer = async (home: string, server: Codework, desk: Desk) => { + const id = await desk.run((rpc) => rpc["servers.add"]({ kind: "remote", url: server.url })); + await synced(home, id, server); + return id; +}; + +/** Each project id a machine's store holds, by name, for the artifacts' labels. */ +const projectLabels = (home: string) => + query(home, "SELECT DISTINCT id, name FROM project ORDER BY name").map( + (row) => [row.id as string, ``] as const, + ); + +it("a period's usage sums each session's threads, counts its threads per day and names its sessions, projects and machine; a deleted session keeps its rows", async () => { + const cwHome = tempDir("metrics-cw"); + const appHome = tempDir("metrics-app"); + const server = await codework(cwHome, await freePort(), { host: "Metrics Host" }); + const desk = await app(appHome); + const machineId = await addServer(appHome, server, desk); + const topics = await follow(desk, Contract.readTopics.metrics); + + // A session with a fork that runs a turn of its own, and a second session in another project. + const forked = await session(server, join(cwHome, "alpha"), "Alpha", "echo one"); + const fork = await server.run((rpc) => + Effect.gen(function* () { + const ref = { sessionId: forked.session.id }; + const reply = (yield* rpc["thread.history"](ref)).entries.at(-1)!; + return (yield* rpc["thread.fork"]({ ...ref, entryId: reply.id })).thread; + }), + ); + await turn(server, { sessionId: forked.session.id, threadId: fork.id }, "echo fork"); + const other = await session(server, join(cwHome, "beta"), "Beta", "echo two"); + const { storeId } = await rolled(appHome, machineId, server); + const notice = await waitFor( + () => topics.events.find((event) => event.topic === Contract.syncTopics.usage), + "a usage notice", + ); + topics.stop(); + + const reported = await server.run((rpc) => + Effect.forEach( + [ + { sessionId: forked.session.id, threadId: forked.id }, + { sessionId: forked.session.id, threadId: fork.id }, + { sessionId: other.session.id, threadId: other.id }, + ], + (ref) => rpc["thread.info"](ref).pipe(Effect.map((thread) => thread.usage)), + ), + ); + const before = await read(desk, everything); + const totals = sum(before.rows); + expect(totals).toEqual({ + requests: 6, + input: reported.reduce((all, usage) => all + usage.input, 0), + output: reported.reduce((all, usage) => all + usage.output, 0), + cacheRead: reported.reduce((all, usage) => all + usage.cacheRead, 0), + cacheWrite: reported.reduce((all, usage) => all + usage.cacheWrite, 0), + cost: reported.reduce((all, usage) => all + usage.cost, 0), + }); + expect(counts(before)).toEqual({ "prompt:": 3, "tool:bash": 3 }); + + // CodeWork starts no subagents yet, so the rollup row is written as the sync would copy one: the fork's, under a thread of its own. + const columns = query(appHome, "SELECT name FROM pragma_table_info('usage')").map((row) => row.name as string); + write( + appHome, + `INSERT INTO usage (${columns.join(", ")}) + SELECT ${columns.map((name) => (name === "thread_id" ? "'thr_subagent'" : name)).join(", ")} FROM usage WHERE thread_id = ?`, + fork.id, + ); + const after = await read(desk, everything); + const threadsOf = (metrics: MetricsUsage, sessionId: string) => + metrics.threads.filter((row) => row.sessionId === sessionId).map((row) => [row.day, row.threads]); + // Requests and tool calls in 1970, prompts today. + expect(threadsOf(after, forked.session.id)).toEqual([ + ["1970-01-01", 3], + [today, 2], + ]); + expect(threadsOf(after, other.session.id)).toEqual([ + ["1970-01-01", 1], + [today, 1], + ]); + expect(sum(after.rows).requests).toBe(8); + expect(after.rows.filter((row) => row.sessionId === forked.session.id)).toHaveLength(1); + expect(after.machines).toEqual([{ id: machineId, name: "Metrics Host" }]); + expect(named(after).projects.map((project) => project.name)).toEqual(["Alpha", "Beta"]); + expect( + after.sessions + .map(({ id, title, deleted }) => ({ id, title, deleted })) + .toSorted((a, b) => a.id.localeCompare(b.id)), + ).toEqual( + [ + { id: forked.session.id, title: "bash echo one", deleted: false }, + { id: other.session.id, title: "bash echo two", deleted: false }, + ].toSorted((a, b) => a.id.localeCompare(b.id)), + ); + expect(after.edges).toEqual([...edges]); + + // A deleted session: its rows stay, its title goes, its project is still named. + await server.run((rpc) => rpc["session.delete"]({ sessionId: other.session.id })); + await synced(appHome, machineId, server); + const deleted = await read(desk, everything); + expect(deleted.rows).toEqual(after.rows); + expect(deleted.sessions.find((entry) => entry.id === other.session.id)).toMatchObject({ + title: null, + deleted: true, + }); + expect(deleted.projects).toEqual(after.projects); + + await snapshot( + "usage", + normalizer( + [cwHome, appHome], + [...projectLabels(appHome), [today, ""], [storeId, ""]], + )({ + notice: notice.payload, + reported, + after: named(after), + deleted: { sessions: deleted.sessions }, + }), + ); +}, 120_000); + +/** A usage row as `usage.sync` would leave it: `requests` requests, each of 10 input and 5 output tokens and $0.5. */ +const put = ( + home: string, + row: { + hour: number; + project: string; + session: string; + thread?: string; + store?: string; + model?: string; + requests?: number; + }, +) => { + const requests = row.requests ?? 1; + const durations = Array.from({ length: edges.length + 1 }, (_, bucket) => (bucket === 1 ? requests : 0)); + write( + home, + `INSERT INTO usage (machine_id, store_id, hour, thread_id, provider, model, thinking, session_id, project_id, + requests, input, output, cache_read, cache_write, reasoning, cost, cost_input, cost_output, cost_cache_read, + cost_cache_write, duration, timed, durations, rev) + VALUES ('local', ?, ?, ?, 'scripted', ?, 'off', ?, ?, ?, ?, ?, 0, 0, 0, ?, ?, ?, 0, 0, ?, ?, ?, 1)`, + row.store ?? "store_one", + row.hour, + row.thread ?? `thr_${row.session}`, + row.model ?? "model", + row.session, + row.project, + requests, + requests * 10, + requests * 5, + requests * 0.5, + requests * 0.25, + requests * 0.25, + requests * 300, + requests, + JSON.stringify(durations), + ); +}; +const prompt = (home: string, row: { hour: number; project: string; session: string; thread: string }) => + write( + home, + `INSERT INTO tally (machine_id, store_id, hour, thread_id, kind, name, session_id, project_id, count, rev) + VALUES ('local', 'store_one', ?, ?, 'prompt', '', ?, ?, 1, 1)`, + row.hour, + row.thread, + row.session, + row.project, + ); + +const utc = (month: number, day: number, hour: number) => Date.UTC(2026, month - 1, day, hour); +const days = (metrics: MetricsUsage) => + Object.fromEntries( + [...new Set(metrics.rows.map((row) => row.day))].map((day) => [ + day, + sum(metrics.rows.filter((row) => row.day === day)).requests, + ]), + ); +const hourly = (hours: ReadonlyArray) => hours.map((row) => [row.hour, row.requests]); + +it("usage lands on the zone's local days and hours: half-hour offsets, both sides of midnight, clock changes, and stores summed", async () => { + const appHome = tempDir("metrics-zones-app"); + const desk = await app(appHome); + + // 18:00 UTC is 23:30 in Kolkata and 23:45 in Kathmandu; 19:00 UTC is 00:30 and 00:45 the next day. + put(appHome, { hour: utc(3, 9, 18), project: "prj_india", session: "india", requests: 1 }); + put(appHome, { hour: utc(3, 9, 19), project: "prj_india", session: "india", requests: 2 }); + // 06:00 UTC is 23:00 the day before in Los Angeles (PDT), 07:00 UTC its midnight. + put(appHome, { hour: utc(3, 10, 6), project: "prj_la", session: "la", requests: 1 }); + put(appHome, { hour: utc(3, 10, 7), project: "prj_la", session: "la", requests: 2 }); + // Every UTC hour from the last before Los Angeles' 1 November to the first after: it is 25 hours long. + for (let hour = utc(11, 1, 6); hour <= utc(11, 2, 8); hour += 3_600_000) + put(appHome, { hour, project: "prj_fall", session: "fall" }); + // And around 8 March, which is 23 hours long. + for (let hour = utc(3, 8, 7); hour <= utc(3, 9, 7); hour += 3_600_000) + put(appHome, { hour, project: "prj_spring", session: "spring" }); + // One hour of a session from two stores of the machine, and a second thread in the newer one. + put(appHome, { hour: utc(5, 1, 12), project: "prj_stores", session: "stores", store: "store_old", requests: 1 }); + put(appHome, { hour: utc(5, 1, 12), project: "prj_stores", session: "stores", store: "store_new", requests: 2 }); + put(appHome, { + hour: utc(5, 1, 13), + project: "prj_stores", + session: "stores", + store: "store_new", + thread: "thr_second", + requests: 4, + }); + prompt(appHome, { hour: utc(3, 9, 19), project: "prj_india", session: "india", thread: "thr_india" }); + + const india = { projectId: "prj_india" }; + const kolkata = { ...india, zone: "Asia/Kolkata" }; + const la = { projectId: "prj_la", zone: "America/Los_Angeles" }; + const fall = { projectId: "prj_fall", zone: "America/Los_Angeles" }; + const spring = { projectId: "prj_spring", zone: "America/Los_Angeles" }; + const span = (scope: Scope, since: string, until = since) => read(desk, { ...scope, since, until }); + const results = { + kolkata: { + both: days(await span(kolkata, "2026-03-09", "2026-03-10")), + first: days(await span(kolkata, "2026-03-09")), + second: days(await span(kolkata, "2026-03-10")), + tallies: (await span(kolkata, "2026-03-10")).tallies, + hoursFirst: hourly(await hoursOf(desk, { ...kolkata, day: "2026-03-09" })), + hoursSecond: hourly(await hoursOf(desk, { ...kolkata, day: "2026-03-10" })), + }, + kathmandu: { + both: days(await span({ ...india, zone: "Asia/Kathmandu" }, "2026-03-09", "2026-03-10")), + hours: hourly(await hoursOf(desk, { ...india, zone: "Asia/Kathmandu", day: "2026-03-10" })), + }, + utc: days(await span({ ...india, zone: "UTC" }, "2026-03-09", "2026-03-10")), + losAngeles: { + both: days(await span(la, "2026-03-09", "2026-03-10")), + second: days(await span(la, "2026-03-10")), + hoursFirst: hourly(await hoursOf(desk, { ...la, day: "2026-03-09" })), + hoursSecond: hourly(await hoursOf(desk, { ...la, day: "2026-03-10" })), + }, + fallBack: { + days: days(await span(fall, "2026-10-31", "2026-11-02")), + hours: hourly(await hoursOf(desk, { ...fall, day: "2026-11-01" })), + }, + springForward: { + days: days(await span(spring, "2026-03-07", "2026-03-09")), + hours: hourly(await hoursOf(desk, { ...spring, day: "2026-03-08" })), + }, + stores: await span({ projectId: "prj_stores", zone: "UTC" }, "2026-05-01"), + }; + + expect(results.kolkata).toMatchObject({ + both: { "2026-03-09": 1, "2026-03-10": 2 }, + first: { "2026-03-09": 1 }, + second: { "2026-03-10": 2 }, + hoursFirst: [[23, 1]], + hoursSecond: [[0, 2]], + }); + expect(results.kolkata.tallies.map((tally) => [tally.day, tally.kind, tally.count])).toEqual([ + ["2026-03-10", "prompt", 1], + ]); + expect(results.kathmandu).toEqual({ both: { "2026-03-09": 1, "2026-03-10": 2 }, hours: [[0, 2]] }); + expect(results.utc).toEqual({ "2026-03-09": 3 }); + expect(results.losAngeles).toEqual({ + both: { "2026-03-09": 1, "2026-03-10": 2 }, + second: { "2026-03-10": 2 }, + hoursFirst: [[23, 1]], + hoursSecond: [[0, 2]], + }); + // 27 UTC hours: one on each neighbour, and all 25 of the long day, whose 1 AM came twice. + expect(results.fallBack.days).toEqual({ "2026-10-31": 1, "2026-11-01": 25, "2026-11-02": 1 }); + expect(results.fallBack.hours).toEqual(Array.from({ length: 24 }, (_, hour) => [hour, hour === 1 ? 2 : 1])); + // 25 UTC hours: one on each neighbour, and the short day's 23, with no 2 AM. + expect(results.springForward.days).toEqual({ "2026-03-07": 1, "2026-03-08": 23, "2026-03-09": 1 }); + expect(results.springForward.hours).toEqual( + Array.from({ length: 24 }, (_, hour) => hour) + .filter((hour) => hour !== 2) + .map((hour) => [hour, 1]), + ); + expect(results.stores.rows).toHaveLength(1); + expect(results.stores.rows[0]).toMatchObject({ requests: 7, timed: 7, tokens: { input: 70 } }); + expect(results.stores.rows[0]!.durations[1]).toBe(7); + expect(results.stores.threads).toEqual([{ day: "2026-05-01", machineId: "local", sessionId: "stores", threads: 2 }]); + expect(results.stores.sessions).toEqual([ + { id: "stores", machineId: "local", projectId: "prj_stores", title: null, deleted: true }, + ]); + expect(results.stores.projects).toEqual([{ id: "prj_stores", name: null }]); + expect(results.stores.machines).toEqual([{ id: "local", name: "This Machine" }]); + + // Bad input is refused with the reason. + const refused = async ( + effect: (rpc: AppRpc) => Effect.Effect, + ) => { + const error = await desk.run((rpc) => effect(rpc).pipe(Effect.flip)); + return { tag: error._tag, message: error.message }; + }; + const refusals = { + zone: await refused((rpc) => + rpc["metrics.usage"]({ zone: "Mars/Olympus", since: "2026-03-09", until: "2026-03-09" }), + ), + backwards: await refused((rpc) => + rpc["metrics.usage"]({ zone: "UTC", since: "2026-03-10", until: "2026-03-09" }), + ), + malformed: await refused((rpc) => rpc["metrics.usage"]({ zone: "UTC", since: "2026-3-9", until: "2026-03-09" })), + impossible: await refused((rpc) => rpc["metrics.hours"]({ zone: "UTC", day: "2026-02-30" })), + hoursZone: await refused((rpc) => rpc["metrics.hours"]({ zone: "", day: "2026-03-09" })), + }; + expect(Object.values(refusals).map((refusal) => refusal.tag)).toEqual(Array(5).fill("Server.MetricsError")); + + await snapshot("zones", { ...results, refusals }); +}, 60_000); + +it("filters by machine and by project; without a filter every machine and project is summed", async () => { + const homeA = tempDir("metrics-filter-a"); + const homeB = tempDir("metrics-filter-b"); + const appHome = tempDir("metrics-filter-app"); + const serverA = await codework(homeA, await freePort(), { host: "Host A" }); + const serverB = await codework(homeB, await freePort(), { host: "Host B" }); + const desk = await app(appHome); + const idA = await addServer(appHome, serverA, desk); + const idB = await addServer(appHome, serverB, desk); + await session(serverA, join(homeA, "alpha"), "Alpha", "echo alpha"); + await session(serverA, join(homeA, "beta"), "Beta", "echo beta"); + await session(serverB, join(homeB, "gamma"), "Gamma", "echo gamma"); + await rolled(appHome, idA, serverA); + await rolled(appHome, idB, serverB); + + const projectId = (name: string) => query(appHome, "SELECT id FROM project WHERE name = ?", name)[0]!.id as string; + const all = await read(desk, everything); + const onA = await read(desk, { ...everything, machineId: idA }); + const onB = await read(desk, { ...everything, machineId: idB }); + const alpha = await read(desk, { ...everything, projectId: projectId("Alpha") }); + const alphaOnB = await read(desk, { ...everything, machineId: idB, projectId: projectId("Alpha") }); + const hoursA = await hoursOf(desk, { zone: "UTC", day: "1970-01-01", machineId: idA }); + + expect(new Set(onA.rows.map((row) => row.machineId))).toEqual(new Set([idA])); + expect(new Set(onB.rows.map((row) => row.machineId))).toEqual(new Set([idB])); + expect(sum(all.rows).requests).toBe(6); + expect(sum(onA.rows).requests + sum(onB.rows).requests).toBe(sum(all.rows).requests); + expect(counts(onA)).toEqual({ "prompt:": 2, "tool:bash": 2 }); + expect(all.machines.map((machine) => machine.name).toSorted()).toEqual(["Host A", "Host B"]); + expect(named(all).projects.map((project) => project.name)).toEqual(["Alpha", "Beta", "Gamma"]); + expect(named(onA).projects.map((project) => project.name)).toEqual(["Alpha", "Beta"]); + expect(alpha.projects.map((project) => project.name)).toEqual(["Alpha"]); + expect(alpha.sessions).toHaveLength(1); + expect(sum(alpha.rows).requests).toBe(2); + expect(alphaOnB).toEqual({ + rows: [], + tallies: [], + threads: [], + sessions: [], + projects: [], + machines: [], + edges: [...edges], + }); + expect(new Set(hoursA.map((row) => row.machineId))).toEqual(new Set([idA])); + expect(hoursA.reduce((all, row) => all + row.requests, 0)).toBe(4); + + await snapshot( + "filters", + normalizer( + [homeA, homeB, appHome], + [...projectLabels(appHome), [today, ""]], + )({ + // Machine ids are random, and rows go by them: each machine's read is kept under its name. + "Host A": named(onA), + "Host B": named(onB), + all: { rows: all.rows.length, machines: all.machines.length, projects: named(all).projects }, + alpha: { sessions: alpha.sessions, projects: alpha.projects }, + hoursA, + }), + ); +}, 120_000); + +it("a machine's old and new stores read as one machine", async () => { + const cwHome = tempDir("metrics-store-cw"); + const appHome = tempDir("metrics-store-app"); + const port = await freePort(); + let server = await codework(cwHome, port); + const desk = await app(appHome); + const machineId = await addServer(appHome, server, desk); + await session(server, join(cwHome, "old"), "Old", "echo old"); + const old = await rolled(appHome, machineId, server); + + await stop(server.proc, "SIGTERM"); + for (const suffix of ["", "-wal", "-shm"]) rmSync(`${server.database}${suffix}`, { force: true }); + server = await codework(cwHome, port); + await session(server, join(cwHome, "new"), "New", "echo new"); + const fresh = await rolled(appHome, machineId, server); + expect(fresh.storeId).not.toBe(old.storeId); + expect(query(appHome, "SELECT DISTINCT store_id FROM usage")).toHaveLength(2); + + const metrics = await read(desk, everything); + expect(metrics.machines).toEqual([{ id: machineId, name: "Fixture" }]); + expect(new Set(metrics.rows.map((row) => row.machineId))).toEqual(new Set([machineId])); + expect(metrics.sessions).toHaveLength(2); + expect(sum(metrics.rows).requests).toBe(4); + expect(counts(metrics)).toEqual({ "prompt:": 2, "tool:bash": 2 }); + + await snapshot( + "stores", + normalizer( + [cwHome, appHome], + [...projectLabels(appHome), [today, ""], [old.storeId, ""], [fresh.storeId, ""]], + )(named(metrics)), + ); +}, 120_000); diff --git a/apps/server/test/reads.test.ts b/apps/server/test/reads.test.ts index c7e49aab..2b11277a 100644 --- a/apps/server/test/reads.test.ts +++ b/apps/server/test/reads.test.ts @@ -11,6 +11,7 @@ import { cleanup, type Codework, codework, + follow, freePort, normalizer, query, @@ -77,24 +78,6 @@ const registerLocal = (appHome: string, server: Codework, port: number) => { const sidebar = (rpc: AppRpc, machines: "default" | "all" = "all") => rpc["sidebar.list"]({ organizationId: Contract.PERSONAL_ORGANIZATION, machines }); -/** Events on `topics` from the moment it resolves, collected until `stop`. */ -const follow = async (desk: Awaited>, topics: ReadonlyArray) => { - const events: Array<{ topic: string; payload: Row }> = []; - let subscribed = false; - const background = desk.background((rpc) => - rpc["event.subscribe"]({ topics }).pipe( - Stream.runForEach((item) => - Effect.sync(() => { - if ("_tag" in item) subscribed = true; - else events.push({ topic: item.topic, payload: item.payload as Row }); - }), - ), - ), - ); - await waitFor(() => (subscribed ? true : undefined), `a subscription to ${topics.join(", ")}`); - return { events, stop: background.stop }; -}; - it("the sidebar reads app.db alone: a run started before the app shows running at launch, a fork's activity moves its session up, and it stays while CodeWork is stopped", async () => { const cwHome = tempDir("sidebar-cw"); const appHome = tempDir("sidebar-app"); diff --git a/apps/server/test/schema.test.ts b/apps/server/test/schema.test.ts index 012e30f4..1d6020f3 100644 --- a/apps/server/test/schema.test.ts +++ b/apps/server/test/schema.test.ts @@ -90,9 +90,19 @@ const insertTree = (db: DatabaseSync, machine: string, suffix: string, groupingI `INSERT INTO inbox (id, thread_id, delivery, text, admitted_seq, created_at) VALUES (?, ?, 'followUp', 'next', 2, ?)`, ).run(`msg_${suffix}`, `thr_${suffix}`, now); + db.prepare( + `INSERT INTO usage (machine_id, store_id, hour, thread_id, provider, model, thinking, session_id, project_id, + requests, input, output, cache_read, cache_write, reasoning, cost, cost_input, cost_output, cost_cache_read, + cost_cache_write, duration, timed, durations, rev) + VALUES (?, 'str', 0, ?, 'openai', 'gpt', 'off', ?, 'prj', 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, '[1]', 1)`, + ).run(machine, `thr_${suffix}`, `ses_${suffix}`); + db.prepare( + `INSERT INTO tally (machine_id, store_id, hour, thread_id, kind, name, session_id, project_id, count, rev) + VALUES (?, 'str', 0, ?, 'tool', 'bash', ?, 'prj', 1, 1)`, + ).run(machine, `thr_${suffix}`, `ses_${suffix}`); }; -const TREE = ["project", "space", "session", "thread", "entry", "part", "inbox"] as const; +const TREE = ["project", "space", "session", "thread", "entry", "part", "inbox", "usage", "tally"] as const; it("creates the sync tables, seeds them, and cascades a removed machine", async () => { const db = new DatabaseSync(database); diff --git a/apps/server/test/usage.machines.live.test.ts b/apps/server/test/usage.machines.live.test.ts new file mode 100644 index 00000000..a2a0d43f --- /dev/null +++ b/apps/server/test/usage.machines.live.test.ts @@ -0,0 +1,537 @@ +/* @effect-diagnostics nodeBuiltinImport:off -- the suite runs CodeWork servers on OrbStack Linux machines. */ +import { Effect, Option, Stream } from "effect"; +import { userInfo } from "node:os"; +import { afterAll, beforeAll, describe, expect, it } from "vite-plus/test"; +import { Contract, type MetricsRow, type MetricsUsage } from "@codeworksh/contract"; +import { Session } from "../../../packages/harness/src/effect.ts"; +import { + type AppRpc, + app, + cleanup, + cursorOf, + lastSeq, + machineRow, + normalizer, + notices, + query, + type Row, + snapshot as match, + stop, + tempDir, + waitFor, +} from "./fixtures/apps.ts"; +import { available, ensure, machine, type Machine } from "./machines/orb.ts"; + +/* + * The usage rollup across real computers: the fixture CodeWork server (scripted model) runs on the + * OrbStack machines cw-a, cw-b and cw-c, each reporting its own host name, all mirrored into one app + * server on the Mac. Each machine's rollup is read inside it with sqlite3 and compared with app.db and + * the metrics reads, across a network cut, an app restart and a deleted session. + * `usage.machines.rollup.json` records the names, rows and totals. + */ + +const userHome = `/home/${userInfo().username}`; +const cwA = machine("cw-a"); +const cwB = machine("cw-b"); +const cwC = machine("cw-c"); +const pool = [cwA, cwB, cwC]; + +type Desk = Awaited>; +type Rollup = { readonly usage: Array; readonly tally: Array }; + +let appHome = ""; +let desk: Desk; +const ids = new Map(); +const stores = new Map(); +const idOf = (m: Machine) => ids.get(m.name)!; + +const work = (m: Machine) => `${m.cwHome}/work`; +const DAY = 86_400_000; +const today = new Date().toISOString().slice(0, 10); +const everything = { zone: "UTC", since: "1970-01-01", until: new Date(Date.now() + DAY).toISOString().slice(0, 10) }; + +/** A machine with a fresh store and its server running, named as the OS names the machine. */ +const fresh = async (m: Machine) => { + await m.net("restore"); + await m.net("fast"); + await m.stop("KILL"); + await m.wipe(); + await m.sh(`mkdir -p ${work(m)}`); + await m.start({ CODEWORK_FIXTURE_HOST_OS: "1" }); +}; + +/** Waits until the app's index cursor for `m` reached its head. */ +const caughtUp = async (m: Machine, timeout = 120_000) => { + const { storeId, rev } = await m.run((rpc) => rpc["server.info"]({})); + await waitFor( + () => { + const row = machineRow(appHome, idOf(m)); + return row?.store_id === storeId && row.cursor >= rev ? row : undefined; + }, + `${m.name} indexed at rev ${rev}`, + timeout, + ); +}; +/** Waits until the app's usage cursor for `m` reached its head. */ +const rolledUp = async (m: Machine, timeout = 120_000) => { + const { storeId, rev } = await m.run((rpc) => rpc["server.info"]({})); + await waitFor( + () => { + const row = cursorOf(appHome, idOf(m)); + return row.rollup_store === storeId && row.rollup >= rev ? row : undefined; + }, + `${m.name} rolled up at rev ${rev}`, + timeout, + ); + stores.set(m.name, storeId); +}; +const settled = async () => { + for (const m of pool) { + await caughtUp(m); + await rolledUp(m); + } +}; + +type Ref = { readonly sessionId: string; readonly threadId?: string }; +const branded = (ref: Ref) => ({ + sessionId: Session.SessionSchema.ID.make(ref.sessionId), + ...(ref.threadId === undefined ? {} : { threadId: Session.ThreadSchema.ID.make(ref.threadId) }), +}); +const turn = (m: Machine, ref: Ref, text: string) => + m.run((rpc) => + rpc["thread.prompt"]({ ...branded(ref), text }).pipe(Effect.andThen(rpc["thread.wait"](branded(ref)))), + ); +const session = async (m: Machine, text: string) => { + const created = await m.run((rpc) => rpc["session.create"]({ directory: work(m) })); + await turn(m, { sessionId: created.session.id }, text); + return { sessionId: created.session.id as string, threadId: created.id as string }; +}; +const fork = async (m: Machine, ref: Ref) => { + const forked = await m.run((rpc) => + Effect.gen(function* () { + const reply = (yield* rpc["thread.history"](branded(ref))).entries.at(-1)!; + return (yield* rpc["thread.fork"]({ ...branded(ref), entryId: reply.id })).thread; + }), + ); + return { sessionId: ref.sessionId, threadId: forked.id as string }; +}; + +// Rows keyed as both stores key them, sorted so each side lines up. +const keyOf = (row: Row) => + [row.hour, row.thread_id, row.provider ?? row.kind, row.model ?? row.name, row.thinking].map(String).join("|"); +const byKey = (rows: ReadonlyArray) => rows.toSorted((a, b) => keyOf(a).localeCompare(keyOf(b))); +const byRev = (rows: ReadonlyArray) => + rows.toSorted((a, b) => (a.rev as number) - (b.rev as number) || keyOf(a).localeCompare(keyOf(b))); + +/** The machine's own rollup, read in it. */ +const harness = async (m: Machine): Promise => ({ + usage: await m.sql("SELECT * FROM usage"), + tally: await m.sql("SELECT * FROM tally"), +}); +/** What app.db mirrors of `m`, without the machine and store it is keyed by. */ +const mirrored = (m: Machine): Rollup & { readonly stores: ReadonlyArray } => { + const strip = (rows: Array) => rows.map(({ machine_id: _m, store_id: _s, ...row }) => row); + const usage = query(appHome, "SELECT * FROM usage WHERE machine_id = ?", idOf(m)); + const tally = query(appHome, "SELECT * FROM tally WHERE machine_id = ?", idOf(m)); + return { + usage: strip(usage), + tally: strip(tally), + stores: [...new Set([...usage, ...tally].map((row) => row.store_id))], + }; +}; +const whole = () => ({ + usage: query(appHome, "SELECT * FROM usage ORDER BY machine_id, rev, thread_id, provider, model"), + tally: query(appHome, "SELECT * FROM tally ORDER BY machine_id, rev, thread_id, kind, name"), +}); + +/** app.db holds exactly the machine's rollup, from its current store. */ +const expectMirror = async (m: Machine) => { + const theirs = await harness(m); + const mine = mirrored(m); + expect(byKey(mine.usage)).toEqual(byKey(theirs.usage)); + expect(byKey(mine.tally)).toEqual(byKey(theirs.tally)); + expect(mine.stores).toEqual([stores.get(m.name)]); + return theirs; +}; + +type Totals = { requests: number; input: number; output: number; cacheRead: number; cacheWrite: number; cost: number }; +const zero: Totals = { requests: 0, input: 0, output: 0, cacheRead: 0, cacheWrite: 0, cost: 0 }; +const add = (a: Totals, b: Totals): Totals => ({ + requests: a.requests + b.requests, + input: a.input + b.input, + output: a.output + b.output, + cacheRead: a.cacheRead + b.cacheRead, + cacheWrite: a.cacheWrite + b.cacheWrite, + cost: a.cost + b.cost, +}); +const ofRows = (rows: ReadonlyArray) => + rows.reduce( + (total, row) => + add(total, { + requests: row.requests as number, + input: row.input as number, + output: row.output as number, + cacheRead: row.cache_read as number, + cacheWrite: row.cache_write as number, + cost: row.cost as number, + }), + zero, + ); +const ofMetrics = (rows: ReadonlyArray) => + rows.reduce( + (total, row) => + add(total, { + requests: row.requests, + input: row.tokens.input, + output: row.tokens.output, + cacheRead: row.tokens.cacheRead, + cacheWrite: row.tokens.cacheWrite, + cost: row.cost.total, + }), + zero, + ); +const tallies = (rows: ReadonlyArray) => { + const by: Record = {}; + for (const row of rows) + by[`${String(row.kind)}:${String(row.name)}`] = + (by[`${String(row.kind)}:${String(row.name)}`] ?? 0) + (row.count as number); + return by; +}; +const metricsTallies = (metrics: MetricsUsage) => + tallies(metrics.tallies.map((tally) => ({ kind: tally.kind, name: tally.name, count: tally.count }))); + +const read = (request: Parameters[0]) => desk.run((rpc) => rpc["metrics.usage"](request)); + +/** What each read that names a machine shows, per machine as Settings lists them. */ +const names = async () => { + const rows = query(appHome, "SELECT id, kind, name, host FROM machine ORDER BY pos"); + const reads = await desk.run((rpc) => + Effect.gen(function* () { + const sidebar = yield* rpc["sidebar.list"]({ + organizationId: Contract.PERSONAL_ORGANIZATION, + machines: "all", + }); + const threads = yield* rpc["threads.list"]({ organizationId: Contract.PERSONAL_ORGANIZATION, by: "status" }); + const statuses = yield* rpc["servers.watch"](undefined).pipe(Stream.runHead, Effect.map(Option.getOrThrow)); + const listed = threads.lists.flatMap((list) => list.threads); + return statuses.map(({ id, name, host, override }) => ({ + id, + name, + host, + override, + sidebar: sidebar.find((project) => project.machine.id === id)?.machine.name, + allThreads: listed.find((thread) => thread.machine.id === id)?.machine.name, + })); + }), + ); + return reads.map((read) => ({ stored: rows.find((row) => row.id === read.id), ...read })); +}; +const shown = (state: Awaited>, m: Machine) => { + const found = state.find((read) => read.id === idOf(m)); + return [found?.name, found?.sidebar, found?.allThreads]; +}; + +// Kept for the artifact as the cases go. +const artifact: Record = {}; +let workload: Record }> = {}; + +const describeMachines = available ? describe : describe.skip; + +describeMachines("usage across OrbStack machines", () => { + beforeAll(async () => { + for (const m of pool) await ensure(m.name); + await Promise.all(pool.map((m) => m.setup())); + await Promise.all(pool.map((m) => fresh(m))); + appHome = tempDir("usage-machines-app"); + desk = await app(appHome); + // No name of its own: each is named by the host its server reports. + for (const m of pool) + ids.set(m.name, await desk.run((rpc) => rpc["servers.add"]({ kind: "remote", url: m.url }))); + + // Different work on each: a turn with a tool call (two requests); two turns; a turn and a fork's turn. + const a = await session(cwA, "bash echo alpha"); + const b = await session(cwB, "first"); + await turn(cwB, b, "second"); + const c = await session(cwC, "bash echo gamma"); + const forked = await fork(cwC, c); + await turn(cwC, forked, "in the fork"); + workload = { "cw-a": { sessions: [a] }, "cw-b": { sessions: [b] }, "cw-c": { sessions: [c, forked] } }; + await settled(); + }, 1_800_000); + + afterAll(async () => { + await Promise.all( + pool.map((m) => + m + .net("fast") + .then(() => m.net("restore")) + .then(() => m.stop("TERM")) + .catch(() => undefined), + ), + ); + await cleanup(); + }, 120_000); + + it("1. each machine is named by its own host name; an override wins and clears back to it", async () => { + const reported = await waitFor(async () => { + const state = await names(); + return pool.every((m) => state.find((read) => read.id === idOf(m))?.stored?.host === m.name) + ? state + : undefined; + }, "every machine's host name"); + for (const m of pool) { + const found = reported.find((read) => read.id === idOf(m))!; + expect(found.stored).toMatchObject({ kind: "remote", name: null, host: m.name }); + expect(found).toMatchObject({ host: m.name, override: null }); + expect(shown(reported, m)).toEqual(Array(3).fill(m.name)); + } + + await desk.run((rpc) => rpc["servers.update"]({ id: idOf(cwB), name: " Build Box " })); + const overridden = await names(); + expect(shown(overridden, cwB)).toEqual(Array(3).fill("Build Box")); + expect(shown(overridden, cwA)).toEqual(Array(3).fill("cw-a")); + const overriddenMetrics = (await read(everything)).machines.find((entry) => entry.id === idOf(cwB)); + expect(overriddenMetrics?.name).toBe("Build Box"); + + await desk.run((rpc) => rpc["servers.update"]({ id: idOf(cwB), name: "" })); + const cleared = await names(); + expect(shown(cleared, cwB)).toEqual(Array(3).fill("cw-b")); + expect(cleared.find((read) => read.id === idOf(cwB))?.stored).toMatchObject({ name: null, host: "cw-b" }); + + const steps = { reported, overridden, overriddenMetrics, cleared }; + expect(JSON.stringify(steps)).not.toContain("CodeWork"); + artifact.names = steps; + }, 300_000); + + it("2. each machine's mirrored usage and tallies are its own rollup, and sum to its threads' usage", async () => { + const rollups: Record = {}; + for (const m of pool) { + const theirs = await expectMirror(m); + const threads = await m.sql("SELECT id, session_id FROM thread ORDER BY created_at, created_seq"); + const reported = await m.run((rpc) => + Effect.forEach(threads, (thread) => + rpc["thread.info"]( + branded({ sessionId: thread.session_id as string, threadId: thread.id as string }), + ).pipe(Effect.map((info) => ({ id: thread.id as string, usage: info.usage }))), + ), + ); + const app = mirrored(m); + for (const { id, usage } of reported) { + const rows = app.usage.filter((row) => row.thread_id === id); + expect(ofRows(rows)).toMatchObject({ + input: usage.input, + output: usage.output, + cacheRead: usage.cacheRead, + cacheWrite: usage.cacheWrite, + cost: usage.cost, + }); + } + rollups[m.name] = { + threads: reported.length, + totals: ofRows(theirs.usage), + tallies: tallies(theirs.tally), + usage: byRev(app.usage), + tally: byRev(app.tally), + }; + } + expect( + Object.fromEntries( + Object.entries(rollups).map(([name, rollup]) => [name, (rollup as { totals: Totals }).totals.requests]), + ), + ).toEqual({ + "cw-a": 2, + "cw-b": 2, + "cw-c": 3, + }); + expect( + Object.fromEntries( + Object.entries(rollups).map(([name, rollup]) => [name, (rollup as { tallies: unknown }).tallies]), + ), + ).toEqual({ + "cw-a": { "prompt:": 1, "tool:bash": 1 }, + "cw-b": { "prompt:": 2 }, + "cw-c": { "prompt:": 2, "tool:bash": 1 }, + }); + artifact.rollup = rollups; + }, 300_000); + + it("3. metrics.usage sums every machine or one, names each machine, and metrics.hours holds the day's totals", async () => { + const own = new Map(); + for (const m of pool) own.set(m.name, await harness(m)); + const all = await read(everything); + const total = pool.reduce((sum, m) => add(sum, ofRows(own.get(m.name)!.usage)), zero); + expect(ofMetrics(all.rows)).toEqual(total); + expect(all.machines.toSorted((a, b) => a.name.localeCompare(b.name))).toEqual( + pool.map((m) => ({ id: idOf(m), name: m.name })), + ); + + const perMachine: Record = {}; + for (const m of pool) { + const one = await read({ ...everything, machineId: idOf(m) }); + const theirs = own.get(m.name)!; + expect(new Set(one.rows.map((row) => row.machineId))).toEqual(new Set([idOf(m)])); + expect(new Set(one.tallies.map((row) => row.machineId))).toEqual(new Set([idOf(m)])); + expect(ofMetrics(one.rows)).toEqual(ofRows(theirs.usage)); + expect(metricsTallies(one)).toEqual(tallies(theirs.tally)); + expect(one.machines).toEqual([{ id: idOf(m), name: m.name }]); + perMachine[m.name] = { totals: ofMetrics(one.rows), tallies: metricsTallies(one), machines: one.machines }; + } + + // The scripted model stamps its requests with its own clock, so they land on the day of its hours; + // prompts land on today's. + const requestDays = [ + ...new Set( + [...own.values()].flatMap((rollup) => + rollup.usage.map((row) => new Date(row.hour as number).toISOString().slice(0, 10)), + ), + ), + ].toSorted(); + const hours = []; + for (const day of requestDays) + hours.push(...(await desk.run((rpc) => rpc["metrics.hours"]({ zone: "UTC", day })))); + const tokens = total.input + total.output + total.cacheRead + total.cacheWrite; + expect({ + requests: hours.reduce((sum, row) => sum + row.requests, 0), + tokens: hours.reduce((sum, row) => sum + row.tokens, 0), + cost: hours.reduce((sum, row) => sum + row.cost, 0), + }).toEqual({ requests: total.requests, tokens, cost: total.cost }); + const onDays = await read({ zone: "UTC", since: requestDays[0]!, until: requestDays.at(-1)! }); + expect(ofMetrics(onDays.rows)).toEqual(total); + const onToday = await read({ zone: "UTC", since: today, until: today }); + const prompts = pool.reduce((sum, m) => sum + (tallies(own.get(m.name)!.tally)["prompt:"] ?? 0), 0); + expect(metricsTallies(onToday)["prompt:"]).toBe(prompts); + + artifact.metrics = { + all: { + totals: ofMetrics(all.rows), + tallies: metricsTallies(all), + // Listed by id, which is random: kept by name. + machines: all.machines.toSorted((a, b) => a.name.localeCompare(b.name)), + }, + perMachine, + requestDays, + hours: hours.toSorted( + (a, b) => + a.hour - b.hour || a.machineId.localeCompare(b.machineId) || a.sessionId.localeCompare(b.sessionId), + ).length, + todayPrompts: metricsTallies(onToday)["prompt:"], + }; + }, 300_000); + + it("4. a cut machine catches up from its cursor with nothing lost or doubled; an app restart changes nothing", async () => { + const [a] = workload["cw-a"]!.sessions; + const cursor = cursorOf(appHome, idOf(cwB)).rollup; + + const cutAt = performance.now(); + await cwB.net("cut"); + // cw-b keeps working over its own loopback, in a new session so its older rows stay before the + // cursor; the others over the network. + const [created] = (await cwB.drive([["session.create", { directory: work(cwB) }]])) as [ + { session: { id: string }; id: string }, + ]; + await cwB.drive([ + ["thread.prompt", { sessionId: created.session.id, text: "bash echo while cut" }], + ["thread.wait", { sessionId: created.session.id }], + ]); + await turn(cwA, a!, "bash echo during the cut"); + await session(cwC, "during the cut"); + await rolledUp(cwA); + await rolledUp(cwC); + // Past the client's ping timeout, so the app has dropped the connection. + const held = 15_000 - (performance.now() - cutAt); + if (held > 0) await new Promise((resolve) => setTimeout(resolve, held)); + expect(cursorOf(appHome, idOf(cwB)).rollup).toBe(cursor); + + const at = lastSeq(appHome); + await cwB.net("restore"); + await rolledUp(cwB); + const changed = notices(appHome, Contract.syncTopics.usage, at).filter( + (notice) => notice.machineId === idOf(cwB), + ); + const since = await harness(cwB); + const pending = [...since.usage, ...since.tally].filter((row) => (row.rev as number) > cursor).length; + const sent = changed.reduce((sum, notice) => sum + (notice.items as number), 0); + expect(sent).toBe(pending); + expect(pending).toBeLessThan(since.usage.length + since.tally.length); + await expectMirror(cwB); + + await turn(cwB, { sessionId: created.session.id }, "after the cut"); + await settled(); + for (const m of pool) await expectMirror(m); + const totals = Object.fromEntries( + await Promise.all(pool.map(async (m) => [m.name, ofRows((await harness(m)).usage)] as const)), + ); + expect(Object.fromEntries(pool.map((m) => [m.name, ofRows(mirrored(m).usage)]))).toEqual(totals); + + // The app restarts: it resumes every machine from its cursor and holds the same rows. + const before = whole(); + const metricsBefore = await read(everything); + await stop(desk.proc); + const restartedAt = lastSeq(appHome); + desk = await app(appHome); + await settled(); + const resent = notices(appHome, Contract.syncTopics.usage, restartedAt).reduce( + (sum, notice) => sum + (notice.items as number), + 0, + ); + expect(whole()).toEqual(before); + expect(resent).toBe(0); + const metricsAfter = await read(everything); + expect(ofMetrics(metricsAfter.rows)).toEqual(ofMetrics(metricsBefore.rows)); + for (const m of pool) await expectMirror(m); + + artifact.resilience = { + cut: "cw-b", + resumed: { sent, pending, notices: changed.length > 0 }, + totals, + restart: { resent, rowsUnchanged: true, metrics: ofMetrics(metricsAfter.rows) }, + }; + }, 300_000); + + it("5. a session deleted on its machine keeps its mirrored usage, marked deleted", async () => { + const [c] = workload["cw-c"]!.sessions; + const before = whole(); + const metricsBefore = await read(everything); + await cwC.run((rpc) => rpc["session.delete"](branded({ sessionId: c!.sessionId }))); + await caughtUp(cwC); + await rolledUp(cwC); + expect(query(appHome, "SELECT deleted_at FROM session WHERE id = ?", c!.sessionId)[0]?.deleted_at).not.toBeNull(); + expect(whole()).toEqual(before); + const after = await read(everything); + expect(after.rows).toEqual(metricsBefore.rows); + const entry = after.sessions.find((one) => one.id === c!.sessionId); + expect(entry).toMatchObject({ machineId: idOf(cwC), title: null, deleted: true }); + expect(after.sessions.filter((one) => one.deleted).map((one) => one.id)).toEqual([c!.sessionId]); + artifact.deleted = { + session: entry, + rows: after.rows.filter((row) => row.sessionId === c!.sessionId).length, + totals: ofMetrics(after.rows.filter((row) => row.sessionId === c!.sessionId)), + }; + }, 300_000); + + it("the artifact: names, rollups, totals and recoveries", async () => { + // Requests carry the scripted model's own hours, prompts the real one. + const hours = (value: unknown, key?: string): unknown => { + if (key === "hour" && typeof value === "number") return value < DAY ? "" : ""; + if (Array.isArray(value)) return value.map((item) => hours(item)); + if (typeof value === "object" && value !== null) + return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, hours(v, k)])); + return value; + }; + const projects = query(appHome, "SELECT DISTINCT id, name FROM project ORDER BY name").map( + (row) => [row.id as string, ``] as const, + ); + await match( + "usage.machines.rollup", + normalizer( + [appHome, userHome], + [ + ...pool.map((m) => [idOf(m), ``] as const), + ...pool.map((m) => [stores.get(m.name)!, ``] as const), + ...projects, + [today, ""], + ], + )(hours(artifact)), + ); + }); +}); diff --git a/apps/server/test/usage.test.ts b/apps/server/test/usage.test.ts new file mode 100644 index 00000000..62c19493 --- /dev/null +++ b/apps/server/test/usage.test.ts @@ -0,0 +1,335 @@ +/* @effect-diagnostics nodeBuiltinImport:off -- the suite stops, copies and removes CodeWork stores in temp dirs. */ +import { Effect, Stream } from "effect"; +import { copyFileSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { afterEach, expect, it } from "vite-plus/test"; +import { Contract } from "@codeworksh/contract"; +import { Codework as Wire } from "@codeworksh/widget/codework"; +import type * as Client from "../../../packages/codework/src/server/client.ts"; +import { edges, type UsageItem } from "../../../packages/codework/src/server/contract.ts"; +import { + addMachine, + app, + cleanup, + type Codework, + codework, + cursorOf, + freePort, + lastSeq, + normalizer, + notices, + query, + rolled, + type Row, + snapshot as match, + stop, + synced, + tempDir, + waitFor, +} from "./fixtures/apps.ts"; + +afterEach(cleanup); + +type UsageRow = Exclude; +type SessionId = Parameters[0]["sessionId"]; + +const snapshot = (name: string, artifact: unknown) => match(`usage.${name}`, artifact); + +/** A session in `home` whose prompt runs one `bash` call: two model requests and a tool call. */ +const session = async (server: Codework, directory: string, command: string) => { + const created = await server.run((rpc) => rpc["session.create"]({ directory })); + await turn(server, created.session.id, command); + return created; +}; +const turn = (server: Codework, sessionId: SessionId, command: string) => + server.run((rpc) => + rpc["thread.prompt"]({ sessionId, text: `bash ${command}` }).pipe( + Effect.andThen(rpc["thread.wait"]({ sessionId })), + ), + ); + +/** What `usage.sync` from `after` sends up to its first `Synced`, read from CodeWork itself. */ +const catchUp = async (server: Codework, after = 0) => { + const items = await server.run((rpc) => + rpc["usage.sync"]({ after }).pipe( + Stream.takeUntil((item) => item._tag === "Synced"), + Stream.runCollect, + ), + ); + const synced = items.at(-1); + if (synced?._tag !== "Synced") throw new Error("usage.sync ended before Synced"); + return { rows: items.filter((item): item is UsageRow => item._tag !== "Synced"), synced }; +}; + +/** A wire row as app.db holds it for `machineId` and `storeId`. */ +const stored = (machineId: string, storeId: string, row: UsageRow): Row => { + const owner = { + machine_id: machineId, + store_id: storeId, + hour: row.hour, + thread_id: row.threadId, + session_id: row.sessionId, + project_id: row.projectId, + rev: row.rev, + }; + if (row._tag === "Tally") return { ...owner, kind: row.kind, name: row.name, count: row.count }; + return { + ...owner, + provider: row.provider, + model: row.model, + thinking: row.thinking, + requests: row.requests, + input: row.tokens.input, + output: row.tokens.output, + cache_read: row.tokens.cacheRead, + cache_write: row.tokens.cacheWrite, + reasoning: row.tokens.reasoning, + cost: row.cost.total, + cost_input: row.cost.input, + cost_output: row.cost.output, + cost_cache_read: row.cost.cacheRead, + cost_cache_write: row.cost.cacheWrite, + duration: row.duration, + timed: row.timed, + durations: JSON.stringify(row.durations), + }; +}; + +const keyOf = (row: Row) => + [ + row.machine_id, + row.store_id, + row.hour, + row.thread_id, + row.provider ?? row.kind, + row.model ?? row.name, + row.thinking, + ] + .map(String) + .join("|"); +const byKey = (rows: ReadonlyArray) => rows.toSorted((a, b) => keyOf(a).localeCompare(keyOf(b))); + +const mirrored = (home: string) => ({ + usage: query(home, "SELECT * FROM usage ORDER BY rev, provider, model"), + tally: query(home, "SELECT * FROM tally ORDER BY rev, kind, name"), +}); +/** What app.db should hold for `rows` of one store: usage rows, then tallies. */ +const expected = (machineId: string, storeId: string, rows: ReadonlyArray) => ({ + usage: byKey(rows.filter((row) => row._tag === "Usage").map((row) => stored(machineId, storeId, row))), + tally: byKey(rows.filter((row) => row._tag === "Tally").map((row) => stored(machineId, storeId, row))), +}); +const sorted = (rows: { usage: ReadonlyArray; tally: ReadonlyArray }) => ({ + usage: byKey(rows.usage), + tally: byKey(rows.tally), +}); + +const requestsOf = (rows: ReadonlyArray) => rows.reduce((sum, row) => sum + (row.requests as number), 0); +const inputOf = (rows: ReadonlyArray) => rows.reduce((sum, row) => sum + (row.input as number), 0); +const itemsOf = (payloads: ReadonlyArray) => payloads.reduce((sum, payload) => sum + (payload.items as number), 0); + +// Requests are made at the scripted model's own times, in the first hour of 1970; prompts in the real one. +const hours = (value: unknown, key?: string): unknown => { + if (key === "hour") return value === 0 ? "" : ""; + if (Array.isArray(value)) return value.map((item) => hours(item)); + if (typeof value === "object" && value !== null) + return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, hours(v, k)])); + return value; +}; + +it("the widget mirror's duration edges are CodeWork's", () => { + expect(Wire.edges).toEqual(edges); +}); + +it("mirrors usage and tallies with replaced running totals, resumes after a restart and keeps a deleted session's rows", async () => { + const cwHome = tempDir("usage-cw"); + const appHome = tempDir("usage-app"); + const server = await codework(cwHome, await freePort()); + let desk = await app(appHome); + const machineId = await addMachine(appHome, server, desk.run); + + // (1) A turn: two requests and a tool call. + const first = await session(server, cwHome, "echo one"); + const sessionId = first.session.id; + const { storeId } = await rolled(appHome, machineId, server); + const one = await catchUp(server); + const afterOne = mirrored(appHome); + expect(sorted(afterOne)).toEqual(expected(machineId, storeId, one.rows)); + expect(cursorOf(appHome, machineId)).toEqual({ rollup: one.synced.rev, rollup_store: storeId }); + expect(one.synced.storeId).toBe(storeId); + expect(afterOne.usage).toHaveLength(1); + expect(afterOne.usage[0]).toMatchObject({ requests: 2, input: 2 }); + expect(afterOne.tally.map((row) => [row.kind, row.name, row.count])).toEqual([ + ["prompt", "", 1], + ["tool", "bash", 1], + ]); + + // (2) A second turn in the same hour: the row's totals are replaced by the larger running ones. + await turn(server, sessionId, "echo two"); + await rolled(appHome, machineId, server); + const two = await catchUp(server); + const afterTwo = mirrored(appHome); + const thread = await server.run((rpc) => rpc["thread.info"]({ sessionId })); + expect(sorted(afterTwo)).toEqual(expected(machineId, storeId, two.rows)); + expect(afterTwo.usage).toHaveLength(1); + expect(afterTwo.usage[0]).toMatchObject({ + requests: 4, + input: thread.usage.input, + output: thread.usage.output, + cost: thread.usage.cost, + }); + expect(afterTwo.tally.map((row) => [row.kind, row.name, row.count])).toEqual([ + ["prompt", "", 2], + ["tool", "bash", 2], + ]); + + // (3) A turn of another session while the app is down: a restart resumes from the cursor. + const cursor = cursorOf(appHome, machineId).rollup; + await stop(desk.proc); + await session(server, cwHome, "echo three"); + const resumed = await catchUp(server, cursor); + const restartedAt = lastSeq(appHome); + desk = await app(appHome); + await rolled(appHome, machineId, server); + const three = await catchUp(server); + const afterThree = mirrored(appHome); + const changed = notices(appHome, Contract.syncTopics.usage, restartedAt); + expect(sorted(afterThree)).toEqual(expected(machineId, storeId, three.rows)); + expect(itemsOf(changed)).toBe(resumed.rows.length); + expect(resumed.rows.length).toBeLessThan(three.rows.length); + expect(afterThree.usage.find((row) => row.thread_id === first.id)).toEqual(afterTwo.usage[0]); + + // (4) A deleted session's rows stay. + await server.run((rpc) => rpc["session.delete"]({ sessionId })); + await synced(appHome, machineId, server); + await rolled(appHome, machineId, server); + expect(query(appHome, "SELECT deleted_at FROM session WHERE id = ?", sessionId)[0]?.deleted_at).not.toBeNull(); + expect(mirrored(appHome)).toEqual(afterThree); + + await snapshot( + "mirror", + normalizer( + [cwHome, appHome], + [[storeId, ""]], + )( + hours({ + first: afterOne, + second: afterTwo, + resumed: { items: resumed.rows.length, changed }, + ...afterThree, + }), + ), + ); +}, 120_000); + +it("a new store's rows accumulate beside the old store's", async () => { + const cwHome = tempDir("usage-store-cw"); + const appHome = tempDir("usage-store-app"); + const port = await freePort(); + let server = await codework(cwHome, port); + const desk = await app(appHome); + const machineId = await addMachine(appHome, server, desk.run); + await session(server, cwHome, "echo old"); + const old = await rolled(appHome, machineId, server); + const before = mirrored(appHome); + + // The app stays up: it reconnects to the new store on its own. + await stop(server.proc, "SIGTERM"); + for (const suffix of ["", "-wal", "-shm"]) rmSync(`${server.database}${suffix}`, { force: true }); + server = await codework(cwHome, port); + await session(server, cwHome, "echo new"); + const fresh = await rolled(appHome, machineId, server); + expect(fresh.storeId).not.toBe(old.storeId); + const now = await catchUp(server); + + const rows = mirrored(appHome); + const ofStore = (list: ReadonlyArray, id: string) => list.filter((row) => row.store_id === id); + expect(ofStore(rows.usage, old.storeId)).toEqual(before.usage); + expect(ofStore(rows.tally, old.storeId)).toEqual(before.tally); + expect(sorted({ usage: ofStore(rows.usage, fresh.storeId), tally: ofStore(rows.tally, fresh.storeId) })).toEqual( + expected(machineId, fresh.storeId, now.rows), + ); + expect(requestsOf(rows.usage)).toBe(requestsOf(before.usage) + requestsOf(ofStore(rows.usage, fresh.storeId))); + expect(requestsOf(rows.usage)).toBe(4); + expect(inputOf(rows.usage)).toBe(4); + expect(cursorOf(appHome, machineId)).toEqual({ rollup: now.synced.rev, rollup_store: fresh.storeId }); + + await snapshot( + "store", + normalizer( + [cwHome, appHome], + [ + [old.storeId, ""], + [fresh.storeId, ""], + ], + )(hours(rows)), + ); +}, 120_000); + +it("a restored store replaces its own rows", async () => { + const cwHome = tempDir("usage-restored-cw"); + const appHome = tempDir("usage-restored-app"); + const backup = tempDir("usage-restored-backup"); + const port = await freePort(); + let server = await codework(cwHome, port); + await session(server, cwHome, "echo kept"); + // A clean stop leaves the database whole in its main file. + await stop(server.proc, "SIGTERM"); + copyFileSync(server.database, join(backup, "codework.db")); + server = await codework(cwHome, port); + + let desk = await app(appHome); + const machineId = await addMachine(appHome, server, desk.run); + const later = await session(server, cwHome, "echo lost"); + const { rev: head } = await rolled(appHome, machineId, server); + expect(query(appHome, "SELECT * FROM usage WHERE thread_id = ?", later.id)).toHaveLength(1); + + await stop(desk.proc); + await stop(server.proc, "SIGTERM"); + for (const suffix of ["-wal", "-shm"]) rmSync(`${server.database}${suffix}`, { force: true }); + copyFileSync(join(backup, "codework.db"), server.database); + server = await codework(cwHome, port); + const restored = await server.run((rpc) => rpc["server.info"]({})); + expect(restored.rev).toBeLessThan(head); + + const restartedAt = lastSeq(appHome); + desk = await app(appHome); + await waitFor( + () => (cursorOf(appHome, machineId).rollup === restored.rev ? true : undefined), + "the re-read of the restored store", + ); + const now = await catchUp(server); + const rows = mirrored(appHome); + expect(sorted(rows)).toEqual(expected(machineId, restored.storeId, now.rows)); + expect(rows.usage.some((row) => row.thread_id === later.id)).toBe(false); + const changed = notices(appHome, Contract.syncTopics.usage, restartedAt); + expect(itemsOf(changed)).toBe(now.rows.length); + + await snapshot( + "restored", + normalizer([cwHome, appHome], [[restored.storeId, ""]])(hours({ changed, ...rows })), + ); +}, 120_000); + +it("a server without usage.sync keeps its index synced and its usage empty, asked again only after a while", async () => { + const cwHome = tempDir("usage-older-cw"); + const appHome = tempDir("usage-older-app"); + const server = await codework(cwHome, await freePort(), { older: true }); + const desk = await app(appHome); + const machineId = await addMachine(appHome, server, desk.run); + const created = await session(server, cwHome, "echo older"); + await synced(appHome, machineId, server); + // Longer than the retry after any other failure. + await new Promise((resolve) => setTimeout(resolve, 3_000)); + await turn(server, created.session.id, "echo again"); + await synced(appHome, machineId, server); + + const told = desk.lines.filter((line) => line.includes("does not serve usage.sync")); + expect(query(appHome, "SELECT id FROM thread")).toHaveLength(1); + expect(query(appHome, "SELECT tokens_input FROM thread")[0]).toEqual({ tokens_input: 4 }); + expect(mirrored(appHome)).toEqual({ usage: [], tally: [] }); + expect(cursorOf(appHome, machineId)).toEqual({ rollup: 0, rollup_store: null }); + expect(told).toHaveLength(1); + expect(desk.lines.filter((line) => line.includes("Usage sync of machine"))).toEqual([]); + + await snapshot("older", { told: told.length, ...mirrored(appHome), cursor: cursorOf(appHome, machineId) }); +}, 90_000); diff --git a/apps/widget/src/codework.ts b/apps/widget/src/codework.ts index 224cb9e8..941c622c 100644 --- a/apps/widget/src/codework.ts +++ b/apps/widget/src/codework.ts @@ -230,6 +230,70 @@ export const IndexItem = Schema.Union([ ]); export type IndexItem = typeof IndexItem.Type; +/** + * Upper bounds in ms of the request duration histogram: bucket `i` of `durations` counts durations + * below `edges[i]` and at or above `edges[i - 1]`; one more bucket past the last edge is open-ended. + */ +export const edges = [ + 250, 374, 561, 839, 1257, 1882, 2818, 4220, 6318, 9461, 14167, 21213, 31764, 47563, 71221, 106645, 159688, 239115, + 358046, 536133, 802797, 1202096, 1800000, +] as const; + +/** Where a `usage.sync` row belongs: a thread's UTC hour. */ +const UsageOwner = { + /** Start of the UTC hour, epoch milliseconds. */ + hour: Schema.Int, + threadId: Schema.String, + /** The thread's session and project when the row was first written. */ + sessionId: Schema.String, + projectId: Schema.String, +}; + +/** A `usage.sync` item: a rollup row with the revision that last wrote it, or progress. */ +export const UsageItem = Schema.Union([ + /** Requests of one thread in one hour to one provider, model and thinking level, aborted ones included. */ + Schema.TaggedStruct("Usage", { + ...UsageOwner, + provider: Schema.String, + model: Schema.String, + thinking: Schema.String, + requests: Schema.Int, + /** `reasoning` is part of `output`. */ + tokens: Schema.Struct({ + input: Schema.Int, + output: Schema.Int, + cacheRead: Schema.Int, + cacheWrite: Schema.Int, + reasoning: Schema.Int, + }), + /** US dollars, from the catalog's prices. */ + cost: Schema.Struct({ + total: Schema.Finite, + input: Schema.Finite, + output: Schema.Finite, + cacheRead: Schema.Finite, + cacheWrite: Schema.Finite, + }), + /** Total ms of the `timed` requests: the committed ones; aborted requests are never timed. */ + duration: Schema.Finite, + timed: Schema.Int, + /** The `timed` requests per bucket of `edges`, one more than there are edges. */ + durations: Schema.Array(Schema.Int), + rev: Schema.Int, + }), + /** Committed prompts (`name` is empty), or tool calls that ran by tool name, of one thread in one hour. */ + Schema.TaggedStruct("Tally", { + ...UsageOwner, + kind: Schema.Literals(["prompt", "tool"]), + name: Schema.String, + count: Schema.Int, + rev: Schema.Int, + }), + /** Every row with a revision up to `rev` has been sent. */ + Schema.TaggedStruct("Synced", { storeId: Schema.String, rev: Schema.Int }), +]); +export type UsageItem = typeof UsageItem.Type; + /** What a client calls a thread its first prompt has not named yet. */ export const UNTITLED = "Session"; @@ -495,6 +559,8 @@ export const Api = RpcGroup.make( storeId: Schema.String, /** The store's head: the last revision taken. */ rev: Schema.Int, + /** The machine it runs on, by the name its user knows it; absent when the OS names it nothing. */ + host: Schema.optionalKey(Schema.Struct({ name: Schema.String })), }), }), /** @@ -506,6 +572,18 @@ export const Api = RpcGroup.make( success: IndexItem, stream: true, }), + /** + * The usage rollup of this machine: every `usage` and `tally` row whose revision is past `after`, + * in revision order, then `Synced` at the head read when the call began; then the same live after + * each commit. `after: 0` reads every row. A row written again moves to a later revision and is + * sent again in full; the latest wins. Rows carry running totals, not deltas: a client replaces a + * row by its key, never adds to it. A deleted session's rows stay. + */ + Rpc.make("usage.sync", { + payload: { after: Schema.Int }, + success: UsageItem, + stream: true, + }), /** * One thread's detail. Without `after`, the newest entries of its line within `budget` bytes and * its pending inbox, then `Synced` with `floor`; with `after`, its own rows past it. Then the same From 1e3f7c41afccd89ead50d36b9b857ca62d0008d9 Mon Sep 17 00:00:00 2001 From: Sanchit Rk <0xsanchit@gmail.com> Date: Sat, 14 Nov 2026 14:16:44 +0530 Subject: [PATCH 4/8] feat(web): metrics page A Metrics page beside Settings in the sidebar: cost, tokens and sessions for a period; a 12-month activity calendar by prompts, sessions, tokens or cost with a per-day view; tokens or cost over time by model or provider; projects, models, response time, weeks, tool calls, thinking levels, cache and top sessions. It reads one year of usage and refreshes as usage syncs. The server name in Settings is optional and shows the machine's own name when empty. Co-Authored-By: Claude Fable 5.2 --- apps/web/src/landing/sidebar.tsx | 13 +- apps/web/src/metrics/activity.tsx | 561 +++++++++++++ apps/web/src/metrics/charts.tsx | 747 ++++++++++++++++++ apps/web/src/metrics/data.ts | 735 +++++++++++++++++ apps/web/src/metrics/filters.tsx | 243 ++++++ apps/web/src/metrics/metrics.tsx | 909 ++++++++++++++++++++++ apps/web/src/metrics/page.tsx | 174 +++++ apps/web/src/metrics/session.tsx | 56 ++ apps/web/src/routeTree.gen.ts | 21 + apps/web/src/routes/_shell.metrics.tsx | 5 + apps/web/src/settings/blocks.tsx | 2 +- apps/web/src/settings/servers/add.tsx | 10 +- apps/web/src/settings/servers/local.tsx | 12 +- apps/web/src/settings/servers/servers.tsx | 8 +- apps/web/src/shell/page.tsx | 8 +- apps/web/src/shell/shell.tsx | 5 +- 16 files changed, 3492 insertions(+), 17 deletions(-) create mode 100644 apps/web/src/metrics/activity.tsx create mode 100644 apps/web/src/metrics/charts.tsx create mode 100644 apps/web/src/metrics/data.ts create mode 100644 apps/web/src/metrics/filters.tsx create mode 100644 apps/web/src/metrics/metrics.tsx create mode 100644 apps/web/src/metrics/page.tsx create mode 100644 apps/web/src/metrics/session.tsx create mode 100644 apps/web/src/routes/_shell.metrics.tsx diff --git a/apps/web/src/landing/sidebar.tsx b/apps/web/src/landing/sidebar.tsx index e296e738..05e5c014 100644 --- a/apps/web/src/landing/sidebar.tsx +++ b/apps/web/src/landing/sidebar.tsx @@ -1,6 +1,7 @@ import { ArrowsDownUpIcon, CaretRightIcon, + ChartBarIcon, ChatsIcon, FolderIcon, FolderOpenIcon, @@ -255,11 +256,19 @@ export function SessionsSidebar() { )} - - + + Settings + + + diff --git a/apps/web/src/metrics/activity.tsx b/apps/web/src/metrics/activity.tsx new file mode 100644 index 00000000..bce30e50 --- /dev/null +++ b/apps/web/src/metrics/activity.tsx @@ -0,0 +1,561 @@ +import { useAtomValue } from "@effect/atom-react"; +import { Option } from "effect"; +import { AsyncResult } from "effect/reactivity"; +import { useLayoutEffect, useMemo, useRef, useState } from "react"; + +import { ProviderIcon } from "@/components/provider"; +import { Card, Columns, Heat, heatCuts, HeatScale, Meter, OTHER, Segmented, Share, SINGLE, TipRows } from "./charts"; +import { + type Day, + formatCost, + formatCount, + formatDay, + formatDayLong, + formatDuration, + formatQuantile, + formatTokens, + hoursAtom, + hoursKey, + listed, + type Metrics, + modelNamer, + modelsOn, + RANGES, + type Scope, + sessionsIn, + unpriced, +} from "./data"; +import { SessionLine } from "./session"; + +const GAP = 2; +const MIN_CELL = 11; +const MAX_CELL = 16; +const LABEL_GAP = 8; +const BAR_HEIGHT = 20; +const WEEKS_FOLDED = 8; +/** Monday first: the working week reads as one run, the weekend at the end. */ +const WEEKDAYS = [ + { day: 1, label: "Mon" }, + { day: 2, label: "Tue" }, + { day: 3, label: "Wed" }, + { day: 4, label: "Thu" }, + { day: 5, label: "Fri" }, + { day: 6, label: "Sat" }, + { day: 0, label: "Sun" }, +] as const; + +const weekdayOf = (day: Day) => new Date(day.date).getDay(); +const month = new Intl.DateTimeFormat(undefined, { month: "short" }); +const monthDay = new Intl.DateTimeFormat(undefined, { month: "short", day: "numeric" }); + +/** What the calendar counts, like a contribution graph's kinds. */ +type Measure = "prompts" | "sessions" | "tokens" | "cost"; +const MEASURE: Record< + Measure, + { readonly label: string; readonly one: string; readonly many: string; readonly format: (value: number) => string } +> = { + prompts: { label: "Prompts", one: "Prompt", many: "Prompts", format: formatCount }, + sessions: { label: "Sessions", one: "Session", many: "Sessions", format: formatCount }, + tokens: { label: "Tokens", one: "Token", many: "Tokens", format: formatTokens }, + cost: { label: "Cost", one: "Spent", many: "Spent", format: formatCost }, +}; +const KEYS = ["prompts", "sessions", "tokens", "cost"] as const; +/** The day view leads with spend. */ +const HEADLINE: ReadonlyArray = ["tokens", "cost", "sessions", "prompts"]; +const MEASURES = KEYS.map((key) => ({ key, label: MEASURE[key].label })); +/** "1,204 Prompts", "$12.40 Spent". */ +const amount = (measure: Measure, value: number) => { + const { one, many, format } = MEASURE[measure]; + return `${format(value)} ${value === 1 ? one : many}`; +}; + +/** The year calendar with when-you-work beside it; a picked day opens its sessions below. */ +export function Activity({ + year, + scope, + colors, +}: { + readonly year: Metrics; + readonly scope: Scope; + /** The page's model colors by model key; a model missing here folds into Other. */ + readonly colors: ReadonlyMap; +}) { + const [picked, setPicked] = useState(undefined); + const [measure, setMeasure] = useState("prompts"); + const pickedDay = picked === undefined ? undefined : year.days[year.days.length - 1 - picked]; + return ( + + } + > +
+
+ setPicked((current) => (current === ago ? undefined : ago))} + /> + +
+ +
+ {pickedDay !== undefined && ( + setPicked(undefined)} + /> + )} +
+ ); +} + +function Calendar({ + metrics, + measure, + picked, + onPick, +}: { + readonly metrics: Metrics; + readonly measure: Measure; + readonly picked: number | undefined; + readonly onPick: (ago: number) => void; +}) { + const { days } = metrics; + const lead = days[0] === undefined ? 0 : weekdayOf(days[0]); + const columns = Math.ceil((lead + days.length) / 7); + const scroller = useRef(null); + const labels = useRef(null); + const [cell, setCell] = useState(MIN_CELL); + // The cell grows with the card so the weeks fill the column; past the minimum it scrolls instead. + useLayoutEffect(() => { + const box = scroller.current; + if (box === null) return; + const fit = () => { + const room = box.clientWidth - (labels.current?.offsetWidth ?? 0) - LABEL_GAP; + setCell(Math.min(MAX_CELL, Math.max(MIN_CELL, Math.floor((room - (columns - 1) * GAP) / columns)))); + }; + const observer = new ResizeObserver(fit); + observer.observe(box); + return () => observer.disconnect(); + }, [columns]); + // Open on the latest weeks when the calendar is wider than the card. + useLayoutEffect(() => { + const box = scroller.current; + if (box !== null) box.scrollLeft = box.scrollWidth; + }, [days, cell]); + + const width = columns * cell + (columns - 1) * GAP; + const months = useMemo(() => { + const starts = days.flatMap((entry, index) => { + const date = new Date(entry.date); + return index === 0 || date.getDate() === 1 + ? [{ column: Math.floor((lead + index) / 7), label: month.format(date) }] + : []; + }); + // The period's opening month only gets a label when there is room before the next one. + return starts.filter((start, index) => index !== 0 || (starts[1]?.column ?? columns) - start.column >= 3); + }, [days, lead, columns]); + const cuts = useMemo(() => heatCuts(days.map((entry) => entry[measure])), [days, measure]); + const dayAt = (ago: number) => days[days.length - 1 - ago]; + const order = [measure, ...KEYS.filter((key) => key !== measure)]; + const { many, format } = MEASURE[measure]; + + return ( +
+

+ {format(metrics.totals[measure])} {many} in the Last 12 + Months +

+
+
+
+ {["", "Mon", "", "Wed", "", "Fri", ""].map((label, index) => ( + {label} + ))} +
+
+
+ {months.map((start) => ( + + {start.label} + + ))} +
+ ({ + ago: entry.ago, + weekday: weekdayOf(entry), + value: entry[measure], + }))} + title={({ ago }) => { + const entry = dayAt(ago); + return ( + ({ + label: MEASURE[key].label, + value: MEASURE[key].format(entry[key]), + })) + } + /> + ); + }} + /> +
+
+
+
+ amount(measure, value)} /> +
+
+ ); +} + +/** The measure's total per weekday over the year; the rows spread to the calendar column's height. */ +function Weekdays({ year, measure }: { readonly year: Metrics; readonly measure: Measure }) { + const weekdays = useMemo(() => { + const sums = new Map(); + for (const day of year.days) sums.set(weekdayOf(day), (sums.get(weekdayOf(day)) ?? 0) + day[measure]); + return WEEKDAYS.map(({ day, label }) => ({ label, total: sums.get(day) ?? 0 })); + }, [year.days, measure]); + const max = Math.max(...weekdays.map((weekday) => weekday.total), 0); + const { label, format } = MEASURE[measure]; + return ( +
+

By Weekday

+
    + {weekdays.map((weekday) => ( +
  • + 0 && weekday.total === max ? "text-ink" : "text-ink-muted"}`}> + {weekday.label} + + + + {format(weekday.total)} + +
  • + ))} +
+
+ ); +} + +function Streaks({ year, measure }: { readonly year: Metrics; readonly measure: Measure }) { + const busiest = year.days.reduce( + (best, day) => (day[measure] > (best?.[measure] ?? 0) ? day : best), + undefined, + ); + return ( +
+ + + + +
+ ); +} + +const days = (count: number) => `${formatCount(count)} ${count === 1 ? "Day" : "Days"}`; + +function Stat({ label, value }: { readonly label: string; readonly value: string }) { + return ( +
+
{label}
+
+ {value} +
+
+ ); +} + +// --------------------------------------------------------------------------------------------- + +const HOURS = Array.from({ length: 24 }, (_, hour) => String(hour).padStart(2, "0")); +const HOUR_TICKS = [0, 6, 12, 18, 23].map((at) => ({ at, label: HOURS[at] ?? "" })); +const MODEL_ROW = + "grid grid-cols-[minmax(0,1fr)_48px_64px] items-center gap-x-3 @xs:grid-cols-[minmax(0,1fr)_48px_48px_64px]"; + +/** A picked day at a glance: headline numbers, tokens through the day by model, then its sessions. */ +function DayDetail({ + day, + edges, + scope, + colors, + onClear, +}: { + readonly day: Day; + readonly edges: ReadonlyArray; + readonly scope: Scope; + readonly colors: ReadonlyMap; + readonly onClear: () => void; +}) { + // Until the day's hours arrive its chart stays empty; its totals are already here. + const hours = Option.getOrElse(AsyncResult.value(useAtomValue(hoursAtom(hoursKey(scope, day.day)))), () => []); + const spent = modelsOn(day, edges, hours); + const name = modelNamer(spent); + const models = spent.map((row) => ({ ...row, label: name(row), color: colors.get(row.key) ?? OTHER })); + const sessions = listed(day.bySession, "tokens"); + const named = models.filter((row) => colors.has(row.key)); + const rest = models.filter((row) => !colors.has(row.key)); + const stacks = [ + ...named.map((row) => ({ label: row.label, color: row.color, values: row.hours })), + ...(rest.length === 0 + ? [] + : [ + { + label: "Other", + color: OTHER, + values: HOURS.map((_, hour) => rest.reduce((sum, row) => sum + (row.hours[hour] ?? 0), 0)), + }, + ]), + ]; + const top = sessions[0]?.tokens ?? 0; + return ( +
+
+

{formatDayLong(day.date)}

+ +
+ {day.bySession.length === 0 ? ( +

No Work on This Day

+ ) : ( + <> +
+ {[ + ...HEADLINE.map((key) => ({ label: MEASURE[key].label, value: MEASURE[key].format(day[key]) })), + { label: "Model Time", value: formatDuration(day.modelTime) }, + ].map((stat) => ( +
+
{stat.label}
+
+ {stat.value} +
+
+ ))} +
+
+
+

Through the Day

+ `${hour}:00`)} + ticks={HOUR_TICKS} + format={formatTokens} + label="Tokens by model per hour" + height={120} + /> +
+
+

Models Used

+ ({ label: row.label, value: row.tokens, color: row.color }))} + format={formatTokens} + label="Tokens by model" + /> +
    + {models.map((row) => ( +
  • + + + + + {row.label} + + + + {row.median === undefined ? "–" : formatQuantile(row.median)} + + + {formatTokens(row.tokens)} + + + {unpriced(row) ? "Unpriced" : formatCost(row.cost)} + +
  • + ))} +
+
+
+
+

+ Sessions {formatCount(day.bySession.length)} +

+
    + {sessions.map((line) => ( + + ))} +
+
+ + )} +
+ ); +} + +// --------------------------------------------------------------------------------------------- + +interface Week { + readonly slots: ReadonlyArray; + readonly label: string; + readonly sessions: number; + readonly prompts: number; + readonly tokens: number; + readonly cost: number; +} + +/** Calendar weeks, Sunday first like the heatmap, newest first. */ +function weeksOf(days: ReadonlyArray): ReadonlyArray { + const groups: Array> = []; + let current: Array | undefined; + for (const day of days) { + const weekday = weekdayOf(day); + if (current === undefined || weekday === 0) { + current = Array.from({ length: 7 }, () => undefined); + groups.push(current); + } + current[weekday] = day; + } + return groups.reverse().map((slots) => { + const present = slots.filter((slot): slot is Day => slot !== undefined); + const first = present[0]; + const last = present.at(-1); + const sum = (pick: (day: Day) => number) => present.reduce((total, day) => total + pick(day), 0); + return { + slots, + label: + first === undefined || last === undefined + ? "" + : last.ago === 0 + ? "This Week" + : rangeLabel(first.date, last.date), + sessions: sessionsIn(present).length, + prompts: sum((day) => day.prompts), + tokens: sum((day) => day.tokens), + cost: sum((day) => day.cost), + }; + }); +} + +/** "Nov 8 – 14", or "Oct 30 – Nov 5" across a month. */ +function rangeLabel(first: number, last: number) { + if (first === last) return monthDay.format(first); + const start = new Date(first); + const end = new Date(last); + return start.getMonth() === end.getMonth() + ? `${monthDay.format(start)} – ${end.getDate()}` + : `${monthDay.format(start)} – ${monthDay.format(end)}`; +} + +const LEDGER = "grid grid-cols-[minmax(0,1fr)_84px_repeat(4,56px)] items-center gap-x-3"; + +/** The period week by week, folded to the latest weeks. */ +export function Weeks({ metrics }: { readonly metrics: Metrics }) { + const [all, setAll] = useState(false); + const weeks = useMemo(() => weeksOf(metrics.days), [metrics.days]); + const shown = all ? weeks : weeks.slice(0, WEEKS_FOLDED); + // One scale for every row, so a tall bar means the same in any week. + const max = Math.max(...metrics.days.map((day) => day.steps), 1); + const label = RANGES.find(({ key }) => key === metrics.range)?.label ?? ""; + return ( + +
+
+ Week + Requests + Sessions + Prompts + Tokens + Cost +
+
    + {shown.map((week) => ( +
  • + {week.label} + + {week.slots.map((slot, index) => ( + + {slot === undefined ? null : slot.steps === 0 ? ( + + ) : ( + + )} + + ))} + + {formatCount(week.sessions)} + {formatCount(week.prompts)} + {formatTokens(week.tokens)} + {formatCost(week.cost)} +
  • + ))} +
+ {shown.length < weeks.length && ( + + )} +
+
+ ); +} diff --git a/apps/web/src/metrics/charts.tsx b/apps/web/src/metrics/charts.tsx new file mode 100644 index 00000000..0eb3bc39 --- /dev/null +++ b/apps/web/src/metrics/charts.tsx @@ -0,0 +1,747 @@ +import { type MouseEvent, type ReactNode, type Ref, useCallback, useLayoutEffect, useRef, useState } from "react"; + +/* Chart pieces and the card they sit in. Marks carry color; text stays in ink tokens. */ + +/** Categorical slots, assigned in this order and never cycled; what is past the last folds into Other. */ +const SERIES = [ + "light-dark(#2a78d6, #3987e5)", + "light-dark(#eb6834, #d95926)", + "light-dark(#1baf7a, #199e70)", + "light-dark(#eda100, #c98500)", + "light-dark(#e87ba4, #d55181)", +] as const; +export const OTHER = "light-dark(#b4b3ad, #5c5b56)"; +/** One hue for a lone series: magnitude, not identity. */ +export const SINGLE = SERIES[0]; +/** Neutral ink at `percent` over the surface: for parts of one thing, never confused with a series. */ +export const ink = (percent: number) => `color-mix(in oklab, var(--cw-ink) ${percent}%, var(--cw-frame))`; +/** An ordered scale in one hue, least to most; the first step is "none" and stays neutral. */ +export const RAMP = [ + "light-dark(#d9d8d3, #3a3936)", + "light-dark(#cfe1f9, #16406f)", + "light-dark(#a3c6f2, #184f95)", + "light-dark(#6da7ec, #1c5cab)", + "light-dark(#2a78d6, #3987e5)", + "light-dark(#1a5bb0, #6ea8ee)", + "light-dark(#104281, #9ec5f4)", +] as const; +/** Quiet to busy, one hue; the first step is an empty day. */ +const HEAT = [ + "color-mix(in oklab, var(--cw-ink) 7%, transparent)", + "light-dark(#b7d3f6, #184f95)", + "light-dark(#6da7ec, #1c5cab)", + "light-dark(#2a78d6, #3987e5)", + "light-dark(#104281, #9ec5f4)", +] as const; + +/** A color for each of the first `SERIES.length` keys, in order; the rest get none and fold into Other. */ +export const palette = (keys: ReadonlyArray): ReadonlyMap => + new Map(SERIES.flatMap((color, index) => (keys[index] === undefined ? [] : [[keys[index], color] as const]))); + +/** Rows with a color keep it, in order; the rest are summed into one trailing Other row and also returned as `rest`. */ +export function fold( + rows: ReadonlyArray, + colors: ReadonlyMap, + sum: (rest: ReadonlyArray) => Row, +): { readonly parts: ReadonlyArray; readonly rest: ReadonlyArray } { + const head = rows.flatMap((row) => { + const color = colors.get(row.key); + return color === undefined ? [] : [{ ...row, color }]; + }); + const rest = rows.filter((row) => !colors.has(row.key)); + return { parts: rest.length === 0 ? head : [...head, { ...sum(rest), color: OTHER }], rest }; +} + +/** Consecutive runs of `size`, the last run short when the items do not divide evenly. */ +export function chunks
(items: ReadonlyArray, size: number): ReadonlyArray> { + const runs: Array> = []; + for (let start = 0; start < items.length; start += size) runs.push(items.slice(start, start + size)); + return runs; +} + +/** + * A round number at or above `max` whose half is also round, so both ticks read clean: whole from 1 + * up, and decimal below it, where only a fraction like a cost can land. + */ +function niceMax(max: number) { + if (max === 0 || (max >= 1 && max <= 2)) return 2; + const half = max / 2; + const magnitude = 10 ** Math.floor(Math.log10(half)); + const step = + [1, 1.5, 2, 2.5, 3, 4, 5, 6, 8, 10].find( + (candidate) => candidate * magnitude >= half && (max < 1 || Number.isInteger(candidate * magnitude)), + ) ?? 10; + return 2 * step * magnitude; +} + +export function Card({ + title, + caption, + info, + action, + className = "", + children, +}: { + readonly title: string; + readonly caption?: string; + /** A note on how to read the card, after the caption. */ + readonly info?: ReactNode; + /** A control at the header's end. */ + readonly action?: ReactNode; + readonly className?: string; + readonly children: ReactNode; +}) { + return ( +
+
+

{title}

+ {caption !== undefined && ( + + {caption} + + )} + {info !== undefined && {info}} + {action} +
+ {children} +
+ ); +} + +/** A few mutually exclusive options in a row. */ +export function Segmented({ + options, + value, + onChange, + label, + className = "", +}: { + readonly options: ReadonlyArray<{ readonly key: Key; readonly label: string }>; + readonly value: Key; + readonly onChange: (key: Key) => void; + readonly label: string; + readonly className?: string; +}) { + return ( +
+ {options.map((option) => ( + + ))} +
+ ); +} + +// --------------------------------------------------------------------------------------------- +// Hover tooltip: one floating card that follows the pointer. + +interface Tip { + readonly x: number; + readonly y: number; + readonly content: ReactNode; +} + +function useTip() { + const [tip, setTip] = useState(null); + const show = useCallback( + (event: MouseEvent, content: ReactNode) => setTip({ x: event.clientX, y: event.clientY, content }), + [], + ); + const hide = useCallback(() => setTip(null), []); + const flip = tip !== null && tip.x > window.innerWidth - 260; + const element = + tip === null ? null : ( +
+ {tip.content} +
+ ); + return { show, hide, element }; +} + +/** A tooltip's body: a title, then one row per value with its swatch. */ +export function TipRows({ + title, + rows, +}: { + readonly title: string; + readonly rows: ReadonlyArray<{ readonly label: string; readonly value: string; readonly color?: string }>; +}) { + return ( +
+ {title} + {rows.map((row) => ( + + {row.color !== undefined && ( + + )} + {row.label} + {row.value} + + ))} +
+ ); +} + +// --------------------------------------------------------------------------------------------- + +interface Stack { + readonly label: string; + readonly color: string; + /** One value per column, oldest first. */ + readonly values: ReadonlyArray; + /** This stack's tooltip value, when the chart's `format` does not fit it. */ + readonly format?: (value: number) => string; +} + +/** + * Stacked columns over time on one axis. Columns cap at 24px and leave the rest of their slot as + * air; segments are parted by a 2px gap of the surface. + */ +export function Columns({ + stacks, + titles, + ticks, + format, + label, + height = 160, +}: { + readonly stacks: ReadonlyArray; + /** A column's name in its tooltip. */ + readonly titles: ReadonlyArray; + /** The few x labels worth showing, by column index. */ + readonly ticks: ReadonlyArray<{ readonly at: number; readonly label: string }>; + readonly format: (value: number) => string; + /** What the chart shows, for a screen reader. */ + readonly label: string; + readonly height?: number; +}) { + const tip = useTip(); + const count = titles.length; + const sums = titles.map((_, index) => stacks.reduce((sum, stack) => sum + (stack.values[index] ?? 0), 0)); + const top = niceMax(Math.max(...sums, 0)); + return ( + +
+ {sums.map((sum, index) => ( +
+ tip.show( + event, + ({ + label: stack.label, + value: (stack.format ?? format)(stack.values[index] ?? 0), + color: stack.color, + })), + { label: "Total", value: format(sum) }, + ] + } + />, + ) + } + > +
120 ? 0 : "12%" }} + > + {stacks.map((stack) => + (stack.values[index] ?? 0) === 0 ? null : ( + + ), + )} +
+
+ ))} +
+
+ ); +} + +/** The frame a time chart sits in: y labels at zero, half and top, a hairline grid, x ticks below. */ +function Plot({ + label, + top, + format, + height, + ticks, + count, + area, + onMouseMove, + onMouseLeave, + tooltip, + children, +}: { + readonly label: string; + readonly top: number; + readonly format: (value: number) => string; + readonly height: number; + readonly ticks: ReadonlyArray<{ readonly at: number; readonly label: string }>; + /** Columns the x axis is split into; a tick sits at its column's centre. */ + readonly count: number; + readonly area?: Ref; + readonly onMouseMove?: (event: MouseEvent) => void; + readonly onMouseLeave: () => void; + readonly tooltip: ReactNode; + readonly children: ReactNode; +}) { + return ( +
+
+
+ {format(top)} + {format(top / 2)} + {format(0)} +
+
+ {[0, 50, 100].map((position) => ( + + ))} + {children} +
+
+
+ {ticks.map((tick) => ( + = count - 1 + ? { right: 0 } + : { left: `${((tick.at + 0.5) / count) * 100}%`, transform: "translateX(-50%)" } + } + > + {tick.label} + + ))} +
+ {tooltip} +
+ ); +} + +interface Series { + readonly label: string; + readonly color: string; + /** One value per column, oldest first; undefined is a gap, not a zero. */ + readonly values: ReadonlyArray; +} + +/** The defined points, as [column, value] pairs. */ +const pointsOf = (values: ReadonlyArray) => + values.flatMap((value, index) => (value === undefined ? [] : [[index, value] as const])); + +/** + * Lines over time on one axis from zero. A line runs straight across the days it has no value, so + * a series used now and then still reads as a trend; the tooltip shows those days as empty. Drawn + * in measured pixels rather than a stretched viewBox, so markers stay round. + */ +export function Lines({ + series, + titles, + ticks, + format, + label, + height = 160, +}: { + readonly series: ReadonlyArray; + /** A column's name in its tooltip. */ + readonly titles: ReadonlyArray; + readonly ticks: ReadonlyArray<{ readonly at: number; readonly label: string }>; + readonly format: (value: number) => string; + readonly label: string; + readonly height?: number; +}) { + const tip = useTip(); + const area = useRef(null); + const [width, setWidth] = useState(0); + const [hover, setHover] = useState(undefined); + useLayoutEffect(() => { + const box = area.current; + if (box === null) return; + const fit = () => setWidth(box.clientWidth); + fit(); + const observer = new ResizeObserver(fit); + observer.observe(box); + return () => observer.disconnect(); + }, []); + const count = titles.length; + const defined = series.flatMap((line) => line.values.filter((value) => value !== undefined)); + const top = niceMax(Math.max(...defined, 0)); + const x = (index: number) => ((index + 0.5) / count) * width; + const y = (value: number) => height - (value / top) * height; + const dot = (key: string, index: number, value: number, color: string, radius: number) => ( + + ); + return ( + { + const box = event.currentTarget.getBoundingClientRect(); + const index = Math.min( + count - 1, + Math.max(0, Math.floor(((event.clientX - box.left) / box.width) * count)), + ); + setHover(index); + tip.show( + event, + { + const value = line.values[index]; + return { label: line.label, value: value === undefined ? "–" : format(value), color: line.color }; + })} + />, + ); + }} + onMouseLeave={() => { + setHover(undefined); + tip.hide(); + }} + tooltip={tip.element} + > + {width > 0 && ( + + {hover !== undefined && ( + + )} + {series.map((line) => { + const points = pointsOf(line.values); + const last = points.at(-1); + return ( + + `${at === 0 ? "M" : "L"}${x(index)} ${y(value)}`) + .join("")} + fill="none" + strokeWidth={2} + strokeLinejoin="round" + strokeLinecap="round" + style={{ stroke: line.color }} + /> + {last !== undefined && dot("end", last[0], last[1], line.color, 4)} + {hover !== undefined && + hover !== last?.[0] && + line.values[hover] !== undefined && + dot("hover", hover, line.values[hover], line.color, 3.5)} + + ); + })} + + )} + + ); +} + +/** Swatch and name for each series; present whenever a chart has more than one. */ +export function Legend({ + items, +}: { + readonly items: ReadonlyArray<{ readonly label: string; readonly color: string }>; +}) { + return ( +
    + {items.map((item) => ( +
  • + + {item.label} +
  • + ))} +
+ ); +} + +/** A trend at a glance: a 1.5px line over a faint wash, no axes. Decorative; its tile carries the number. */ +export function Spark({ + values, + color = SINGLE, + className = "h-8 w-full", +}: { + readonly values: ReadonlyArray; + readonly color?: string; + readonly className?: string; +}) { + const max = Math.max(...values, 0); + if (values.length < 2 || max === 0) + return ( + + + + ); + const points = values.map((value, index) => [(index / (values.length - 1)) * 100, 30 - (value / max) * 28] as const); + const line = points.map(([x, y], index) => `${index === 0 ? "M" : "L"}${x.toFixed(2)} ${y.toFixed(2)}`).join(""); + return ( + + + + + ); +} + +/** Parts of a whole on one line, parted by 2px gaps. Rows too small to see still get a sliver. */ +export function Share({ + parts, + format, + label, +}: { + readonly parts: ReadonlyArray<{ readonly label: string; readonly value: number; readonly color: string }>; + readonly format: (value: number) => string; + readonly label: string; +}) { + const tip = useTip(); + const sum = parts.reduce((all, part) => all + part.value, 0); + return ( +
+ {sum === 0 ? ( + + ) : ( + parts.map((part) => + part.value === 0 ? null : ( + + tip.show( + event, + , + ) + } + /> + ), + ) + )} + {tip.element} +
+ ); +} + +/** A value against the largest in its column: the bar a table row carries. */ +export function Meter({ + value, + max, + color = SINGLE, +}: { + readonly value: number; + readonly max: number; + readonly color?: string; +}) { + return ( + + 0 ? 2 : 0)}%`, + backgroundColor: color, + }} + /> + + ); +} + +/** + * The upper bounds of the lower three heat levels: quartiles of the active values, so one outlier day + * does not wash out the rest. + */ +export function heatCuts(values: ReadonlyArray): readonly [number, number, number] { + const active = values.filter((value) => value > 0).toSorted((a, b) => a - b); + const cut = (share: number) => active[Math.min(active.length - 1, Math.floor(active.length * share))] ?? 0; + return [cut(0.25), cut(0.5), cut(0.75)]; +} + +/** A calendar of days, a column per week with Sunday on top, shaded by `heatCuts` levels. */ +export function Heat({ + days, + cuts, + title, + onPick, + picked, + cell = 11, +}: { + /** Oldest first; `weekday` 0 is Sunday. */ + readonly days: ReadonlyArray<{ readonly ago: number; readonly weekday: number; readonly value: number }>; + readonly cuts: readonly [number, number, number]; + readonly title: (day: { readonly ago: number; readonly value: number }) => ReactNode; + readonly onPick?: (ago: number) => void; + readonly picked?: number; + readonly cell?: number; +}) { + const tip = useTip(); + const level = (value: number) => (value === 0 ? 0 : 1 + cuts.filter((threshold) => value > threshold).length); + const lead = days[0]?.weekday ?? 0; + return ( +
+ {Array.from({ length: lead }, (_, index) => ( + + ))} + {days.map((day) => { + const style = { backgroundColor: HEAT[level(day.value)] }; + const className = `rounded-[2px] ${picked === day.ago ? "outline-2 outline-offset-1 outline-ink" : ""}`; + return onPick === undefined ? ( + tip.show(event, title(day))} + /> + ) : ( +
+ ); +} + +/** The scale under a heatmap; each step names its range on hover. */ +export function HeatScale({ + cuts, + format, +}: { + readonly cuts: readonly [number, number, number]; + readonly format: (value: number) => string; +}) { + const [low, mid, high] = cuts; + const ranges = [ + "None", + `Up to ${format(low)}`, + `${format(low)} – ${format(mid)}`, + `${format(mid)} – ${format(high)}`, + `Over ${format(high)}`, + ]; + return ( + + Less + {HEAT.map((color, index) => ( + + ))} + More + + ); +} diff --git a/apps/web/src/metrics/data.ts b/apps/web/src/metrics/data.ts new file mode 100644 index 00000000..301076d1 --- /dev/null +++ b/apps/web/src/metrics/data.ts @@ -0,0 +1,735 @@ +import { type MetricsHour, type MetricsRow, type MetricsUsage, readTopics } from "@codeworksh/contract"; +import { Effect, Stream } from "effect"; +import { Atom } from "effect/reactivity"; + +import { Server } from "../kernel/rpc"; + +/* + * The Metrics page reads a year of usage once, at the server's grain (local day × session × + * provider × model × thinking level), and derives every period, card and colour from it here. + */ + +export type RangeKey = "7d" | "30d" | "90d" | "1y"; +export const RANGES: ReadonlyArray<{ readonly key: RangeKey; readonly label: string; readonly days: number }> = [ + { key: "7d", label: "7 Days", days: 7 }, + { key: "30d", label: "30 Days", days: 30 }, + { key: "90d", label: "90 Days", days: 90 }, + { key: "1y", label: "Year", days: 365 }, +]; +const YEAR = 365; +const daysOf = (range: RangeKey) => RANGES.find(({ key }) => key === range)?.days ?? 30; + +// --------------------------------------------------------------------------------------------- +// Reads. + +/** What the page is filtered to; null is every one. */ +export interface Scope { + readonly machineId: string | null; + readonly projectId: string | null; +} + +const zone = Intl.DateTimeFormat().resolvedOptions().timeZone; + +const pad = (value: number) => String(value).padStart(2, "0"); +/** A local day as the server takes it, `YYYY-MM-DD`. */ +export const localDay = (date: Date) => `${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())}`; +/** Local noon `ago` days before `day`: safe to format across a clock change. */ +const noonOf = (day: string, ago: number) => { + const [year = 0, month = 1, date = 1] = day.split("-").map(Number); + return new Date(year, month - 1, date - ago, 12).getTime(); +}; + +const scoped = (scope: Scope) => ({ + ...(scope.machineId === null ? {} : { machineId: scope.machineId }), + ...(scope.projectId === null ? {} : { projectId: scope.projectId }), +}); + +/** The year ending `today` for `scope`, as a `usageAtom` key. */ +export const usageKey = (scope: Scope, today: string) => + JSON.stringify({ zone, since: localDay(new Date(noonOf(today, YEAR - 1))), until: today, ...scoped(scope) }); + +/** One day of `scope` by hour, as an `hoursAtom` key. */ +export const hoursKey = (scope: Scope, day: string) => JSON.stringify({ zone, day, ...scoped(scope) }); + +interface Request { + readonly zone: string; + readonly machineId?: string; + readonly projectId?: string; +} + +/** `read` when subscribed, and again after each change to usage; a failure ends it until refreshed. */ +const follow = (read: Effect.Effect) => + Server.runtime.atom( + Stream.unwrap( + Server.use((client) => Effect.succeed(client("event.subscribe", { topics: readTopics.metrics }))), + ).pipe( + // `Synced` first, once subscribed: the first read can miss no change. + Stream.debounce("50 millis"), + Stream.mapEffect(() => read), + ), + ); + +export const usageAtom = Atom.family((key: string) => { + const request = JSON.parse(key) as Request & { readonly since: string; readonly until: string }; + return follow(Server.use((client) => client("metrics.usage", request))); +}); + +export const hoursAtom = Atom.family((key: string) => { + const request = JSON.parse(key) as Request & { readonly day: string }; + return follow(Server.use((client) => client("metrics.hours", request))); +}); + +// --------------------------------------------------------------------------------------------- +// The year, indexed. + +/** Every sum a row carries, each one additive. */ +const SUMMED = [ + "prompts", + "steps", + "input", + "output", + "reasoning", + "cacheRead", + "cacheWrite", + "cost", + "costInput", + "costOutput", + "costCacheRead", + "costCacheWrite", + "cacheSavings", + /** Ms, every timed request's duration summed. */ + "modelTime", + /** Requests with a duration: interrupted ones have none. */ + "timed", + /** Output of rows with any timed request, for speed. */ + "timedOutput", +] as const; +type Sums = Record<(typeof SUMMED)[number], number>; + +const zero = (): Sums => Object.fromEntries(SUMMED.map((field) => [field, 0])) as Sums; + +/** Bucket counts summed, bucket by bucket. */ +export const plus = (a: ReadonlyArray, b: ReadonlyArray) => + Array.from({ length: Math.max(a.length, b.length) }, (_, at) => (a[at] ?? 0) + (b[at] ?? 0)); + +export interface Project { + readonly id: string; + readonly name: string; +} +export interface Session { + readonly id: string; + readonly title: string; + readonly deleted: boolean; + readonly project: Project; +} + +/** A response row on its day. */ +interface Item extends Sums { + readonly ago: number; + readonly machineId: string; + readonly sessionId: string; + readonly provider: string; + readonly model: string; + readonly thinking: string; + readonly histogram: ReadonlyArray; +} + +/** Prompts, or one tool's calls, of a session on a day. */ +interface Count { + readonly ago: number; + readonly machineId: string; + readonly sessionId: string; + readonly name: string; + readonly count: number; +} + +export interface Totals extends Sums { + /** Sessions with any work in the period. */ + readonly sessions: number; + /** Every token billed: input, output, cache read and cache write. */ + readonly tokens: number; + readonly activeDays: number; + /** Timed requests per duration bucket. */ + readonly histogram: ReadonlyArray; +} + +interface Spend { + readonly tokens: number; + readonly cost: number; +} + +interface ModelSpend extends Spend { + readonly provider: string; + readonly model: string; + readonly timed: number; + readonly histogram: ReadonlyArray; +} + +interface SessionDay extends Spend { + readonly session: Session; + /** Threads worked that day. */ + readonly threads: number; +} + +export interface Day extends Totals { + readonly ago: number; + /** `YYYY-MM-DD`. */ + readonly day: string; + /** Local noon, for formatting. */ + readonly date: number; + /** The sessions worked that day, so a longer span can count each session once. */ + readonly worked: ReadonlyArray; + /** Its sessions' spend, most tokens first. */ + readonly bySession: ReadonlyArray; + /** By `modelKey`. */ + readonly models: ReadonlyMap; + readonly providers: ReadonlyMap; +} + +export interface Year { + /** Upper bounds in ms of the duration buckets. */ + readonly edges: ReadonlyArray; + /** Oldest first, every day of the year, quiet ones included. */ + readonly days: ReadonlyArray; + readonly items: ReadonlyArray; + readonly prompts: ReadonlyArray; + readonly tools: ReadonlyArray; + readonly sessions: ReadonlyMap; + readonly projects: ReadonlyArray; + readonly machines: ReadonlyArray<{ readonly id: string; readonly name: string }>; +} + +const UNKNOWN_PROJECT = "Unknown Project"; +const tokensOf = (sums: Pick) => + sums.input + sums.output + sums.cacheRead + sums.cacheWrite; +const modelKey = (provider: string, model: string) => `${provider}/${model}`; + +/** What cache reads saved over paying the input price for them, by the row's own prices. */ +const savingsOf = ({ tokens, cost }: MetricsRow) => + tokens.input > 0 && tokens.cacheRead > 0 + ? tokens.cacheRead * (cost.input / tokens.input - cost.cacheRead / tokens.cacheRead) + : 0; + +const itemOf = (row: MetricsRow, ago: number): Item => ({ + ago, + machineId: row.machineId, + sessionId: row.sessionId, + provider: row.provider, + model: row.model, + thinking: row.thinking, + histogram: row.durations, + prompts: 0, + steps: row.requests, + input: row.tokens.input, + output: row.tokens.output, + reasoning: row.tokens.reasoning, + cacheRead: row.tokens.cacheRead, + cacheWrite: row.tokens.cacheWrite, + cost: row.cost.total, + costInput: row.cost.input, + costOutput: row.cost.output, + costCacheRead: row.cost.cacheRead, + costCacheWrite: row.cost.cacheWrite, + cacheSavings: savingsOf(row), + modelTime: row.duration, + timed: row.timed, + timedOutput: row.timed > 0 ? row.tokens.output : 0, +}); + +function groupBy(items: ReadonlyArray
, key: (item: A) => K): Map> { + const groups = new Map>(); + for (const item of items) { + const group = groups.get(key(item)); + if (group === undefined) groups.set(key(item), [item]); + else group.push(item); + } + return groups; +} + +function total(items: ReadonlyArray, prompts: ReadonlyArray): Totals { + const sum = zero(); + let histogram: ReadonlyArray = []; + for (const item of items) { + for (const field of SUMMED) sum[field] += item[field]; + histogram = plus(histogram, item.histogram); + } + for (const prompt of prompts) sum.prompts += prompt.count; + return { + ...sum, + tokens: tokensOf(sum), + sessions: new Set([...items, ...prompts].map((item) => item.sessionId)).size, + activeDays: new Set(items.filter((item) => item.steps > 0).map((item) => item.ago)).size, + histogram, + }; +} + +const spendOf = (items: ReadonlyArray) => ({ + tokens: items.reduce((sum, item) => sum + tokensOf(item), 0), + cost: items.reduce((sum, item) => sum + item.cost, 0), +}); + +/** The response for the year ending `today`, indexed by day; a row outside the year is left out. */ +export function yearOf(usage: MetricsUsage, today: string): Year { + const dates = Array.from({ length: YEAR }, (_, ago) => localDay(new Date(noonOf(today, ago)))); + const agoOf = new Map(dates.map((day, ago) => [day, ago])); + const items = usage.rows.flatMap((row) => { + const ago = agoOf.get(row.day); + return ago === undefined ? [] : [itemOf(row, ago)]; + }); + const counts = (kind: "prompt" | "tool") => + usage.tallies.flatMap((tally) => { + const ago = agoOf.get(tally.day); + return tally.kind !== kind || ago === undefined + ? [] + : [{ ago, machineId: tally.machineId, sessionId: tally.sessionId, name: tally.name, count: tally.count }]; + }); + const prompts = counts("prompt"); + + const projects = usage.projects.map((project) => ({ id: project.id, name: project.name ?? UNKNOWN_PROJECT })); + const projectOf = new Map(projects.map((project) => [project.id, project])); + const sessions = new Map( + usage.sessions.map((session) => [ + session.id, + { + id: session.id, + title: session.title ?? "", + deleted: session.deleted, + project: projectOf.get(session.projectId) ?? { id: session.projectId, name: UNKNOWN_PROJECT }, + }, + ]), + ); + const threads = new Map(); + for (const row of usage.threads) { + const key = `${row.day}\u0000${row.sessionId}`; + threads.set(key, (threads.get(key) ?? 0) + row.threads); + } + + const itemsByDay = groupBy(items, (item) => item.ago); + const promptsByDay = groupBy(prompts, (prompt) => prompt.ago); + const days = dates + .map((day, ago): Day => { + const dayItems = itemsByDay.get(ago) ?? []; + const dayPrompts = promptsByDay.get(ago) ?? []; + const totals = total(dayItems, dayPrompts); + const models = new Map(); + for (const [key, group] of groupBy(dayItems, (item) => modelKey(item.provider, item.model))) { + const { provider, model } = group[0] ?? { provider: "", model: "" }; + const { timed, histogram } = total(group, []); + models.set(key, { ...spendOf(group), provider, model, timed, histogram }); + } + const providers = new Map( + [...groupBy(dayItems, (item) => item.provider)].map(([key, group]) => [key, spendOf(group)] as const), + ); + const worked = [...new Set([...dayItems, ...dayPrompts].map((item) => item.sessionId))]; + const byItems = groupBy(dayItems, (item) => item.sessionId); + return { + ...totals, + ago, + day, + date: noonOf(today, ago), + worked, + bySession: worked + .map((id) => ({ + ...spendOf(byItems.get(id) ?? []), + session: sessionOf(sessions, id), + threads: threads.get(`${day}\u0000${id}`) ?? 0, + })) + .sort((a, b) => b.tokens - a.tokens), + models, + providers, + }; + }) + .reverse(); + + return { + edges: usage.edges, + days, + items, + prompts, + tools: counts("tool"), + sessions, + projects, + machines: usage.machines, + }; +} + +/** A session the dictionary misses reads as deleted, so it folds away with them. */ +const sessionOf = (sessions: ReadonlyMap, id: string): Session => + sessions.get(id) ?? { id, title: "", deleted: true, project: { id: "", name: UNKNOWN_PROJECT } }; + +// --------------------------------------------------------------------------------------------- +// A period. + +type Dimension = "project" | "model" | "provider" | "thinking"; + +export interface SliceRow extends Totals { + readonly key: string; + readonly label: string; + /** A second line: a model's provider. */ + readonly hint: string | undefined; +} + +interface SessionRow extends Totals { + readonly session: Session; +} + +export interface Metrics { + readonly range: RangeKey; + /** Oldest first, every day of the period, quiet ones included. */ + readonly days: ReadonlyArray; + readonly totals: Totals; + /** The period before, for a change; the year has none. */ + readonly previous: Totals | undefined; + /** Consecutive active days ending today or yesterday. */ + readonly streak: number; + readonly bestStreak: number; + readonly sessions: ReadonlyArray; + /** Most calls first. */ + readonly tools: ReadonlyArray<{ readonly name: string; readonly calls: number }>; + readonly slice: (dimension: Dimension) => ReadonlyArray; + readonly edges: ReadonlyArray; +} + +/** Ordinal, least to most effort. */ +const THINKING_LEVELS = ["off", "minimal", "low", "medium", "high", "xhigh", "max"] as const; +const THINKING_LABEL: Partial> = { + off: "Off", + minimal: "Minimal", + low: "Low", + medium: "Medium", + high: "High", + xhigh: "Extra High", + max: "Max", +}; +/** A thinking level's place on the off → max scale. */ +export const levelIndex = (key: string) => THINKING_LEVELS.findIndex((level) => level === key); + +/** The metrics of a period of `year`. */ +export function query(year: Year, range: RangeKey): Metrics { + const span = daysOf(range); + const within = (list: ReadonlyArray, from: number, to: number) => + list.filter((item) => item.ago >= from && item.ago < to); + const items = within(year.items, 0, span); + const prompts = within(year.prompts, 0, span); + const days = year.days.slice(YEAR - span); + + let streak = 0; + for (let ago = (year.days.at(-1)?.steps ?? 0) > 0 ? 0 : 1; (year.days[YEAR - 1 - ago]?.steps ?? 0) > 0; ago++) + streak++; + let bestStreak = 0; + let run = 0; + for (const day of days) { + run = day.steps > 0 ? run + 1 : 0; + bestStreak = Math.max(bestStreak, run); + } + + const promptsBySession = groupBy(prompts, (prompt) => prompt.sessionId); + const itemsBySession = groupBy(items, (item) => item.sessionId); + const sessions = [...new Set([...itemsBySession.keys(), ...promptsBySession.keys()])].map((id) => ({ + ...total(itemsBySession.get(id) ?? [], promptsBySession.get(id) ?? []), + session: sessionOf(year.sessions, id), + })); + + const tools = new Map(); + for (const tool of within(year.tools, 0, span)) tools.set(tool.name, (tools.get(tool.name) ?? 0) + tool.count); + + return { + range, + days, + totals: total(items, prompts), + previous: + range === "1y" ? undefined : total(within(year.items, span, span * 2), within(year.prompts, span, span * 2)), + streak, + bestStreak, + sessions, + tools: [...tools].map(([name, calls]) => ({ name, calls })).sort((a, b) => b.calls - a.calls), + slice: (dimension) => { + const keyOf = (sessionId: string, item?: Item) => { + if (dimension === "project") { + const { project } = sessionOf(year.sessions, sessionId); + return { key: project.id, label: project.name, hint: undefined }; + } + if (item === undefined) return undefined; + if (dimension === "model") + return { key: modelKey(item.provider, item.model), label: item.model, hint: item.provider }; + if (dimension === "provider") return { key: item.provider, label: item.provider, hint: undefined }; + return { key: item.thinking, label: THINKING_LABEL[item.thinking] ?? item.thinking, hint: undefined }; + }; + const groups = new Map< + string, + { label: string; hint: string | undefined; items: Array; prompts: Array } + >(); + const into = (named: { key: string; label: string; hint: string | undefined }) => { + const group = groups.get(named.key) ?? { label: named.label, hint: named.hint, items: [], prompts: [] }; + groups.set(named.key, group); + return group; + }; + for (const item of items) { + const named = keyOf(item.sessionId, item); + if (named !== undefined) into(named).items.push(item); + } + // Prompts belong to a session, not a request, so only the project slice counts them. + if (dimension === "project") + for (const prompt of prompts) { + const named = keyOf(prompt.sessionId); + if (named !== undefined) into(named).prompts.push(prompt); + } + const rows = [...groups] + .map(([key, group]) => ({ + ...total(group.items, group.prompts), + key, + label: group.label, + hint: group.hint, + })) + // A model or provider whose requests all failed produced nothing; its requests still count by thinking level. + .filter((row) => row.tokens > 0 || dimension === "project" || dimension === "thinking"); + return dimension === "thinking" + ? rows.sort((a, b) => levelIndex(a.key) - levelIndex(b.key)) + : rows.sort((a, b) => b.prompts - a.prompts); + }, + edges: year.edges, + }; +} + +/** Each session once, across the given days. */ +export const sessionsIn = (days: ReadonlyArray): ReadonlyArray => [ + ...new Set(days.flatMap((day) => day.worked)), +]; + +/** A session list's line: a session, or every deleted one folded into one. */ +export type Listed = + | { + readonly kind: "session"; + readonly key: string; + readonly session: Session; + readonly threads: number | undefined; + readonly tokens: number; + readonly cost: number; + } + | { + readonly kind: "deleted"; + readonly key: string; + readonly count: number; + readonly tokens: number; + readonly cost: number; + }; + +/** Sessions most `by` first, the deleted ones summed into one line that sorts by its sum. */ +export function listed( + rows: ReadonlyArray<{ + readonly session: Session; + readonly tokens: number; + readonly cost: number; + readonly threads?: number; + }>, + by: "tokens" | "cost", +): ReadonlyArray { + const deleted = rows.filter((row) => row.session.deleted); + const lines: Array = rows + .filter((row) => !row.session.deleted) + .map((row) => ({ + kind: "session", + key: row.session.id, + session: row.session, + threads: row.threads, + tokens: row.tokens, + cost: row.cost, + })); + if (deleted.length > 0) + lines.push({ + kind: "deleted", + key: "deleted", + count: deleted.length, + tokens: deleted.reduce((sum, row) => sum + row.tokens, 0), + cost: deleted.reduce((sum, row) => sum + row.cost, 0), + }); + return lines.sort((a, b) => b[by] - a[by]); +} + +interface ModelDay extends Spend { + readonly key: string; + readonly provider: string; + readonly model: string; + /** Its median request duration that day; none when no request was timed. */ + readonly median: Quantile | undefined; + /** Tokens in each local hour, 0 to 23. */ + readonly hours: ReadonlyArray; +} + +/** A day's models, most tokens first, with their tokens by hour from that day's hourly read. */ +export function modelsOn(day: Day, edges: ReadonlyArray, hours: ReadonlyArray) { + const byModel = groupBy(hours, (row) => modelKey(row.provider, row.model)); + return [...day.models] + .filter(([, spend]) => spend.tokens > 0) + .map(([key, spend]): ModelDay => { + const perHour = Array.from({ length: 24 }, () => 0); + for (const row of byModel.get(key) ?? []) perHour[row.hour] = (perHour[row.hour] ?? 0) + row.tokens; + return { + key, + provider: spend.provider, + model: spend.model, + tokens: spend.tokens, + cost: spend.cost, + median: spend.timed === 0 ? undefined : quantile(spend.histogram, edges, 0.5), + hours: perHour, + }; + }) + .sort((a, b) => b.tokens - a.tokens); +} + +export interface ProjectOption { + readonly project: Project; + readonly prompts: number; + readonly sessions: number; +} + +/** Every project's prompts and active sessions in a period, on one machine or all. */ +export function projectOptions(year: Year, range: RangeKey, machineId: string | null): ReadonlyArray { + const span = daysOf(range); + const rows = new Map(year.projects.map((project) => [project.id, { prompts: 0, sessions: new Set() }])); + for (const item of [...year.items, ...year.prompts]) { + if (item.ago >= span || (machineId !== null && item.machineId !== machineId)) continue; + const row = rows.get(sessionOf(year.sessions, item.sessionId).project.id); + if (row === undefined) continue; + row.sessions.add(item.sessionId); + if ("count" in item) row.prompts += item.count; + } + return year.projects.map((project) => { + const row = rows.get(project.id); + return { project, prompts: row?.prompts ?? 0, sessions: row?.sessions.size ?? 0 }; + }); +} + +// --------------------------------------------------------------------------------------------- +// Response time, from duration buckets. + +/** A duration read from buckets; `open` when it fell in the last, unbounded one: at least `ms`. */ +interface Quantile { + readonly ms: number; + readonly open: boolean; +} + +/** + * The `q` quantile of bucket counts, interpolated linearly inside its bucket. Bucket i spans + * `edges[i - 1]` to `edges[i]`, the first from 0; the last is open-ended, so its lower edge stands in. + */ +export function quantile(histogram: ReadonlyArray, edges: ReadonlyArray, q: number) { + const sum = histogram.reduce((all, count) => all + count, 0); + if (sum === 0) return undefined; + const target = q * sum; + let seen = 0; + for (const [at, count] of histogram.entries()) { + if (count > 0 && seen + count >= target) { + const low = at === 0 ? 0 : (edges[at - 1] ?? 0); + const high = edges[at]; + if (high === undefined) return { ms: low, open: true } satisfies Quantile; + return { ms: low + ((high - low) * (target - seen)) / count, open: false } satisfies Quantile; + } + seen += count; + } + return undefined; +} + +export interface Response { + readonly median: Quantile; + readonly p95: Quantile; + /** Output tokens per second of request time; none when no time was recorded. */ + readonly speed: number | undefined; +} + +/** Response time over timed requests; durations are right-skewed, so median and p95, never a mean. */ +export function responseOf( + totals: Pick, + edges: ReadonlyArray, +): Response | undefined { + if (totals.timed === 0) return undefined; + const median = quantile(totals.histogram, edges, 0.5); + const p95 = quantile(totals.histogram, edges, 0.95); + if (median === undefined || p95 === undefined) return undefined; + return { + median, + p95, + speed: totals.modelTime > 0 ? totals.timedOutput / (totals.modelTime / 1000) : undefined, + }; +} + +/** Names a model by its id, adding its provider when the id occurs under more than one provider in `rows`. */ +export function modelNamer(rows: ReadonlyArray<{ readonly provider: string; readonly model: string }>) { + const providers = new Map>(); + for (const row of rows) providers.set(row.model, (providers.get(row.model) ?? new Set()).add(row.provider)); + return (row: { readonly provider: string; readonly model: string }) => + (providers.get(row.model)?.size ?? 0) > 1 ? `${row.model} · ${row.provider}` : row.model; +} + +/** A free or preview model reports tokens but no cost; "$0" would read as a bargain. */ +export const unpriced = (spend: Spend) => spend.cost === 0 && spend.tokens > 0; + +// --------------------------------------------------------------------------------------------- +// Formatting. + +const whole = new Intl.NumberFormat("en-US", { maximumFractionDigits: 0 }); +const compact = new Intl.NumberFormat("en-US", { notation: "compact", maximumFractionDigits: 1 }); +const dollars = new Intl.NumberFormat("en-US", { style: "currency", currency: "USD" }); +const dollarsWhole = new Intl.NumberFormat("en-US", { style: "currency", currency: "USD", maximumFractionDigits: 0 }); +const dollarsSmall = new Intl.NumberFormat("en-US", { + style: "currency", + currency: "USD", + minimumFractionDigits: 0, + maximumFractionDigits: 4, +}); +const shortDay = new Intl.DateTimeFormat(undefined, { month: "short", day: "numeric" }); +const longDay = new Intl.DateTimeFormat(undefined, { + weekday: "short", + month: "short", + day: "numeric", + year: "numeric", +}); + +export const formatCount = (value: number) => whole.format(value); +/** 48.3K, 1.2M, 2.4B. */ +export const formatTokens = (value: number) => compact.format(value); +/** Whole dollars from a thousand, cents from a cent, up to four decimals below; "<$0.0001" rather than a rounded zero. */ +export const formatCost = (value: number) => + value === 0 + ? "$0" + : value >= 1000 + ? dollarsWhole.format(value) + : value >= 0.01 + ? dollars.format(value) + : value >= 0.0001 + ? dollarsSmall.format(value) + : "<$0.0001"; +/** A day by its local noon. */ +export const formatDay = (date: number) => shortDay.format(date); +export const formatDayLong = (date: number) => longDay.format(date); +/** 850 ms, 4.6 s, 1m 12s, 30m, 2h 05m. */ +export function formatDuration(ms: number) { + if (ms === 0) return "0 s"; + if (Math.round(ms) < 1000) return `${Math.round(ms)} ms`; + const tenths = Math.round(ms / 100); + if (tenths < 600) return `${(tenths / 10).toFixed(1)} s`; + const seconds = Math.round(ms / 1000); + if (seconds < 3600) return `${Math.floor(seconds / 60)}m${seconds % 60 === 0 ? "" : ` ${pad(seconds % 60)}s`}`; + const minutes = Math.round(ms / 60_000); + return `${Math.floor(minutes / 60)}h ${pad(minutes % 60)}m`; +} +/** "4.6 s", or "≥ 30m" past the last bucket. */ +export const formatQuantile = (value: Quantile) => `${value.open ? "≥ " : ""}${formatDuration(value.ms)}`; +/** 52 tok/s. */ +export const formatSpeed = (perSecond: number) => `${Math.round(perSecond)} tok/s`; +export const formatPercent = (ratio: number) => `${Math.round(ratio * 100)}%`; +/** The change from `before` to `now` as a signed percent, or undefined when there is nothing to compare. */ +export function change(now: number, before: number | undefined) { + if (before === undefined || before === 0) return undefined; + return (now - before) / before; +} +export const formatChange = (ratio: number) => + `${ratio >= 0 ? "+" : "−"}${Math.abs(ratio) >= 10 ? "999+" : Math.round(Math.abs(ratio) * 100)}%`; +/** + * Share of input served from cache, or null with no input. A cache write counts as a miss: that + * input was processed in full. + */ +export function cacheHit(totals: Pick) { + const all = totals.input + totals.cacheRead + totals.cacheWrite; + return all === 0 ? null : totals.cacheRead / all; +} diff --git a/apps/web/src/metrics/filters.tsx b/apps/web/src/metrics/filters.tsx new file mode 100644 index 00000000..a4ac152d --- /dev/null +++ b/apps/web/src/metrics/filters.tsx @@ -0,0 +1,243 @@ +import { CaretUpDownIcon, CheckIcon, DesktopIcon, SquaresFourIcon } from "@phosphor-icons/react"; +import { type ReactNode, useEffect, useState } from "react"; + +import { Command, CommandEmpty, CommandGroup, CommandInput, CommandItem, CommandList } from "@/components/ui/command"; +import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover"; +import { Monogram } from "@/components/monogram"; +import { formatCount, type Project, type ProjectOption } from "./data"; + +const ALL = "all"; +const ROW = "min-h-7 gap-2 py-1 text-[13px]"; +const GROUP = "[&_[cmdk-group-heading]]:text-2xs [&_[cmdk-group-heading]]:pb-1"; + +/** cmdk filters on the value, so it carries the name; the id keeps two same-named projects apart. */ +const valueOf = (item: { readonly id: string; readonly name: string }) => `${item.name}\u0000${item.id}`; + +/** Whether a key press belongs to a field rather than to the page's shortcuts. */ +const typing = (target: EventTarget | null) => + target instanceof HTMLElement && + (target.isContentEditable || target.closest("input, textarea, select, [role=textbox]") !== null); + +/** The current pick's check, or the same room empty, so counts line up on every row. */ +function Check({ on }: { readonly on: boolean }) { + return on ? ( + + ) : ( + + ); +} + +/** A header filter's button: what is picked, and the list it opens. */ +function Filter({ + label, + icon, + name, + open, + onOpenChange, + width, + children, +}: { + /** What it filters, for a screen reader: "Project". */ + readonly label: string; + readonly icon: ReactNode; + readonly name: string; + readonly open: boolean; + readonly onOpenChange: (open: boolean) => void; + readonly width: string; + readonly children: ReactNode; +}) { + return ( + + + + + + {children} + + + ); +} + +/** The page's project scope: every project, or one picked from a filterable list ordered by activity. */ +export function ProjectSelect({ + value, + onChange, + options, +}: { + readonly value: string | null; + readonly onChange: (projectId: string | null) => void; + readonly options: ReadonlyArray; +}) { + const [open, setOpen] = useState(false); + const [search, setSearch] = useState(""); + const [highlight, setHighlight] = useState(ALL); + + const active = options.filter((row) => row.prompts > 0).toSorted((a, b) => b.prompts - a.prompts); + const idle = options + .filter((row) => row.prompts === 0) + .toSorted((a, b) => a.project.name.localeCompare(b.project.name)); + const sessions = options.reduce((sum, row) => sum + row.sessions, 0); + const selected = options.find((row) => row.project.id === value)?.project; + + const show = (next: boolean) => { + if (next) { + setSearch(""); + setHighlight(selected === undefined ? ALL : valueOf(selected)); + } + setOpen(next); + }; + const pick = (projectId: string | null) => { + onChange(projectId); + setOpen(false); + }; + + useEffect(() => { + const onKeyDown = (event: KeyboardEvent) => { + if (event.key.toLowerCase() !== "p" || event.metaKey || event.ctrlKey || event.altKey || event.shiftKey) + return; + if (event.repeat || typing(event.target)) return; + event.preventDefault(); + show(true); + }; + window.addEventListener("keydown", onKeyDown); + return () => window.removeEventListener("keydown", onKeyDown); + }); + + const row = (project: Project, prompts: number | undefined) => ( + pick(project.id)} className={ROW}> + + + {project.name} + + + {prompts !== undefined && formatCount(prompts)} + + + + ); + + return ( + + ) : ( + + ) + } + name={selected?.name ?? "All Projects"} + open={open} + onOpenChange={show} + width="w-96" + > + +
+ + + P + +
+ + + No Projects Match “{search.trim()}” + + + pick(null)} className={ROW}> + + All Projects + + {formatCount(sessions)} + + + + + {active.length > 0 && ( + + {active.map(({ project, prompts }) => row(project, prompts))} + + )} + {idle.length > 0 && ( + + {idle.map(({ project }) => row(project, undefined))} + + )} + +
+
+ ); +} + +/** The page's machine scope: every machine, or one by name. */ +export function MachineSelect({ + value, + onChange, + machines, +}: { + readonly value: string | null; + readonly onChange: (machineId: string | null) => void; + readonly machines: ReadonlyArray<{ readonly id: string; readonly name: string }>; +}) { + const [open, setOpen] = useState(false); + const sorted = machines.toSorted((a, b) => a.name.localeCompare(b.name)); + const selected = machines.find((machine) => machine.id === value); + const pick = (machineId: string | null) => { + onChange(machineId); + setOpen(false); + }; + return ( + } + name={selected?.name ?? "All Machines"} + open={open} + onOpenChange={setOpen} + width="w-64" + > + + + + pick(null)} className={ROW}> + All Machines + + + + {sorted.map((machine) => ( + pick(machine.id)} + className={ROW} + > + + {machine.name} + + + + + ))} + + + + + ); +} diff --git a/apps/web/src/metrics/metrics.tsx b/apps/web/src/metrics/metrics.tsx new file mode 100644 index 00000000..dc5cc617 --- /dev/null +++ b/apps/web/src/metrics/metrics.tsx @@ -0,0 +1,909 @@ +import { ChartBarIcon, InfoIcon } from "@phosphor-icons/react"; +import { type ReactNode, useMemo, useState } from "react"; + +import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover"; +import { ProviderIcon } from "@/components/provider"; +import { Activity, Weeks } from "./activity"; +import { + Card, + chunks, + Columns, + fold, + ink, + Legend, + Lines, + Meter, + OTHER, + palette, + RAMP, + Segmented, + Share, + Spark, +} from "./charts"; +import { + cacheHit, + change, + type Day, + formatChange, + formatCost, + formatCount, + formatDay, + formatDuration, + formatPercent, + formatQuantile, + formatSpeed, + formatTokens, + levelIndex, + listed, + type Metrics, + modelNamer, + plus, + quantile, + RANGES, + responseOf, + type Scope, + sessionsIn, + type SliceRow, + type Totals, + unpriced, +} from "./data"; +import { SessionLine } from "./session"; + +/** The page below the header: the period's metrics, already scoped to the chosen machine and project. */ +export function MetricsBody({ + year, + metrics, + scope, + projectName, + onProject, +}: { + /** The whole year, for the calendar and the colours. */ + readonly year: Metrics; + readonly metrics: Metrics; + readonly scope: Scope; + /** The project the page is scoped to, for an empty state to name. */ + readonly projectName: string | undefined; + /** Scopes the page to a project, or back to all with null. */ + readonly onProject: (projectId: string | null) => void; +}) { + // Colors follow the year's ranking, so a model keeps its color in every period and card. + const modelColors = useMemo(() => palette(byTokens(year.slice("model")).map((row) => row.key)), [year]); + const providerColors = useMemo(() => palette(byTokens(year.slice("provider")).map((row) => row.key)), [year]); + const models = useMemo( + () => fold(byTokens(metrics.slice("model")).map(partOf), modelColors, rest), + [metrics, modelColors], + ); + const providers = useMemo( + () => fold(byTokens(metrics.slice("provider")).map(partOf), providerColors, rest), + [metrics, providerColors], + ); + // Where a label shows only the model id, an id under two providers also names its provider. + const named = useMemo(() => { + const name = modelNamer(models.parts.flatMap(modelOf).concat(models.rest.flatMap(modelOf))); + return models.parts.map((part) => { + const [model] = modelOf(part); + return model === undefined ? part : { ...part, label: name(model) }; + }); + }, [models]); + const period = RANGES.find(({ key }) => key === metrics.range)?.label.toLowerCase() ?? ""; + if (metrics.totals.steps === 0 && year.totals.steps === 0) + return ; + return ( +
+ + + +
+ + +
+ + + + + + +
+ ); +} + +function Empty({ projectName, period }: { readonly projectName: string | undefined; readonly period: string }) { + return ( +
+ + + +
+

No Activity Yet

+

+ Nothing ran in {projectName ?? "any project"} over the last{" "} + {period}. +

+
+
+ ); +} + +// --------------------------------------------------------------------------------------------- + +type Stat = "cost" | "tokens" | "sessions"; + +/** A year of daily points is too dense for a small chart; weeks read better. */ +const buckets = (metrics: Metrics) => chunks(metrics.days, metrics.range === "1y" ? 7 : 1); + +const sumOf = (days: ReadonlyArray, pick: (day: Day) => number) => days.reduce((sum, day) => sum + pick(day), 0); + +/** Each session counted once per bucket. */ +const trend = (metrics: Metrics, key: Stat) => + buckets(metrics).map((days) => (key === "sessions" ? sessionsIn(days).length : sumOf(days, (day) => day[key]))); + +const hitLabel = (totals: Pick) => { + const hit = cacheHit(totals); + return hit === null ? "–" : formatPercent(hit); +}; + +/** A change badge in neutral ink, or nothing when there is nothing to compare. */ +function Delta({ ratio, period }: { readonly ratio: number | undefined; readonly period: string }) { + if (ratio === undefined) return null; + return ( + + {ratio >= 0 ? "↑" : "↓"} {formatChange(ratio)} + + ); +} + +function Stats({ metrics, period }: { readonly metrics: Metrics; readonly period: string }) { + const { totals, previous } = metrics; + const tiles: ReadonlyArray<{ key: Stat; label: string; value: string; hint: string | undefined }> = [ + { + key: "cost", + label: "Cost", + value: formatCost(totals.cost), + hint: totals.sessions === 0 ? undefined : `${formatCost(totals.cost / totals.sessions)} per Session`, + }, + { + key: "tokens", + label: "Tokens", + value: formatTokens(totals.tokens), + hint: `${hitLabel(totals)} Cached`, + }, + { + key: "sessions", + label: "Sessions", + value: formatCount(totals.sessions), + hint: `${formatCount(totals.prompts)} Prompts`, + }, + ]; + return ( +
+ {tiles.map((tile) => ( +
+

{tile.label}

+
+ + {tile.value} + + +
+ {tile.hint} + +
+ ))} +
+ ); +} + +// --------------------------------------------------------------------------------------------- + +interface Part { + readonly key: string; + readonly label: string; + readonly hint: string | undefined; + readonly sessions: number; + readonly prompts: number; + readonly tokens: number; + readonly cost: number; + readonly input: number; + readonly cacheRead: number; + readonly cacheWrite: number; + readonly output: number; + readonly modelTime: number; + readonly timed: number; + readonly timedOutput: number; + readonly histogram: ReadonlyArray; +} +type Colored = Part & { readonly color: string }; + +const partOf = (row: SliceRow): Part => ({ + key: row.key, + label: row.label, + hint: row.hint, + sessions: row.sessions, + prompts: row.prompts, + tokens: row.tokens, + cost: row.cost, + input: row.input, + cacheRead: row.cacheRead, + cacheWrite: row.cacheWrite, + output: row.output, + modelTime: row.modelTime, + timed: row.timed, + timedOutput: row.timedOutput, + histogram: row.histogram, +}); + +const OTHER_KEY = "other"; + +/** A model slice row's provider and id: its hint and label; Other has none. */ +const modelOf = (part: Part) => + part.key === OTHER_KEY || part.hint === undefined ? [] : [{ provider: part.hint, model: part.label }]; + +const byTokens = (rows: ReadonlyArray) => rows.toSorted((a, b) => b.tokens - a.tokens); + +function rest(rows: ReadonlyArray): Part { + const sum = (pick: (row: Part) => number) => rows.reduce((total, row) => total + pick(row), 0); + return { + key: OTHER_KEY, + label: "Other", + hint: undefined, + sessions: sum((row) => row.sessions), + prompts: sum((row) => row.prompts), + tokens: sum((row) => row.tokens), + cost: sum((row) => row.cost), + input: sum((row) => row.input), + cacheRead: sum((row) => row.cacheRead), + cacheWrite: sum((row) => row.cacheWrite), + output: sum((row) => row.output), + modelTime: sum((row) => row.modelTime), + timed: sum((row) => row.timed), + timedOutput: sum((row) => row.timedOutput), + histogram: rows.reduce>((all, row) => plus(all, row.histogram), []), + }; +} + +type Metric = "tokens" | "cost"; +const METRICS: ReadonlyArray<{ readonly key: Metric; readonly label: string }> = [ + { key: "tokens", label: "Tokens" }, + { key: "cost", label: "Cost" }, +]; + +/** Up to four evenly spread column indices: both ends and two between. */ +const spread = (count: number) => + [...new Set([0, Math.round((count - 1) / 3), Math.round(((count - 1) * 2) / 3), count - 1])].filter((at) => at >= 0); + +/** The period's columns, a day each or a week for the year, with their tooltip names and x ticks. */ +function timeline(metrics: Metrics) { + const weekly = metrics.range === "1y"; + const columns = buckets(metrics); + const startOf = (at: number) => columns[at]?.[0]?.date ?? 0; + return { + weekly, + columns, + titles: columns.map((_, at) => (weekly ? `Week of ${formatDay(startOf(at))}` : formatDay(startOf(at)))), + ticks: spread(columns.length).map((at) => ({ at, label: formatDay(startOf(at)) })), + }; +} + +type View = "model" | "provider"; +const VIEWS: ReadonlyArray<{ readonly key: View; readonly label: string }> = [ + { key: "model", label: "Model" }, + { key: "provider", label: "Provider" }, +]; + +function Work({ + metrics, + models, + providers, +}: { + readonly metrics: Metrics; + readonly models: ReadonlyArray; + readonly providers: ReadonlyArray; +}) { + const [metric, setMetric] = useState("tokens"); + const [view, setView] = useState("model"); + const { weekly, columns, titles, ticks } = timeline(metrics); + const parts = view === "model" ? models : providers; + const named = parts.filter((part) => part.key !== OTHER_KEY); + const valueOf = (days: ReadonlyArray, key: string) => + sumOf(days, (day) => (view === "model" ? day.models : day.providers).get(key)?.[metric] ?? 0); + const stacks = parts.map((model) => ({ + label: model.label, + color: model.color, + ...(metric === "cost" && unpriced(model) ? { format: () => "Unpriced" } : {}), + values: columns.map((days) => + model.key === OTHER_KEY + ? Math.max( + 0, + sumOf(days, (day) => day[metric]) - named.reduce((sum, head) => sum + valueOf(days, head.key), 0), + ) + : valueOf(days, model.key), + ), + })); + const noun = metric === "tokens" ? "Tokens" : "Cost"; + return ( + + + +
+ } + > + + + + ); +} + +// --------------------------------------------------------------------------------------------- + +interface Column { + readonly label: string; + readonly cell: (part: Colored) => ReactNode; + /** Shown only when the card is wide enough. */ + readonly extra?: true; +} + +function Breakdown({ + title, + caption, + head, + noun, + parts, + rest, + by, + grid, + columns, + describe, + provider = false, + onPick, +}: { + readonly title: string; + readonly caption: string; + /** The first column's heading. */ + readonly head: string; + /** What a row is, plural: "Show All 12 Models". */ + readonly noun: string; + readonly parts: ReadonlyArray; + /** The rows folded into Other, listed under it on demand. */ + readonly rest: ReadonlyArray; + readonly by: "prompts" | "tokens"; + /** The row's grid template: the name, then one track per column. */ + readonly grid: string; + readonly columns: ReadonlyArray; + /** A row's tooltip, for what hidden columns would have said. */ + readonly describe?: (part: Colored) => string; + /** The hint is a provider, so it wears the provider's mark. */ + readonly provider?: boolean; + /** Makes each row, Other aside, a button. */ + readonly onPick?: (key: string) => void; +}) { + const [open, setOpen] = useState(false); + const row = `grid ${grid} items-center gap-x-3`; + const pressable = `${row} -mx-2 w-[calc(100%+16px)] rounded-md px-2 py-2 text-left outline-none hover:bg-ink/4 focus-visible:ring-2 focus-visible:ring-ring`; + const hide = (column: Column) => (column.extra === true ? "hidden @sm:block" : ""); + const body = (part: Colored, hint: string | undefined) => ( + <> + + + + {part.label} + {hint !== undefined && ( + + {provider && part.key !== OTHER_KEY && } + {hint} + + )} + + + {columns.map((column) => ( + + {column.cell(part)} + + ))} + + ); + const item = (part: Colored) => ( +
  • + {onPick === undefined ? ( +
    {body(part, part.hint)}
    + ) : ( + + )} +
  • + ); + return ( + + ({ label: part.label, value: part[by], color: part.color }))} + format={by === "tokens" ? formatTokens : formatCount} + label={`${title} by ${by}`} + /> +
    +
    + {head} + {columns.map((column) => ( + + {column.label} + + ))} +
    +
      + {parts.map((part) => + part.key !== OTHER_KEY ? ( + item(part) + ) : ( +
    • + + {open && ( +
        + {rest + .toSorted((a, b) => b[by] - a[by]) + .map((folded) => item({ ...folded, color: OTHER }))} +
      + )} +
    • + ), + )} +
    +
    +
    + ); +} + +const muted = (text: string) => {text}; +const secondary = (text: string) => {text}; +const strong = (text: string) => {text}; + +/** The project breakdown, each row scoping the page to its project. */ +function Projects({ + rows, + projectId, + projectName, + onProject, +}: { + readonly rows: ReadonlyArray; + readonly projectId: string | null; + readonly projectName: string | undefined; + readonly onProject: (projectId: string | null) => void; +}) { + if (projectId === null) { + const { parts, rest: folded } = fold(rows.map(partOf), palette(rows.map((row) => row.key)), rest); + return ( + secondary(formatCount(part.sessions)) }, + { label: "Tokens", cell: (part) => muted(formatTokens(part.tokens)), extra: true }, + { label: "Cost", cell: (part) => strong(formatCost(part.cost)) }, + ]} + onPick={onProject} + /> + ); + } + return ( + +
    + + Showing {projectName ?? rows[0]?.label ?? "Project"} + + +
    +
    + ); +} + +const perMillion = (part: Part) => + part.tokens === 0 || unpriced(part) ? "–" : formatCost((part.cost / part.tokens) * 1_000_000); + +function Models({ + parts, + rest, + edges, +}: { + readonly parts: ReadonlyArray; + readonly rest: ReadonlyArray; + readonly edges: ReadonlyArray; +}) { + return ( + secondary(formatTokens(part.tokens)) }, + { label: "Cached", cell: (part) => muted(hitLabel(part)), extra: true }, + { label: "$/M", cell: (part) => muted(perMillion(part)), extra: true }, + { + label: "Cost", + cell: (part) => (unpriced(part) ? muted("Unpriced") : strong(formatCost(part.cost))), + }, + ]} + describe={(part) => { + const response = responseOf(part, edges); + const speed = response?.speed === undefined ? "" : ` · ${formatSpeed(response.speed)}`; + const pace = response === undefined ? "" : ` · Median ${formatQuantile(response.median)}${speed}`; + return `${hitLabel(part)} Cached · ${perMillion(part)}/M${pace}`; + }} + provider + /> + ); +} + +// --------------------------------------------------------------------------------------------- + +const RESPONSE_ROW = "grid grid-cols-[minmax(0,1fr)_52px_52px_60px] items-center gap-x-3"; + +/** + * Median request duration over time for the page's coloured models, beside a table that doubles as + * the legend. Other gets no line: a median across unrelated models means nothing. + */ +function ResponseTime({ + metrics, + models, + more, +}: { + readonly metrics: Metrics; + readonly models: ReadonlyArray; + /** Models folded into Other. */ + readonly more: number; +}) { + const { columns, titles, ticks } = timeline(metrics); + const named = models.filter((model) => model.key !== OTHER_KEY); + const overall = responseOf(metrics.totals, metrics.edges); + return ( + } + action={ + overall !== undefined && ( + + Median {formatQuantile(overall.median)} · P95{" "} + {formatQuantile(overall.p95)} + + ) + } + > + {overall === undefined ? ( +

    No Requests

    + ) : ( +
    + ({ + label: model.label, + color: model.color, + values: columns.map((days) => { + const spent = days.flatMap((day) => day.models.get(model.key) ?? []); + // No timed request is a gap, never a zero. + return spent.some((spend) => spend.timed > 0) + ? quantile( + spent.reduce>((all, spend) => plus(all, spend.histogram), []), + metrics.edges, + 0.5, + )?.ms + : undefined; + }), + }))} + titles={titles} + ticks={ticks} + format={formatDuration} + label="Median response time by model" + height={168} + /> +
    +
    + Model + Median + P95 + Speed +
    +
      + {named.map((model) => { + const response = responseOf(model, metrics.edges); + return ( +
    • + + + {response === undefined ? "–" : formatQuantile(response.median)} + + + {response === undefined ? "–" : formatQuantile(response.p95)} + + + {response?.speed === undefined ? "–" : formatSpeed(response.speed)} + +
    • + ); + })} + {more > 0 && ( +
    • + {formatCount(more)} More {more === 1 ? "Model" : "Models"} +
    • + )} +
    +
    +
    + )} +
    + ); +} + +/** What the response figures are estimated from, on click or Enter. */ +function Estimated() { + return ( + + + + + + Estimated From Duration Buckets +

    + Median and P95 are accurate to one duration bucket. Interrupted requests are not timed. +

    +
    +
    + ); +} + +// --------------------------------------------------------------------------------------------- + +/** A swatch and a name: the first cell of a legend row. */ +function Swatch({ color, label }: { readonly color: string; readonly label: string }) { + return ( + + + {label} + + ); +} + +const TOOLS_SHOWN = 12; +const TOOL_ROW = "grid grid-cols-[minmax(0,1fr)_64px_48px_36px] items-center gap-x-3"; + +/** Calls per tool, ranked down each column so the list fills the row. */ +function Tools({ metrics }: { readonly metrics: Metrics }) { + const { tools } = metrics; + const callsOf = (list: ReadonlyArray<{ readonly calls: number }>) => list.reduce((all, tool) => all + tool.calls, 0); + const sum = callsOf(tools); + const shown = tools.slice(0, TOOLS_SHOWN); + const hidden = tools.slice(TOOLS_SHOWN); + const top = shown[0]?.calls ?? 0; + const share = (calls: number) => (sum === 0 ? "–" : formatPercent(calls / sum)); + const item = `${TOOL_ROW} break-inside-avoid border-b border-edge/50 py-1.5`; + return ( + + {tools.length === 0 ? ( +

    No Tool Calls

    + ) : ( +
      + {shown.map((tool) => ( +
    1. + + {tool.name} + + + {formatCount(tool.calls)} + {share(tool.calls)} +
    2. + ))} + {hidden.length > 0 && ( +
    3. + + Other + {formatCount(hidden.length)} more + + + {formatCount(callsOf(hidden))} + {share(callsOf(hidden))} +
    4. + )} +
    + )} +
    + ); +} + +// --------------------------------------------------------------------------------------------- + +const SESSION_ORDER: ReadonlyArray<{ readonly key: "cost" | "tokens"; readonly label: string }> = [ + { key: "cost", label: "Cost" }, + { key: "tokens", label: "Tokens" }, +]; + +function TopSessions({ metrics }: { readonly metrics: Metrics }) { + const [by, setBy] = useState<"cost" | "tokens">("cost"); + const top = listed(metrics.sessions, by).slice(0, 8); + return ( + } + > +
      + {top.map((line) => ( + + ))} +
    +
    + ); +} + +/** Requests by the reasoning effort asked for, on its ordered off → max scale. */ +function Thinking({ metrics }: { readonly metrics: Metrics }) { + const rows = metrics + .slice("thinking") + .filter((row) => row.steps > 0) + .map((row) => ({ ...row, color: RAMP[levelIndex(row.key)] ?? OTHER })); + const { steps, reasoning, output } = metrics.totals; + return ( + + Reasoning {formatTokens(reasoning)} + {output > 0 && ` · ${formatPercent(reasoning / output)} of Output`} +
    + } + > + ({ label: row.label, value: row.steps, color: row.color }))} + format={formatCount} + label="Requests by thinking level" + /> +
      + {rows.map((row) => ( +
    • + + + + {formatCount(row.steps)} + + {steps === 0 ? "–" : formatPercent(row.steps / steps)} · {formatTokens(row.reasoning)} + +
    • + ))} +
    + + ); +} + +/** Token types as ink tints, so they never read as one of the page's models. */ +const TYPES = [ + { key: "input", label: "Input", color: ink(60), tokens: "input", cost: "costInput" }, + { key: "cacheRead", label: "Cache Read", color: ink(30), tokens: "cacheRead", cost: "costCacheRead" }, + { key: "cacheWrite", label: "Cache Write", color: ink(72), tokens: "cacheWrite", cost: "costCacheWrite" }, + { key: "output", label: "Output", color: ink(100), tokens: "output", cost: "costOutput" }, +] as const; + +const CACHE_ROW = "grid grid-cols-[minmax(0,1fr)_56px_64px] items-center gap-x-3"; + +/** Hit rate per bucket; a bucket with no input repeats the last known rate so the line has no holes. */ +function hitTrend(metrics: Metrics) { + const rates = buckets(metrics).map((days) => + cacheHit({ + input: sumOf(days, (day) => day.input), + cacheRead: sumOf(days, (day) => day.cacheRead), + cacheWrite: sumOf(days, (day) => day.cacheWrite), + }), + ); + let last = rates.find((rate) => rate !== null) ?? 0; + return rates.map((rate) => { + if (rate !== null) last = rate; + return last; + }); +} + +function Cache({ metrics }: { readonly metrics: Metrics }) { + const { totals, previous } = metrics; + const hit = cacheHit(totals); + const before = previous === undefined ? undefined : (cacheHit(previous) ?? undefined); + const period = RANGES.find(({ key }) => key === metrics.range)?.label.toLowerCase() ?? ""; + return ( + +
    +
    + Cache Hit Rate +
    + + {hit === null ? "–" : formatPercent(hit)} + + +
    + +
    +
    Saved by Cache
    +
    {formatCost(totals.cacheSavings)}
    +
    +
    +
    + Tokens by Type + ({ label: type.label, value: totals[type.tokens], color: type.color }))} + format={formatTokens} + label="Tokens by type" + /> + Cost by Type + ({ label: type.label, value: totals[type.cost], color: type.color }))} + format={formatCost} + label="Cost by type" + /> +
    +
    +
    + Type + Tokens + Cost +
    +
      + {TYPES.map((type) => ( +
    • + + + {formatTokens(totals[type.tokens])} + + {formatCost(totals[type.cost])} +
    • + ))} +
    +
    +
    +
    + ); +} diff --git a/apps/web/src/metrics/page.tsx b/apps/web/src/metrics/page.tsx new file mode 100644 index 00000000..28492d00 --- /dev/null +++ b/apps/web/src/metrics/page.tsx @@ -0,0 +1,174 @@ +import { MetricsError, type MetricsUsage } from "@codeworksh/contract"; +import { useAtomRefresh, useAtomValue } from "@effect/atom-react"; +import { ChartBarIcon, WarningCircleIcon } from "@phosphor-icons/react"; +import { Option } from "effect"; +import { AsyncResult } from "effect/reactivity"; +import { useMemo, useState, useSyncExternalStore } from "react"; + +import { Skeleton } from "@/components/ui/skeleton"; +import { minuteTick } from "../kernel/clock"; +import { Page } from "../shell/page"; +import { Segmented } from "./charts"; +import { + localDay, + projectOptions, + query, + RANGES, + type RangeKey, + type Scope, + usageAtom, + usageKey, + yearOf, +} from "./data"; +import { MachineSelect, ProjectSelect } from "./filters"; +import { MetricsBody } from "./metrics"; + +const EVERYTHING: Scope = { machineId: null, projectId: null }; + +/** + * The year of usage for `scope`, read again on every change to it. A new scope or day keeps the last + * year on screen until its own arrives; each year is indexed against the day it was read for. + */ +function useYear(scope: Scope, today: string) { + const atom = usageAtom(usageKey(scope, today)); + const result = useAtomValue(atom); + const retry = useAtomRefresh(atom); + const current = Option.getOrUndefined(AsyncResult.value(result)); + const [kept, setKept] = useState<{ readonly usage: MetricsUsage; readonly today: string }>(); + if (current !== undefined && current !== kept?.usage) setKept({ usage: current, today }); + const usage = current ?? kept?.usage; + const day = current === undefined ? kept?.today : today; + const year = useMemo( + () => (usage === undefined || day === undefined ? undefined : yearOf(usage, day)), + [usage, day], + ); + const error = Option.getOrUndefined(AsyncResult.error(result)); + const failure = !AsyncResult.isFailure(result) + ? undefined + : error instanceof MetricsError + ? error.message + : "The App Server Is Unreachable"; + return { year, failure, retry }; +} + +/** Usage across machines and projects; the header's filters and period scope everything below. */ +export function MetricsPage() { + const [range, setRange] = useState("30d"); + const [machineId, setMachineId] = useState(null); + const [projectId, setProjectId] = useState(null); + const today = localDay(new Date(useSyncExternalStore(minuteTick.subscribe, minuteTick.getSnapshot))); + const scope = useMemo(() => ({ machineId, projectId }), [machineId, projectId]); + const { year, failure, retry } = useYear(scope, today); + // Filtered reads name only what they cover, so the filters list from the unfiltered year: the same read when nothing is filtered. + const everything = useYear(EVERYTHING, today).year ?? year; + const metrics = useMemo(() => (year === undefined ? undefined : query(year, range)), [year, range]); + const whole = useMemo(() => (year === undefined ? undefined : query(year, "1y")), [year]); + const projects = useMemo( + () => (everything === undefined ? [] : projectOptions(everything, range, machineId)), + [everything, range, machineId], + ); + const machines = everything?.machines ?? []; + + return ( + + {machines.length > 1 && } + + + + } + > +
    + {failure !== undefined && } + {metrics !== undefined && whole !== undefined ? ( + row.project.id === projectId)?.project.name} + onProject={setProjectId} + /> + ) : ( + failure === undefined && + )} +
    +
    + ); +} + +/** What failed and a way to try again: in place of the page with nothing to show, or above what is shown. */ +function Failed({ + detail, + onRetry, + alone, +}: { + readonly detail: string; + readonly onRetry: () => void; + /** Nothing is shown below it. */ + readonly alone: boolean; +}) { + return ( +
    + + + + Could Not Load Metrics + + + {detail} + + + +
    + ); +} + +/** The first cards' shapes at their sizes, until the first year arrives. */ +function Loading() { + const card = "flex flex-col gap-4 rounded-lg border border-edge p-4"; + return ( +
    +
    + {[0, 1, 2].map((tile) => ( +
    + + + +
    + ))} +
    + {[330, 290].map((height) => ( +
    + + +
    + ))} +
    + {[0, 1].map((half) => ( +
    + + + +
    + ))} +
    +
    + ); +} diff --git a/apps/web/src/metrics/session.tsx b/apps/web/src/metrics/session.tsx new file mode 100644 index 00000000..c115c9da --- /dev/null +++ b/apps/web/src/metrics/session.tsx @@ -0,0 +1,56 @@ +import { Monogram } from "@/components/monogram"; +import { Meter } from "./charts"; +import { formatCost, formatCount, formatTokens, type Listed } from "./data"; + +/** + * One session on one line (two when narrow): title, then project and threads, then a meter against + * the list's top line by `by`, which also takes the ink. Deleted sessions are one muted line. + */ +export function SessionLine({ + line, + by, + max, +}: { + readonly line: Listed; + readonly by: "tokens" | "cost"; + /** The list's top value of `by`, for the meter. */ + readonly max: number; +}) { + const tone = (key: "tokens" | "cost") => (key === by ? "text-ink" : "text-ink-secondary"); + const title = + line.kind === "deleted" + ? `${formatCount(line.count)} Deleted ${line.count === 1 ? "Session" : "Sessions"}` + : line.session.title; + return ( +
  • + + {title} + + {line.kind === "session" && ( + + + + {line.session.project.name} + + {line.threads !== undefined && ( + + {formatCount(line.threads)} {line.threads === 1 ? "Thread" : "Threads"} + + )} + + )} + + + + + + {formatTokens(line.tokens)} + + {formatCost(line.cost)} + +
  • + ); +} diff --git a/apps/web/src/routeTree.gen.ts b/apps/web/src/routeTree.gen.ts index bde17270..3635dfd8 100644 --- a/apps/web/src/routeTree.gen.ts +++ b/apps/web/src/routeTree.gen.ts @@ -11,6 +11,7 @@ import { Route as rootRouteImport } from './routes/__root' import { Route as IndexRouteImport } from './routes/index' import { Route as ShellRouteImport } from './routes/_shell' +import { Route as ShellMetricsRouteImport } from './routes/_shell.metrics' import { Route as ShellProjectsRouteImport } from './routes/_shell.projects' import { Route as ShellSettingsRouteImport } from './routes/_shell.settings' import { Route as ShellThreadsRouteImport } from './routes/_shell.threads' @@ -38,6 +39,11 @@ const ShellRoute = ShellRouteImport.update({ id: '/_shell', getParentRoute: () => rootRouteImport, } as any) +const ShellMetricsRoute = ShellMetricsRouteImport.update({ + id: '/metrics', + path: '/metrics', + getParentRoute: () => ShellRoute, +} as any) const ShellProjectsRoute = ShellProjectsRouteImport.update({ id: '/projects', path: '/projects', @@ -128,6 +134,7 @@ const ShellWorkspacesWorkspaceIdRoute = export interface FileRoutesByFullPath { '/': typeof IndexRoute + '/metrics': typeof ShellMetricsRoute '/projects': typeof ShellProjectsRoute '/settings': typeof ShellSettingsRouteWithChildren '/threads': typeof ShellThreadsRoute @@ -148,6 +155,7 @@ export interface FileRoutesByFullPath { } export interface FileRoutesByTo { '/': typeof IndexRoute + '/metrics': typeof ShellMetricsRoute '/projects': typeof ShellProjectsRoute '/threads': typeof ShellThreadsRoute '/settings/archive': typeof ShellSettingsArchiveRoute @@ -169,6 +177,7 @@ export interface FileRoutesById { __root__: typeof rootRouteImport '/': typeof IndexRoute '/_shell': typeof ShellRouteWithChildren + '/_shell/metrics': typeof ShellMetricsRoute '/_shell/projects': typeof ShellProjectsRoute '/_shell/settings': typeof ShellSettingsRouteWithChildren '/_shell/threads': typeof ShellThreadsRoute @@ -191,6 +200,7 @@ export interface FileRouteTypes { fileRoutesByFullPath: FileRoutesByFullPath fullPaths: | '/' + | '/metrics' | '/projects' | '/settings' | '/threads' @@ -211,6 +221,7 @@ export interface FileRouteTypes { fileRoutesByTo: FileRoutesByTo to: | '/' + | '/metrics' | '/projects' | '/threads' | '/settings/archive' @@ -231,6 +242,7 @@ export interface FileRouteTypes { | '__root__' | '/' | '/_shell' + | '/_shell/metrics' | '/_shell/projects' | '/_shell/settings' | '/_shell/threads' @@ -271,6 +283,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof ShellRouteImport parentRoute: typeof rootRouteImport } + '/_shell/metrics': { + id: '/_shell/metrics' + path: '/metrics' + fullPath: '/metrics' + preLoaderRoute: typeof ShellMetricsRouteImport + parentRoute: typeof ShellRoute + } '/_shell/projects': { id: '/_shell/projects' path: '/projects' @@ -430,6 +449,7 @@ const ShellSettingsRouteWithChildren = ShellSettingsRoute._addFileChildren( ) interface ShellRouteChildren { + ShellMetricsRoute: typeof ShellMetricsRoute ShellProjectsRoute: typeof ShellProjectsRoute ShellSettingsRoute: typeof ShellSettingsRouteWithChildren ShellThreadsRoute: typeof ShellThreadsRoute @@ -437,6 +457,7 @@ interface ShellRouteChildren { } const ShellRouteChildren: ShellRouteChildren = { + ShellMetricsRoute: ShellMetricsRoute, ShellProjectsRoute: ShellProjectsRoute, ShellSettingsRoute: ShellSettingsRouteWithChildren, ShellThreadsRoute: ShellThreadsRoute, diff --git a/apps/web/src/routes/_shell.metrics.tsx b/apps/web/src/routes/_shell.metrics.tsx new file mode 100644 index 00000000..59e2e9b4 --- /dev/null +++ b/apps/web/src/routes/_shell.metrics.tsx @@ -0,0 +1,5 @@ +import { createFileRoute } from "@tanstack/react-router"; + +import { MetricsPage } from "../metrics/page"; + +export const Route = createFileRoute("/_shell/metrics")({ component: MetricsPage }); diff --git a/apps/web/src/settings/blocks.tsx b/apps/web/src/settings/blocks.tsx index 31030ac2..e553d233 100644 --- a/apps/web/src/settings/blocks.tsx +++ b/apps/web/src/settings/blocks.tsx @@ -84,7 +84,7 @@ export function SavingInput({ }: { value: string; label: string; - placeholder?: string; + placeholder?: string | undefined; autoFocus?: boolean; className?: string; onCommit: (name: string) => void; diff --git a/apps/web/src/settings/servers/add.tsx b/apps/web/src/settings/servers/add.tsx index ef7ff9a0..b5c756b1 100644 --- a/apps/web/src/settings/servers/add.tsx +++ b/apps/web/src/settings/servers/add.tsx @@ -28,19 +28,20 @@ const STEPS = ["Kind", "Identity", "Config"] as const; */ export function AddServer({ onClose, - defaultName, + placeholder, detected, onAdded, }: { onClose: () => void; - defaultName: string; + /** The name the server is shown by when it is given none. */ + placeholder: string | undefined; /** The `codework` found without a choice, shown as the executable's default. */ detected: string | undefined; onAdded: (id: string) => void; }) { const add = useAtomSet(addAtom, { mode: "promise" }); const [step, setStep] = useState(0); - const [name, setName] = useState(defaultName); + const [name, setName] = useState(""); const [executable, setExecutable] = useState(""); const [home, setHome] = useState(""); // A free port, so the new server runs beside the others rather than on the default one. @@ -62,7 +63,7 @@ export function AddServer({ .finally(() => setAdding(false)); }; - const canContinue = step === 0 || (step === 1 && name.trim() !== "") || (step === 2 && home.trim() !== ""); + const canContinue = step !== 2 || home.trim() !== ""; return ( !next && onClose()}> @@ -110,6 +111,7 @@ export function AddServer({ setName(event.target.value)} diff --git a/apps/web/src/settings/servers/local.tsx b/apps/web/src/settings/servers/local.tsx index b34d27dc..890674df 100644 --- a/apps/web/src/settings/servers/local.tsx +++ b/apps/web/src/settings/servers/local.tsx @@ -46,6 +46,10 @@ export const Badge = ({ tone, children }: { tone: Tone; children: ReactNode }) = export const Mark = () => ; +/** What a server is called without the user's override: as shown while there is none. */ +export const reportedName = (status: LocalServerStatus) => + status.override === null ? status.name : (status.host ?? undefined); + const Version = ({ version }: { version: string }) => ( v{version} ); @@ -87,6 +91,9 @@ export function LocalItem({ status }: { status: LocalServerStatus }) {
    {status.name} + {status.host !== null && status.host !== status.name && ( + {status.host} + )} {cli !== null && "version" in cli && } {problem !== undefined && ( @@ -172,11 +179,10 @@ export function LocalDetail({ status }: { status: LocalServerStatus }) { { - // The server keeps its name when the field is cleared. - if (!next) return; setNameError(null); void configure({ name: next }).then((failure) => setNameError(failure ?? null)); }} diff --git a/apps/web/src/settings/servers/servers.tsx b/apps/web/src/settings/servers/servers.tsx index abc55460..d5d88dbb 100644 --- a/apps/web/src/settings/servers/servers.tsx +++ b/apps/web/src/settings/servers/servers.tsx @@ -1,6 +1,6 @@ import { ArrowClockwiseIcon, ArrowCounterClockwiseIcon, MinusIcon, PlusIcon } from "@phosphor-icons/react"; import { useAtom, useAtomSet, useAtomValue } from "@effect/atom-react"; -import { DEFAULT_SERVER_HEALTH, type LocalServerStatus } from "@codeworksh/contract"; +import { DEFAULT_SERVER_HEALTH, LOCAL_SERVER, type LocalServerStatus } from "@codeworksh/contract"; import { AsyncResult } from "effect/reactivity"; import { useEffect, useState } from "react"; @@ -12,7 +12,7 @@ import { Server } from "../../kernel/rpc"; import { serversAtom } from "../../kernel/servers"; import { Group, Page, quietField, Row, SavingInput, Section } from "../blocks"; import { AddServer } from "./add"; -import { LocalDetail, LocalItem } from "./local"; +import { LocalDetail, LocalItem, reportedName } from "./local"; const checkAtom = Server.mutation("servers.check"); const healthKey = ["health"]; @@ -35,6 +35,8 @@ export function ServersSettings() { const [adding, setAdding] = useState(false); // The default until one is picked, and again when the picked one is gone. const selected = servers?.find((server) => server.id === selectedId) ?? servers?.find((server) => server.isDefault); + // A new local server runs on the same machine, so it reports the built-in one's name. + const builtin = servers?.find((server) => server.id === LOCAL_SERVER); // Shown as the new server's executable default: the binary found without a choice. const detected = servers?.flatMap((server) => server.cli !== null && !server.cli.custom ? [server.cli.path] : [], @@ -85,7 +87,7 @@ export function ServersSettings() { {adding && ( setAdding(false)} - defaultName={`CodeWork ${(servers?.length ?? 0) + 1}`} + placeholder={builtin && reportedName(builtin)} detected={detected} onAdded={setSelectedId} /> diff --git a/apps/web/src/shell/page.tsx b/apps/web/src/shell/page.tsx index b71a8941..a54a1211 100644 --- a/apps/web/src/shell/page.tsx +++ b/apps/web/src/shell/page.tsx @@ -15,11 +15,14 @@ export function Page({ instanceId, title, icon: Icon, + actions, children, }: { readonly instanceId: string; readonly title: string; readonly icon: Icon; + /** Controls at the header's end. */ + readonly actions?: ReactNode; readonly children: ReactNode; }) { const registry = useContext(RegistryContext); @@ -42,9 +45,10 @@ export function Page({
    -
    +
    -

    {title}

    +

    {title}

    + {actions}
    {children}
    diff --git a/apps/web/src/shell/shell.tsx b/apps/web/src/shell/shell.tsx index 45232d28..44ac2a76 100644 --- a/apps/web/src/shell/shell.tsx +++ b/apps/web/src/shell/shell.tsx @@ -11,7 +11,7 @@ import { settingsShortcut, workspaceShortcut } from "./switcher"; /** * Lives in the pathless `_shell` layout so switching workspaces (a URL change) never remounts * the desktop: widget order, pins, and per-workspace layout survive navigation. - * Settings, All Threads and Projects open over the desktop, which stays mounted underneath. + * Settings, All Threads, Projects and Metrics open over the desktop, which stays mounted underneath. */ export function Shell() { const { workspaceId } = useParams({ strict: false }); @@ -24,7 +24,8 @@ export function Shell() { select: (location) => location.pathname.startsWith("/settings") || location.pathname === "/threads" || - location.pathname === "/projects", + location.pathname === "/projects" || + location.pathname === "/metrics", }); // An id that no longer exists (a deleted workspace) shows the first one. const shown = workspaces.find((workspace) => workspace.id === workspaceId) ?? workspaces[0]; From 9ebc4702cebfa33f5350de17d2462b7bb6e49fbe Mon Sep 17 00:00:00 2001 From: Sanchit Rk <0xsanchit@gmail.com> Date: Sat, 14 Nov 2026 16:24:25 +0530 Subject: [PATCH 5/8] fix(web): keep metrics charts still under the pointer and the view across navigation Tooltips live in their own store, so following the pointer re-renders only the tooltip; the calendar is sized before its first paint and keeps its scroll on a refresh. The year's read outlives the page for a minute and the period and filters persist across navigation, a removed machine's filter falling back to every machine. Change badges carry direction in the arrow, the cache badge shows points, and a model id shared by two providers is labelled provider first. Co-Authored-By: Claude Opus 5.5 --- apps/web/src/metrics/activity.tsx | 6 ++- apps/web/src/metrics/charts.tsx | 77 ++++++++++++++++++++++--------- apps/web/src/metrics/data.ts | 37 +++++++++------ apps/web/src/metrics/metrics.tsx | 22 +++++++-- apps/web/src/metrics/page.tsx | 21 ++++++--- 5 files changed, 115 insertions(+), 48 deletions(-) diff --git a/apps/web/src/metrics/activity.tsx b/apps/web/src/metrics/activity.tsx index bce30e50..ef2a7721 100644 --- a/apps/web/src/metrics/activity.tsx +++ b/apps/web/src/metrics/activity.tsx @@ -140,15 +140,17 @@ function Calendar({ const room = box.clientWidth - (labels.current?.offsetWidth ?? 0) - LABEL_GAP; setCell(Math.min(MAX_CELL, Math.max(MIN_CELL, Math.floor((room - (columns - 1) * GAP) / columns)))); }; + // Fitted before the first paint, so the calendar never paints at the minimum and then grows. + fit(); const observer = new ResizeObserver(fit); observer.observe(box); return () => observer.disconnect(); }, [columns]); - // Open on the latest weeks when the calendar is wider than the card. + // Open on the latest weeks when the calendar is wider than the card; a refresh of the data keeps the scroll. useLayoutEffect(() => { const box = scroller.current; if (box !== null) box.scrollLeft = box.scrollWidth; - }, [days, cell]); + }, [cell]); const width = columns * cell + (columns - 1) * GAP; const months = useMemo(() => { diff --git a/apps/web/src/metrics/charts.tsx b/apps/web/src/metrics/charts.tsx index 0eb3bc39..6e84f0aa 100644 --- a/apps/web/src/metrics/charts.tsx +++ b/apps/web/src/metrics/charts.tsx @@ -1,4 +1,13 @@ -import { type MouseEvent, type ReactNode, type Ref, useCallback, useLayoutEffect, useRef, useState } from "react"; +import { + type MouseEvent, + type ReactNode, + type Ref, + useCallback, + useLayoutEffect, + useRef, + useState, + useSyncExternalStore, +} from "react"; /* Chart pieces and the card they sit in. Marks carry color; text stays in ink tokens. */ @@ -153,29 +162,55 @@ interface Tip { readonly content: ReactNode; } +interface TipStore { + readonly get: () => Tip | null; + readonly set: (tip: Tip | null) => void; + readonly subscribe: (listener: () => void) => () => void; +} + +const tipStore = (): TipStore => { + let tip: Tip | null = null; + const listeners = new Set<() => void>(); + return { + get: () => tip, + set: (next) => { + if (next === tip) return; + tip = next; + for (const listener of listeners) listener(); + }, + subscribe: (listener) => { + listeners.add(listener); + return () => listeners.delete(listener); + }, + }; +}; + +/** The tooltip lives in its own store, so following the pointer re-renders only the tooltip, never the chart. */ function useTip() { - const [tip, setTip] = useState(null); + const [store] = useState(tipStore); const show = useCallback( - (event: MouseEvent, content: ReactNode) => setTip({ x: event.clientX, y: event.clientY, content }), - [], + (event: MouseEvent, content: ReactNode) => store.set({ x: event.clientX, y: event.clientY, content }), + [store], + ); + const hide = useCallback(() => store.set(null), [store]); + return { show, hide, element: }; +} + +function TipLayer({ store }: { readonly store: TipStore }) { + const tip = useSyncExternalStore(store.subscribe, store.get); + if (tip === null) return null; + const flip = tip.x > window.innerWidth - 260; + return ( +
    + {tip.content} +
    ); - const hide = useCallback(() => setTip(null), []); - const flip = tip !== null && tip.x > window.innerWidth - 260; - const element = - tip === null ? null : ( -
    - {tip.content} -
    - ); - return { show, hide, element }; } /** A tooltip's body: a title, then one row per value with its swatch. */ diff --git a/apps/web/src/metrics/data.ts b/apps/web/src/metrics/data.ts index 301076d1..ed7ebb21 100644 --- a/apps/web/src/metrics/data.ts +++ b/apps/web/src/metrics/data.ts @@ -57,17 +57,22 @@ interface Request { readonly projectId?: string; } -/** `read` when subscribed, and again after each change to usage; a failure ends it until refreshed. */ +/** + * `read` when subscribed, and again after each change to usage; a failure ends it until refreshed. + * It outlives the page for a minute, so leaving and coming back shows the last read at once. + */ const follow = (read: Effect.Effect) => - Server.runtime.atom( - Stream.unwrap( - Server.use((client) => Effect.succeed(client("event.subscribe", { topics: readTopics.metrics }))), - ).pipe( - // `Synced` first, once subscribed: the first read can miss no change. - Stream.debounce("50 millis"), - Stream.mapEffect(() => read), - ), - ); + Server.runtime + .atom( + Stream.unwrap( + Server.use((client) => Effect.succeed(client("event.subscribe", { topics: readTopics.metrics }))), + ).pipe( + // `Synced` first, once subscribed: the first read can miss no change. + Stream.debounce("50 millis"), + Stream.mapEffect(() => read), + ), + ) + .pipe(Atom.setIdleTTL("1 minute")); export const usageAtom = Atom.family((key: string) => { const request = JSON.parse(key) as Request & { readonly since: string; readonly until: string }; @@ -653,12 +658,15 @@ export function responseOf( }; } -/** Names a model by its id, adding its provider when the id occurs under more than one provider in `rows`. */ +/** + * Names a model by its id, led by its provider when the id occurs under more than one provider in + * `rows`: the provider is what tells those apart, so a truncated label must keep it. + */ export function modelNamer(rows: ReadonlyArray<{ readonly provider: string; readonly model: string }>) { const providers = new Map>(); for (const row of rows) providers.set(row.model, (providers.get(row.model) ?? new Set()).add(row.provider)); return (row: { readonly provider: string; readonly model: string }) => - (providers.get(row.model)?.size ?? 0) > 1 ? `${row.model} · ${row.provider}` : row.model; + (providers.get(row.model)?.size ?? 0) > 1 ? `${row.provider} · ${row.model}` : row.model; } /** A free or preview model reports tokens but no cost; "$0" would read as a bargain. */ @@ -723,8 +731,11 @@ export function change(now: number, before: number | undefined) { if (before === undefined || before === 0) return undefined; return (now - before) / before; } +/** Unsigned: the badge's arrow carries the direction. */ export const formatChange = (ratio: number) => - `${ratio >= 0 ? "+" : "−"}${Math.abs(ratio) >= 10 ? "999+" : Math.round(Math.abs(ratio) * 100)}%`; + Math.abs(ratio) >= 10 ? ">999%" : `${Math.round(Math.abs(ratio) * 100)}%`; +/** A change between two rates, unsigned: "10 pts". */ +export const formatPoints = (difference: number) => `${Math.round(Math.abs(difference) * 100)} pts`; /** * Share of input served from cache, or null with no input. A cache write counts as a miss: that * input was processed in full. diff --git a/apps/web/src/metrics/metrics.tsx b/apps/web/src/metrics/metrics.tsx index dc5cc617..a8f2d7dc 100644 --- a/apps/web/src/metrics/metrics.tsx +++ b/apps/web/src/metrics/metrics.tsx @@ -30,6 +30,7 @@ import { formatDay, formatDuration, formatPercent, + formatPoints, formatQuantile, formatSpeed, formatTokens, @@ -86,8 +87,7 @@ export function MetricsBody({ }); }, [models]); const period = RANGES.find(({ key }) => key === metrics.range)?.label.toLowerCase() ?? ""; - if (metrics.totals.steps === 0 && year.totals.steps === 0) - return ; + if (year.totals.steps === 0) return ; return (
    @@ -148,11 +148,19 @@ const hitLabel = (totals: Pick) => }; /** A change badge in neutral ink, or nothing when there is nothing to compare. */ -function Delta({ ratio, period }: { readonly ratio: number | undefined; readonly period: string }) { +function Delta({ + ratio, + period, + format = formatChange, +}: { + readonly ratio: number | undefined; + readonly period: string; + readonly format?: (ratio: number) => string; +}) { if (ratio === undefined) return null; return ( - {ratio >= 0 ? "↑" : "↓"} {formatChange(ratio)} + {ratio >= 0 ? "↑" : "↓"} {format(ratio)} ); } @@ -863,7 +871,11 @@ function Cache({ metrics }: { readonly metrics: Metrics }) { {hit === null ? "–" : formatPercent(hit)} - +
    diff --git a/apps/web/src/metrics/page.tsx b/apps/web/src/metrics/page.tsx index 28492d00..17923885 100644 --- a/apps/web/src/metrics/page.tsx +++ b/apps/web/src/metrics/page.tsx @@ -1,8 +1,8 @@ import { MetricsError, type MetricsUsage } from "@codeworksh/contract"; -import { useAtomRefresh, useAtomValue } from "@effect/atom-react"; +import { useAtom, useAtomRefresh, useAtomValue } from "@effect/atom-react"; import { ChartBarIcon, WarningCircleIcon } from "@phosphor-icons/react"; import { Option } from "effect"; -import { AsyncResult } from "effect/reactivity"; +import { Atom, AsyncResult } from "effect/reactivity"; import { useMemo, useState, useSyncExternalStore } from "react"; import { Skeleton } from "@/components/ui/skeleton"; @@ -25,6 +25,9 @@ import { MetricsBody } from "./metrics"; const EVERYTHING: Scope = { machineId: null, projectId: null }; +/** The period and filters, kept across navigation so coming back shows the same view. */ +const viewAtom = Atom.make({ range: "30d", ...EVERYTHING }).pipe(Atom.keepAlive); + /** * The year of usage for `scope`, read again on every change to it. A new scope or day keeps the last * year on screen until its own arrives; each year is indexed against the day it was read for. @@ -53,14 +56,18 @@ function useYear(scope: Scope, today: string) { /** Usage across machines and projects; the header's filters and period scope everything below. */ export function MetricsPage() { - const [range, setRange] = useState("30d"); - const [machineId, setMachineId] = useState(null); - const [projectId, setProjectId] = useState(null); + const [{ range, machineId: picked, projectId }, setView] = useAtom(viewAtom); + const setRange = (next: RangeKey) => setView((view) => ({ ...view, range: next })); + const setMachineId = (next: string | null) => setView((view) => ({ ...view, machineId: next })); + const setProjectId = (next: string | null) => setView((view) => ({ ...view, projectId: next })); const today = localDay(new Date(useSyncExternalStore(minuteTick.subscribe, minuteTick.getSnapshot))); + // Filtered reads name only what they cover, so the filters list from the unfiltered year: the same read when nothing is filtered. + const unfiltered = useYear(EVERYTHING, today).year; + // A machine removed since it was picked is no longer listed: every machine shows instead. + const machineId = unfiltered?.machines.some((machine) => machine.id === picked) === false ? null : picked; const scope = useMemo(() => ({ machineId, projectId }), [machineId, projectId]); const { year, failure, retry } = useYear(scope, today); - // Filtered reads name only what they cover, so the filters list from the unfiltered year: the same read when nothing is filtered. - const everything = useYear(EVERYTHING, today).year ?? year; + const everything = unfiltered ?? year; const metrics = useMemo(() => (year === undefined ? undefined : query(year, range)), [year, range]); const whole = useMemo(() => (year === undefined ? undefined : query(year, "1y")), [year]); const projects = useMemo( From 67e48f3870a4d92d650d55c3d391d74087005b5e Mon Sep 17 00:00:00 2001 From: Sanchit Rk <0xsanchit@gmail.com> Date: Sat, 14 Nov 2026 16:24:25 +0530 Subject: [PATCH 6/8] feat: detect a restored store by its boot history and keep a removed server's usage Each open of the harness database records a boot with the revision it opened at. usage.sync takes the client's last boot and, when the store left that line (a restored backup), reads from zero with reset; a read from zero replaces the store's mirrored rows in one transaction. The app server keys usage by store with no foreign key to machine, so a removed server's history stays and the server added again replaces it rather than adding to it. Metrics also refresh on index changes, for session titles and deletions. Co-Authored-By: Claude Opus 5.5 --- apps/contract/src/contract.ts | 6 +- apps/server/src/db.ts | 19 +- apps/server/src/mirror.ts | 4 +- apps/server/src/servers.ts | 2 +- apps/server/src/sync.ts | 35 +-- apps/server/test/__artifacts__/schema.json | 12 +- .../test/__artifacts__/usage.passed.json | 207 ++++++++++++++++++ .../test/__artifacts__/usage.removed.json | 114 ++++++++++ apps/server/test/schema.test.ts | 6 +- apps/server/test/usage.test.ts | 97 ++++++++ apps/widget/src/codework.ts | 17 +- packages/codework/src/server/contract.ts | 18 +- packages/codework/src/server/handlers.ts | 37 +++- .../test/__artifacts__/sync.usage.json | 12 +- packages/codework/test/usage.test.ts | 3 +- packages/harness/src/db/db.ts | 2 + packages/harness/src/db/migrations.ts | 13 ++ packages/harness/src/db/store.ts | 24 +- packages/harness/test/db/db.test.ts | 2 + 19 files changed, 569 insertions(+), 61 deletions(-) create mode 100644 apps/server/test/__artifacts__/usage.passed.json create mode 100644 apps/server/test/__artifacts__/usage.removed.json diff --git a/apps/contract/src/contract.ts b/apps/contract/src/contract.ts index b8faf95e..302d7283 100644 --- a/apps/contract/src/contract.ts +++ b/apps/contract/src/contract.ts @@ -624,14 +624,14 @@ export type AllThreads = typeof AllThreads.Type; * `sync/delta/` streams its draft between checkpoints. * - `threads.list`: `sync/changed` naming a thread or a removed session; every `sync/status` and * `sync/tracking`. - * - `metrics.usage` and `metrics.hours`: every `sync/usage`; the titles and names they carry are - * read again with it. + * - `metrics.usage` and `metrics.hours`: every `sync/usage`, and every `sync/changed` for the session + * titles, deletions and project names they carry. */ export const readTopics = { sidebar: [syncTopics.changed, syncTopics.status, syncTopics.tracking], threads: [syncTopics.changed, syncTopics.status, syncTopics.tracking], line: (threadId: string) => [syncTopics.changed, syncTopics.delta(threadId)], - metrics: [syncTopics.usage], + metrics: [syncTopics.usage, syncTopics.changed], }; const [UsageWire, TallyWire] = UsageItem.members; diff --git a/apps/server/src/db.ts b/apps/server/src/db.ts index b8914f73..27997f6a 100644 --- a/apps/server/src/db.ts +++ b/apps/server/src/db.ts @@ -275,17 +275,20 @@ const migrations = { yield* sql`INSERT INTO organization ${sql.insert({ id: "personal", name: "Personal", createdAt: now, updatedAt: now })}`; }), - // Each machine's usage rollup, as `usage.sync` sends it. Keyed by store too: a machine whose - // store is recreated starts a new one, and the old store's rows are then the only history left. - // No keys to project, session or thread: the rows outlive them. + // Each machine's usage rollup, as `usage.sync` sends it, keyed by store: a recreated store starts + // new rows and keeps the old ones as history, and a removed server re-added later replaces its own. + // No foreign keys: the rows outlive their machine, project, session and thread; `machine_id` is the + // machine that last sent them. "202611140001_usage": Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; - // `rollup` is the usage cursor, counted in `rollup_store`: it moves apart from `cursor` and `store_id`. + // `rollup` is the usage cursor, counted in `rollup_store` while `rollup_boot` was its latest open: + // it moves apart from `cursor` and `store_id`. yield* sql`ALTER TABLE machine ADD COLUMN rollup INTEGER NOT NULL DEFAULT 0`; yield* sql`ALTER TABLE machine ADD COLUMN rollup_store TEXT`; + yield* sql`ALTER TABLE machine ADD COLUMN rollup_boot TEXT`; yield* sql` CREATE TABLE usage ( - machine_id TEXT NOT NULL REFERENCES machine(id) ON DELETE CASCADE, + machine_id TEXT NOT NULL, store_id TEXT NOT NULL, hour INTEGER NOT NULL, thread_id TEXT NOT NULL, @@ -309,12 +312,12 @@ const migrations = { timed INTEGER NOT NULL, durations TEXT NOT NULL, rev INTEGER NOT NULL, - PRIMARY KEY (machine_id, store_id, hour, thread_id, provider, model, thinking) + PRIMARY KEY (store_id, hour, thread_id, provider, model, thinking) ) `; yield* sql` CREATE TABLE tally ( - machine_id TEXT NOT NULL REFERENCES machine(id) ON DELETE CASCADE, + machine_id TEXT NOT NULL, store_id TEXT NOT NULL, hour INTEGER NOT NULL, thread_id TEXT NOT NULL, @@ -324,7 +327,7 @@ const migrations = { project_id TEXT NOT NULL, count INTEGER NOT NULL, rev INTEGER NOT NULL, - PRIMARY KEY (machine_id, store_id, hour, thread_id, kind, name) + PRIMARY KEY (store_id, hour, thread_id, kind, name) ) `; }), diff --git a/apps/server/src/mirror.ts b/apps/server/src/mirror.ts index 40292a5d..102fb619 100644 --- a/apps/server/src/mirror.ts +++ b/apps/server/src/mirror.ts @@ -290,13 +290,13 @@ export const usage = (machineId: string, storeId: string, rows: ReadonlyArray Effect.gen(function* () { const info = yield* client("server.info", {}); - const [machine] = yield* sql<{ rollup: number; rollupStore: string | null }>` - SELECT rollup, rollup_store FROM machine WHERE id = ${machineId} + const [machine] = yield* sql<{ rollup: number; rollupStore: string | null; rollupBoot: string | null }>` + SELECT rollup, rollup_store, rollup_boot FROM machine WHERE id = ${machineId} `; if (machine === undefined) return yield* Effect.never; yield* unclaimed(machineId, info.storeId); - const same = info.storeId === machine.rollupStore; - let restored = same && info.rev < machine.rollup; + const resumed = info.storeId === machine.rollupStore && info.rev >= machine.rollup; + let whole = !resumed; yield* batches( - client("usage.sync", { after: same && !restored ? machine.rollup : 0 }), + client("usage.sync", { + after: resumed ? machine.rollup : 0, + ...(resumed && machine.rollupBoot !== null ? { boot: machine.rollupBoot } : {}), + }), (item) => item._tag === "Synced", () => Effect.succeed(true), (rows, synced) => Effect.gen(function* () { if (synced.storeId !== info.storeId) return yield* Effect.fail("The usage stream changed store"); + const replaced = whole || synced.reset === true; yield* transact((append) => Effect.gen(function* () { - if (restored) { - yield* sql`DELETE FROM usage WHERE machine_id = ${machineId} AND store_id = ${synced.storeId}`; - yield* sql`DELETE FROM tally WHERE machine_id = ${machineId} AND store_id = ${synced.storeId}`; + if (replaced) { + yield* sql`DELETE FROM usage WHERE store_id = ${synced.storeId}`; + yield* sql`DELETE FROM tally WHERE store_id = ${synced.storeId}`; } yield* Mirror.usage(machineId, synced.storeId, rows); yield* sql` - UPDATE machine SET rollup = ${synced.rev}, rollup_store = ${synced.storeId} WHERE id = ${machineId} + UPDATE machine SET rollup = ${synced.rev}, rollup_store = ${synced.storeId}, + rollup_boot = ${synced.boot ?? null} + WHERE id = ${machineId} `; - if (rows.length > 0 || restored) + if (rows.length > 0 || replaced) yield* append({ topic: syncTopics.usage, source: "sync", @@ -424,7 +433,7 @@ export const layer = Layer.effect( }); }), ); - restored = false; + whole = false; }), ); return yield* Effect.fail("The usage stream ended"); diff --git a/apps/server/test/__artifacts__/schema.json b/apps/server/test/__artifacts__/schema.json index 6fbeef63..7ce0f74e 100644 --- a/apps/server/test/__artifacts__/schema.json +++ b/apps/server/test/__artifacts__/schema.json @@ -70,7 +70,7 @@ "type": "table", "name": "machine", "tbl_name": "machine", - "sql": "CREATE TABLE machine ( id TEXT PRIMARY KEY, kind TEXT NOT NULL, executable TEXT, home TEXT, args TEXT NOT NULL DEFAULT '', env TEXT NOT NULL DEFAULT '[]', url TEXT, namespace TEXT NOT NULL, is_default INTEGER NOT NULL DEFAULT 0, pos INTEGER NOT NULL, store_id TEXT UNIQUE, cursor INTEGER NOT NULL DEFAULT 0, created_at INTEGER NOT NULL , rollup INTEGER NOT NULL DEFAULT 0, rollup_store TEXT, host TEXT, name TEXT)" + "sql": "CREATE TABLE machine ( id TEXT PRIMARY KEY, kind TEXT NOT NULL, executable TEXT, home TEXT, args TEXT NOT NULL DEFAULT '', env TEXT NOT NULL DEFAULT '[]', url TEXT, namespace TEXT NOT NULL, is_default INTEGER NOT NULL DEFAULT 0, pos INTEGER NOT NULL, store_id TEXT UNIQUE, cursor INTEGER NOT NULL DEFAULT 0, created_at INTEGER NOT NULL , rollup INTEGER NOT NULL DEFAULT 0, rollup_store TEXT, rollup_boot TEXT, host TEXT, name TEXT)" }, { "type": "table", @@ -154,7 +154,7 @@ "type": "table", "name": "tally", "tbl_name": "tally", - "sql": "CREATE TABLE tally ( machine_id TEXT NOT NULL REFERENCES machine(id) ON DELETE CASCADE, store_id TEXT NOT NULL, hour INTEGER NOT NULL, thread_id TEXT NOT NULL, kind TEXT NOT NULL, name TEXT NOT NULL, session_id TEXT NOT NULL, project_id TEXT NOT NULL, count INTEGER NOT NULL, rev INTEGER NOT NULL, PRIMARY KEY (machine_id, store_id, hour, thread_id, kind, name) )" + "sql": "CREATE TABLE tally ( machine_id TEXT NOT NULL, store_id TEXT NOT NULL, hour INTEGER NOT NULL, thread_id TEXT NOT NULL, kind TEXT NOT NULL, name TEXT NOT NULL, session_id TEXT NOT NULL, project_id TEXT NOT NULL, count INTEGER NOT NULL, rev INTEGER NOT NULL, PRIMARY KEY (store_id, hour, thread_id, kind, name) )" }, { "type": "table", @@ -202,7 +202,7 @@ "type": "table", "name": "usage", "tbl_name": "usage", - "sql": "CREATE TABLE usage ( machine_id TEXT NOT NULL REFERENCES machine(id) ON DELETE CASCADE, store_id TEXT NOT NULL, hour INTEGER NOT NULL, thread_id TEXT NOT NULL, provider TEXT NOT NULL, model TEXT NOT NULL, thinking TEXT NOT NULL, session_id TEXT NOT NULL, project_id TEXT NOT NULL, requests INTEGER NOT NULL, input INTEGER NOT NULL, output INTEGER NOT NULL, cache_read INTEGER NOT NULL, cache_write INTEGER NOT NULL, reasoning INTEGER NOT NULL, cost REAL NOT NULL, cost_input REAL NOT NULL, cost_output REAL NOT NULL, cost_cache_read REAL NOT NULL, cost_cache_write REAL NOT NULL, duration REAL NOT NULL, timed INTEGER NOT NULL, durations TEXT NOT NULL, rev INTEGER NOT NULL, PRIMARY KEY (machine_id, store_id, hour, thread_id, provider, model, thinking) )" + "sql": "CREATE TABLE usage ( machine_id TEXT NOT NULL, store_id TEXT NOT NULL, hour INTEGER NOT NULL, thread_id TEXT NOT NULL, provider TEXT NOT NULL, model TEXT NOT NULL, thinking TEXT NOT NULL, session_id TEXT NOT NULL, project_id TEXT NOT NULL, requests INTEGER NOT NULL, input INTEGER NOT NULL, output INTEGER NOT NULL, cache_read INTEGER NOT NULL, cache_write INTEGER NOT NULL, reasoning INTEGER NOT NULL, cost REAL NOT NULL, cost_input REAL NOT NULL, cost_output REAL NOT NULL, cost_cache_read REAL NOT NULL, cost_cache_write REAL NOT NULL, duration REAL NOT NULL, timed INTEGER NOT NULL, durations TEXT NOT NULL, rev INTEGER NOT NULL, PRIMARY KEY (store_id, hour, thread_id, provider, model, thinking) )" }, { "type": "table", @@ -235,6 +235,7 @@ "created_at": "
    "], diff --git a/packages/harness/src/db/db.ts b/packages/harness/src/db/db.ts index ff663934..625f6cdf 100644 --- a/packages/harness/src/db/db.ts +++ b/packages/harness/src/db/db.ts @@ -1,6 +1,7 @@ import { SqliteClient, SqliteMigrator } from "@effect/sql-sqlite-node"; import { Config, Effect, Layer, String as Str } from "effect"; import { Migrator, SqlClient } from "effect/sql"; +import { uuidv7 } from "uuidv7"; import * as Global from "../global.ts"; import { fileSystem } from "../host.ts"; import { posix } from "../util/posix.ts"; @@ -25,6 +26,7 @@ const setup = Effect.gen(function* () { yield* sql`PRAGMA cache_size = -64000`; yield* sql`PRAGMA foreign_keys = ON`; yield* SqliteMigrator.run({ loader: Migrator.fromRecord(migrations) }); + yield* sql`INSERT INTO boot (id, rev) SELECT ${uuidv7()}, rev FROM store`; }); // Provides `SqlClient` (and `SqliteClient`) for the database at `location`, diff --git a/packages/harness/src/db/migrations.ts b/packages/harness/src/db/migrations.ts index f776955c..77c79b36 100644 --- a/packages/harness/src/db/migrations.ts +++ b/packages/harness/src/db/migrations.ts @@ -378,6 +378,19 @@ export const migrations = { ), ); }), + + // Each open of the database (db.ts), with the revision it opened at: a client that remembers the + // latest one beside its cursor can tell a restored backup from a store that only moved on. + "202611140002_boot": Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* sql` + CREATE TABLE boot ( + seq INTEGER PRIMARY KEY, + id TEXT NOT NULL UNIQUE, + rev INTEGER NOT NULL + ) + `; + }), }; /** diff --git a/packages/harness/src/db/store.ts b/packages/harness/src/db/store.ts index 97bb5cda..8cbdb9fe 100644 --- a/packages/harness/src/db/store.ts +++ b/packages/harness/src/db/store.ts @@ -1,10 +1,11 @@ import { Clock, Context, Effect, Layer, PubSub, type Scope } from "effect"; import { SqlClient, type SqlError } from "effect/sql"; -/** The store's identity and its last revision taken. */ +/** The store's identity, its last revision taken, and its latest open (`boot`). */ export interface Head { readonly id: string; readonly rev: number; + readonly boot: string; } export interface Interface { @@ -18,6 +19,12 @@ export interface Interface { /** The enclosing write's revision, taken on first use; every row the write touches is stamped with it. */ readonly rev: Effect.Effect; readonly head: Effect.Effect; + /** + * Whether a client that read up to `rev` while `boot` was the latest open is still on this store's + * line: `boot` is known and every open since came at `rev` or later. A restored backup fails it once + * the client had read past the backup. + */ + readonly onLine: (boot: string, rev: number) => Effect.Effect; /** * A doorbell for the caller's scope, rung after every commit that took a revision. Register it * before reading `head`, so no commit falls between the read and the first ring. @@ -83,11 +90,22 @@ export const layer = Layer.effect( }).pipe(Effect.orDie); const head = Effect.gen(function* () { - const [row] = yield* sql`SELECT id, rev FROM store`; + const [row] = yield* sql` + SELECT id, rev, (SELECT id FROM boot ORDER BY seq DESC LIMIT 1) AS boot FROM store + `; if (row === undefined) return yield* Effect.die(new Error("store row is missing")); return row; }).pipe(Effect.orDie); + const onLine = Effect.fn("Store.onLine")(function* (boot: string, rev: number) { + const [own] = yield* sql<{ readonly seq: number }>`SELECT seq FROM boot WHERE id = ${boot}`; + if (own === undefined) return false; + const [later] = yield* sql<{ readonly since: number | null }>` + SELECT MIN(rev) AS since FROM boot WHERE seq > ${own.seq} + `; + return later === undefined || later.since === null || later.since >= rev; + }, Effect.orDie); + const wake = Effect.gen(function* () { const pubsub = yield* PubSub.sliding(1); const subscription = yield* PubSub.subscribe(pubsub); @@ -98,6 +116,6 @@ export const layer = Layer.effect( return subscription; }); - return Service.of({ write, rev, head, wake }); + return Service.of({ write, rev, head, onLine, wake }); }), ); diff --git a/packages/harness/test/db/db.test.ts b/packages/harness/test/db/db.test.ts index 4650bb32..f361886d 100644 --- a/packages/harness/test/db/db.test.ts +++ b/packages/harness/test/db/db.test.ts @@ -94,6 +94,7 @@ describe("Database", () => { expect(applied.map((row) => `${row.migrationId}_${row.name}`)).toEqual([ "202610290001_init", "202611140001_usage", + "202611140002_boot", ]); const tables = yield* sql<{ name: string }>` SELECT name FROM sqlite_master @@ -101,6 +102,7 @@ describe("Database", () => { ORDER BY name `; expect(tables.map((row) => row.name)).toEqual([ + "boot", "document", "entry", "event", From 00f94a5683540c1fe13a51d6002c8806d80a6b3a Mon Sep 17 00:00:00 2001 From: Sanchit Rk <0xsanchit@gmail.com> Date: Sat, 14 Nov 2026 16:24:25 +0530 Subject: [PATCH 7/8] test(server): simulate a long-lived desktop against seeded OrbStack machines seed.ts writes a history of sessions, models and tools through the thread store; the live suite checks the metrics reads against each machine's own totals across live turns, app and machine restarts, a paused machine, a network cut, a restored backup, a server removed and added again, and a recreated store. The fixture model can stamp replies with the real time (CODEWORK_FIXTURE_NOW=1). Co-Authored-By: Claude Opus 5.5 --- .../__artifacts__/metrics.simulation.json | 176 ++++++++ apps/server/test/fixtures/codework.ts | 12 +- apps/server/test/machines/seed.ts | 268 ++++++++++++ .../test/metrics.simulation.live.test.ts | 411 ++++++++++++++++++ 4 files changed, 866 insertions(+), 1 deletion(-) create mode 100644 apps/server/test/__artifacts__/metrics.simulation.json create mode 100644 apps/server/test/machines/seed.ts create mode 100644 apps/server/test/metrics.simulation.live.test.ts diff --git a/apps/server/test/__artifacts__/metrics.simulation.json b/apps/server/test/__artifacts__/metrics.simulation.json new file mode 100644 index 00000000..2612b80d --- /dev/null +++ b/apps/server/test/__artifacts__/metrics.simulation.json @@ -0,0 +1,176 @@ +{ + "seeded": { + "cw-a": { + "sessions": 300, + "prompts": 2277, + "requests": 7321, + "aborted": 196, + "failed": 137, + "calls": 7778 + }, + "cw-b": { + "sessions": 120, + "prompts": 881, + "requests": 2839, + "aborted": 98, + "failed": 60, + "calls": 3042 + } + }, + "connected": { + "before": { + "all": { + "requests": 10160, + "tokens": 496714759, + "cost": 228.254801875, + "timed": 9866, + "prompts": 3158, + "tools": 10315 + }, + "sessions": 420, + "deleted": 0 + }, + "after": { + "all": { + "requests": 10160, + "tokens": 496714759, + "cost": 228.254801875, + "timed": 9866, + "prompts": 3158, + "tools": 10315 + }, + "sessions": 420, + "deleted": 7 + } + }, + "live": { + "all": { + "requests": 10166, + "tokens": 496714771, + "cost": 228.254801875, + "timed": 9872, + "prompts": 3161, + "tools": 10318 + }, + "sessions": 421, + "deleted": 7 + }, + "resumed": { + "applied": [ + 3, + 3 + ], + "totals": { + "all": { + "requests": 10178, + "tokens": 496714795, + "cost": 228.254801875, + "timed": 9884, + "prompts": 3167, + "tools": 10324 + }, + "sessions": 423, + "deleted": 7 + } + }, + "paused": { + "paused": { + "all": { + "requests": 10182, + "tokens": 496714803, + "cost": 228.254801875, + "timed": 9888, + "prompts": 3169, + "tools": 10326 + }, + "sessions": 424, + "deleted": 7 + }, + "resumed": { + "all": { + "requests": 10182, + "tokens": 496714803, + "cost": 228.254801875, + "timed": 9888, + "prompts": 3169, + "tools": 10326 + }, + "sessions": 424, + "deleted": 7 + } + }, + "restartedAway": { + "all": { + "requests": 10186, + "tokens": 496714811, + "cost": 228.254801875, + "timed": 9892, + "prompts": 3171, + "tools": 10328 + }, + "sessions": 425, + "deleted": 7 + }, + "cut": { + "all": { + "requests": 10188, + "tokens": 496714815, + "cost": 228.254801875, + "timed": 9894, + "prompts": 3172, + "tools": 10329 + }, + "sessions": 426, + "deleted": 7 + }, + "restored": { + "all": { + "requests": 10192, + "tokens": 496714823, + "cost": 228.254801875, + "timed": 9898, + "prompts": 3174, + "tools": 10331 + }, + "sessions": 428, + "deleted": 9 + }, + "readded": { + "kept": { + "all": { + "requests": 10192, + "tokens": 496714823, + "cost": 228.254801875, + "timed": 9898, + "prompts": 3174, + "tools": 10331 + }, + "sessions": 428, + "deleted": 130 + }, + "again": { + "all": { + "requests": 10192, + "tokens": 496714823, + "cost": 228.254801875, + "timed": 9898, + "prompts": 3174, + "tools": 10331 + }, + "sessions": 428, + "deleted": 9 + } + }, + "recreated": { + "all": { + "requests": 10196, + "tokens": 496714831, + "cost": 228.254801875, + "timed": 9902, + "prompts": 3176, + "tools": 10333 + }, + "sessions": 429, + "deleted": 130 + } +} diff --git a/apps/server/test/fixtures/codework.ts b/apps/server/test/fixtures/codework.ts index 6a332cae..2ab7670d 100644 --- a/apps/server/test/fixtures/codework.ts +++ b/apps/server/test/fixtures/codework.ts @@ -8,7 +8,7 @@ import { existsSync, mkdirSync, writeFileSync } from "node:fs"; import { createServer } from "node:http"; import { join } from "node:path"; import { createInterface } from "node:readline"; -import { assistant } from "../../../../packages/harness/test/fixtures/llm.ts"; +import { assistant as scriptedAssistant } from "../../../../packages/harness/test/fixtures/llm.ts"; import { testModel } from "../../../../packages/harness/test/fixtures/model.ts"; import type * as LLM from "../../../../packages/harness/src/runner/llm.ts"; import * as Clones from "../../../../packages/codework/src/server/clones.ts"; @@ -79,6 +79,16 @@ const bashOf = (input: LLM.Input) => { return /^bash (.+)$/s.exec(promptOf(input))?.[1]; }; +// With `CODEWORK_FIXTURE_NOW=1` a reply took the last 1.5 s of real time, so usage lands in today's +// hours; otherwise its times are its index, in the first hour of 1970, the same on every run. +const assistant: typeof scriptedAssistant = (input, index, overrides) => + scriptedAssistant(input, index, { + ...(process.env.CODEWORK_FIXTURE_NOW === "1" + ? { time: { created: Date.now() - 1_500, completed: Date.now() } } + : {}), + ...overrides, + }); + // Ids unique across restarts of this process: entry ids are the reply's message ids. let count = 0; const scripted: LLM.Open = (input, signal) => diff --git a/apps/server/test/machines/seed.ts b/apps/server/test/machines/seed.ts new file mode 100644 index 00000000..f95932ad --- /dev/null +++ b/apps/server/test/machines/seed.ts @@ -0,0 +1,268 @@ +/* @effect-diagnostics nodeBuiltinImport:off -- a one-shot script run inside an OrbStack machine. */ +/** + * Fills every session of a stopped CodeWork store with a history of turns across the past `days` + * days, through the thread store's own writes, so the usage rollup, thread totals and revisions come + * out as a live run makes them: + * node --conditions=development seed.ts [] + * Each session gets a start day (a little busier recently, quieter at weekends) and 2-14 turns on its days; + * each turn is a prompt and 1-6 requests to one model of its project's mix, the last answering and + * the others calling tools. A few requests fail, a few are interrupted (aborted, never timed), and a + * few calls error or are skipped. Prints one JSON line of what it wrote. + */ +import { Effect, Layer } from "effect"; +import { randomUUID } from "node:crypto"; +import { Database, Event, EventRegistry, Usage } from "../../../../packages/harness/src/effect.ts"; +import * as Document from "../../../../packages/harness/src/document/document.ts"; +import * as ThreadSchema from "../../../../packages/harness/src/thread/schema.ts"; +import * as Thread from "../../../../packages/harness/src/thread/thread.ts"; +import type { ToolStatus } from "../../../../packages/harness/src/db/schema.sql.ts"; + +const [database, daysArg = "365", seedArg = "1", untilArg] = process.argv.slice(2); +if (database === undefined) throw new Error("usage: seed.ts []"); +const DAYS = Number(daysArg); +const UNTIL = untilArg === undefined ? Date.now() : Number(untilArg); +const DAY = 86_400_000; + +// mulberry32: the same seed writes the same history. +let state = Number(seedArg) >>> 0; +const random = () => { + state = (state + 0x6d2b79f5) >>> 0; + let t = state; + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4_294_967_296; +}; +const int = (low: number, high: number) => low + Math.floor(random() * (high - low + 1)); +const pick = (items: ReadonlyArray): A => items[Math.floor(random() * items.length)]!; +const weighted = (items: ReadonlyArray): A => { + let left = random() * items.reduce((sum, [, weight]) => sum + weight, 0); + for (const [item, weight] of items) if ((left -= weight) <= 0) return item; + return items.at(-1)![0]; +}; +/** Right-skewed, like real response times: median `median`, rarely 10x. */ +const skewed = (median: number) => Math.round(median * Math.exp((random() + random() + random() - 1.5) * 1.6)); + +interface Model { + readonly provider: string; + readonly name: string; + readonly model: string; + /** USD per million tokens: input, output, cache read, cache write; zero for an unpriced one. */ + readonly price: readonly [number, number, number, number]; + readonly speed: number; +} +const MODELS: ReadonlyArray = [ + { provider: "anthropic", name: "Anthropic", model: "claude-opus-5-5", price: [5, 25, 0.5, 6.25], speed: 55 }, + { provider: "anthropic", name: "Anthropic", model: "claude-sonnet-5-5", price: [3, 15, 0.3, 3.75], speed: 80 }, + { provider: "anthropic", name: "Anthropic", model: "claude-haiku-5-5", price: [1, 5, 0.1, 1.25], speed: 140 }, + { provider: "openai", name: "OpenAI", model: "gpt-6-luna", price: [1.25, 10, 0.125, 0], speed: 70 }, + { provider: "openai-codex", name: "OpenAI Codex", model: "gpt-6-luna", price: [0, 0, 0, 0], speed: 65 }, + { provider: "github-copilot", name: "GitHub Copilot", model: "kimi-k3", price: [0.6, 2.5, 0.15, 0], speed: 45 }, + { provider: "openrouter", name: "OpenRouter", model: "crouton-alpha", price: [0, 0, 0, 0], speed: 90 }, +]; +/** Priced models are the common case; unpriced ones show up now and then. */ +const pickModel = () => weighted(MODELS.map((model) => [model, model.price[0] === 0 ? 1 : 3] as const)); +const THINKING = [ + ["off", 2], + ["minimal", 1], + ["low", 3], + ["medium", 4], + ["high", 5], + ["xhigh", 1], +] as const; +const TOOLS = [ + ["bash", 30], + ["read", 25], + ["edit", 14], + ["grep", 10], + ["write", 6], + ["glob", 5], + ["todo", 4], + ["web_fetch", 2], + ["task", 2], +] as const; +const PROMPTS = [ + "Fix the failing test in the sync suite", + "Add a retry with backoff to the uploader", + "Explain how the cursor resumes after a restart", + "Refactor the settings loader into its own module", + "Why does the sidebar flicker when a thread finishes?", + "Write a migration that adds an index on hour", + "Review this diff for race conditions", + "Make the chart tooltip follow the pointer", +]; + +const layer = Thread.layer.pipe( + Layer.provideMerge(Usage.layer), + Layer.provideMerge(Document.layer), + Layer.provideMerge(Event.layer), + Layer.provideMerge(EventRegistry.layer()), + Layer.provideMerge(Database.layer(database)), +); + +/** A start day across the period, leaning recent, a third as busy at weekends. */ +const startDay = () => { + for (;;) { + const ago = Math.floor(DAYS * random() ** 1.25); + const weekday = new Date(UNTIL - ago * DAY).getUTCDay(); + if (weekday !== 0 && weekday !== 6) return ago; + if (random() < 0.33) return ago; + } +}; + +const program = Effect.gen(function* () { + const sql = yield* Database.SqlClient.SqlClient; + const threads = yield* Thread.Service; + const sessions = yield* sql<{ readonly id: string; readonly threadId: string; readonly projectId: string }>` + SELECT s.id, t.id AS thread_id, sp.project_id FROM session s + JOIN thread t ON t.session_id = s.id JOIN space sp ON sp.id = s.space_id + ORDER BY s.id + `; + // Each project leans on two or three models. + const mixes = new Map>(); + const mixOf = (projectId: string) => { + const known = mixes.get(projectId); + if (known !== undefined) return known; + const mix = Array.from({ length: int(2, 3) }, (_, rank) => [pickModel(), 6 - rank * 2] as const); + mixes.set(projectId, mix); + return mix; + }; + const wrote = { sessions: 0, prompts: 0, requests: 0, aborted: 0, failed: 0, calls: 0 }; + + for (const session of sessions) { + const threadId = ThreadSchema.ID.make(session.threadId); + const [head] = yield* sql<{ readonly seq: number }>`SELECT seq FROM sequence WHERE aggregate_id = ${session.id}`; + let seq = head?.seq ?? 1; + let day = startDay(); + const mix = mixOf(session.projectId); + const model = weighted(mix); + const thinking = weighted(THINKING); + let context = int(8_000, 30_000); + for (let turn = 0, turns = int(2, 14); turn < turns && day >= 0; turn++) { + // Work hours, 08:00-20:00 UTC, so most zones see a working day. + let at = UNTIL - day * DAY - (UNTIL % DAY) + int(8, 19) * 3_600_000 + int(0, 3_599_999); + if (at > UNTIL) at = UNTIL - int(60_000, 3_600_000); + const promptId = randomUUID(); + yield* threads.append({ + id: promptId, + threadId, + seq: ++seq, + type: "user", + data: JSON.stringify({ messageId: promptId, role: "user", time: { created: at } }), + parts: [{ type: "text", data: JSON.stringify({ type: "text", text: pick(PROMPTS) }) }], + }); + wrote.prompts++; + const requests = int(1, 6); + for (let step = 0; step < requests; step++) { + const last = step === requests - 1; + const fails = random() < 0.02; + const interrupted = !fails && random() < 0.03; + const input = fails ? int(200, 2_000) : int(300, 4_000); + const cacheRead = fails ? 0 : Math.round(context * (0.55 + random() * 0.4)); + const cacheWrite = fails ? 0 : int(0, 3_000); + const output = fails ? 0 : last ? int(150, 2_500) : int(40, 600); + const reasoning = thinking === "off" || fails ? 0 : Math.round(output * random() * 0.6); + const duration = skewed(1_200 + (output / model.speed) * 1_000); + const [pIn, pOut, pRead, pWrite] = model.price.map((perMillion) => perMillion / 1_000_000) as [ + number, + number, + number, + number, + ]; + const cost = { + input: input * pIn, + output: output * pOut, + cacheRead: cacheRead * pRead, + cacheWrite: cacheWrite * pWrite, + total: input * pIn + output * pOut + cacheRead * pRead + cacheWrite * pWrite, + }; + const calls = + last || fails + ? [] + : Array.from( + { + length: weighted([ + [1, 6], + [2, 3], + [3, 1], + ] as const), + }, + () => weighted(TOOLS), + ); + const messageId = randomUUID(); + const envelope = { + messageId, + role: "assistant", + protocol: model.provider, + provider: { env: [], id: model.provider, name: model.name, source: "custom" }, + model: model.model, + thinkingLevel: thinking, + usage: { + input, + output, + cacheRead, + cacheWrite, + reasoning, + totalTokens: input + output + cacheRead + cacheWrite, + cost, + }, + stopReason: fails ? "error" : calls.length > 0 ? "toolUse" : interrupted ? "aborted" : "stop", + time: { created: at, completed: at + duration }, + ...(fails ? { errorMessage: "provider returned 529: overloaded" } : {}), + }; + const parts = [ + ...(output > 0 + ? [{ type: "text" as const, data: JSON.stringify({ type: "text", text: "Working on it." }) }] + : []), + ...calls.map((name, index) => { + const callId = `call_${messageId}_${index}`; + const status: ToolStatus = interrupted + ? "aborted" + : random() < 0.06 + ? "error" + : random() < 0.02 + ? "skipped" + : "completed"; + return { + type: "toolCall" as const, + callId, + toolName: name, + status, + data: JSON.stringify({ + type: "toolCall", + callID: callId, + name, + arguments: {}, + status, + time: { start: at + duration, end: at + duration + int(50, 4_000) }, + ...(status === "completed" || status === "error" + ? { result: { content: [{ type: "text", text: "ok" }], isError: status === "error" } } + : {}), + }), + }; + }), + ]; + const entry = { id: messageId, threadId, data: JSON.stringify(envelope), parts }; + if (interrupted) { + yield* threads.append({ ...entry, seq: ++seq, type: "assistant", state: "draft" }); + yield* threads.abort({ ...entry, seq: ++seq }); + wrote.aborted++; + } else yield* threads.append({ ...entry, seq: ++seq, type: "assistant" }); + wrote.requests++; + if (fails) wrote.failed++; + wrote.calls += parts.filter((part) => part.type === "toolCall").length; + context += input + output; + at += duration + int(500, 20_000); + if (interrupted || fails) break; + } + // The next turn: the same day, or a later one. + if (random() < 0.55) day -= int(1, 9); + } + // Live events of this session continue after the history. + yield* sql`UPDATE sequence SET seq = ${seq} WHERE aggregate_id = ${session.id}`; + wrote.sessions++; + } + return wrote; +}); + +const wrote = await Effect.runPromise(program.pipe(Effect.provide(layer))); +console.log(JSON.stringify(wrote)); diff --git a/apps/server/test/metrics.simulation.live.test.ts b/apps/server/test/metrics.simulation.live.test.ts new file mode 100644 index 00000000..294ff336 --- /dev/null +++ b/apps/server/test/metrics.simulation.live.test.ts @@ -0,0 +1,411 @@ +/* @effect-diagnostics nodeBuiltinImport:off -- the suite runs CodeWork servers on OrbStack Linux machines. */ +import { Effect, Stream } from "effect"; +import { afterAll, beforeAll, describe, expect, it } from "vite-plus/test"; +import { Contract, type MetricsUsage } from "@codeworksh/contract"; +import { Session } from "../../../packages/harness/src/effect.ts"; +import { + type AppRpc, + app, + cleanup, + cursorOf, + lastSeq, + machineRow, + notices, + query, + type Row, + snapshot, + stop, + tempDir, + waitFor, +} from "./fixtures/apps.ts"; +import { available, ensure, machine, type Machine, orb } from "./machines/orb.ts"; + +/* + * A long-lived desktop against two seeded machines: cw-a holds a year of history (300 sessions over + * six projects), cw-b 200 days (120 sessions), both written through the thread store by + * `machines/seed.ts`. Every case reads the totals inside each machine with sqlite3 and checks the + * app's metrics reads against them: the first connect, live turns, the app down while work goes on, + * a machine paused and resumed (and the app down across it), a network cut, a backup restored after + * its head passed the app's cursor, a server removed and added again, and a store recreated. + * `metrics.simulation.json` records each case's totals. + */ + +const cwA = machine("cw-a"); +const cwB = machine("cw-b"); +const pool = [cwA, cwB]; +type Desk = Awaited>; + +let appHome = ""; +let desk: Desk; +const ids = new Map(); +const idOf = (m: Machine) => ids.get(m.name)!; +const DAY = 86_400_000; +const until = () => new Date(Date.now() + 2 * DAY).toISOString().slice(0, 10); +const everything = (zone = "UTC") => ({ zone, since: "1970-01-01", until: until() }); +const PROJECTS = ["codework", "billing-api", "mobile-app", "infra", "docs-site", "ml-pipeline"]; +const live = { CODEWORK_FIXTURE_HOST_OS: "1", CODEWORK_FIXTURE_NOW: "1" }; + +/** The machine's server stopped and its home moved aside: the next start makes a new store. */ +const aside = async (m: Machine) => { + await m.net("restore"); + await m.stop("KILL"); + await m.sh(`[ -d ${m.cwHome} ] && mv ${m.cwHome} ${m.cwHome}.sim-$(date +%s%N); mkdir -p ${m.cwHome}/work`); +}; + +/** A new store on `m` with `count` sessions and `days` of history, served again. */ +const seeded = async (m: Machine, count: number, days: number, seed: number) => { + await aside(m); + await m.start(live); + await m.sh(`mkdir -p ${PROJECTS.map((project) => `${m.cwHome}/work/${project}`).join(" ")}`); + for (let i = 0; i < count; i++) { + const project = PROJECTS[(i * 7 + seed) % PROJECTS.length]!; + await m.run((rpc) => + rpc["session.create"]({ + directory: `${m.cwHome}/work/${project}`, + title: `Session ${i + 1} in ${project}`, + project: { name: project }, + }), + ); + } + await m.stop("TERM"); + const wrote = await m.sh( + `cd ~/codework && ~/.local/node/bin/node --conditions=development apps/server/test/machines/seed.ts ${m.database} ${days} ${seed}`, + ); + await m.start(live); + return JSON.parse(wrote.trim().split("\n").at(-1)!) as Record; +}; + +type Ref = { readonly sessionId: string }; +const turn = (m: Machine, ref: Ref, text: string) => + m.run((rpc) => { + const sessionId = Session.SessionSchema.ID.make(ref.sessionId); + return rpc["thread.prompt"]({ sessionId, text }).pipe(Effect.andThen(rpc["thread.wait"]({ sessionId }))); + }); +/** New work on `m`: a new session in a project with `turns` tool turns (two requests each). */ +const work = async (m: Machine, turns: number) => { + const created = await m.run((rpc) => + rpc["session.create"]({ + directory: `${m.cwHome}/work/codework`, + title: "Live work", + project: { name: "codework" }, + }), + ); + for (let i = 0; i < turns; i++) await turn(m, { sessionId: created.session.id }, `bash echo live ${i}`); + return { sessionId: created.session.id as string }; +}; + +/** The app's latest warnings, for a wait that timed out. */ +const warnings = () => { + const last = desk.lines.findLastIndex((line) => line.includes("WARN") || line.includes("ERROR")); + return last === -1 + ? "" + : desk.lines + .slice(last, last + 12) + .join("\n") + .slice(0, 3_000); +}; + +/** Waits until the app's usage and index cursors for `m` reached its head, which `m` must be serving. */ +const rolled = (m: Machine, { index = true, timeout = 180_000 } = {}) => + caught(m, index, timeout).catch((error: Error) => { + throw new Error(`${error.message}\n${warnings()}`); + }); +const caught = async (m: Machine, index: boolean, timeout: number) => { + const { storeId, rev } = await m.run((rpc) => rpc["server.info"]({})); + await waitFor( + () => { + const row = cursorOf(appHome, idOf(m)); + return row.rollup_store === storeId && row.rollup >= rev ? row : undefined; + }, + `${m.name} rolled up at rev ${rev}`, + timeout, + ); + if (index) + await waitFor( + () => { + const row = machineRow(appHome, idOf(m)); + return row?.store_id === storeId && row.cursor >= rev ? row : undefined; + }, + `${m.name} indexed at rev ${rev}`, + timeout, + ); + return { storeId, rev }; +}; + +/** What `usage.sync` past `after` sends up to its first `Synced`: what a resume should apply. */ +const pending = async (m: Machine, after: number) => + ( + await m.run((rpc) => + rpc["usage.sync"]({ after }).pipe( + Stream.takeUntil((item) => item._tag === "Synced"), + Stream.runCollect, + ), + ) + ).filter((item) => item._tag !== "Synced").length; + +interface Truth { + readonly requests: number; + readonly tokens: number; + readonly cost: number; + readonly timed: number; + readonly prompts: number; + readonly tools: number; +} +const none: Truth = { requests: 0, tokens: 0, cost: 0, timed: 0, prompts: 0, tools: 0 }; +const plus = (a: Truth, b: Truth): Truth => ({ + requests: a.requests + b.requests, + tokens: a.tokens + b.tokens, + cost: a.cost + b.cost, + timed: a.timed + b.timed, + prompts: a.prompts + b.prompts, + tools: a.tools + b.tools, +}); +/** The machine's rollup totals, read inside it. */ +const truth = async (m: Machine): Promise => { + const [usage] = await m.sql(` + SELECT COALESCE(SUM(requests), 0) AS requests, COALESCE(SUM(input + output + cache_read + cache_write), 0) AS tokens, + COALESCE(SUM(cost), 0) AS cost, COALESCE(SUM(timed), 0) AS timed FROM usage + `); + const [tally] = await m.sql(` + SELECT COALESCE(SUM(CASE kind WHEN 'prompt' THEN count END), 0) AS prompts, + COALESCE(SUM(CASE kind WHEN 'tool' THEN count END), 0) AS tools FROM tally + `); + return { + requests: usage!.requests as number, + tokens: usage!.tokens as number, + cost: usage!.cost as number, + timed: usage!.timed as number, + prompts: tally!.prompts as number, + tools: tally!.tools as number, + }; +}; +const read = (request: Parameters[0]) => desk.run((rpc) => rpc["metrics.usage"](request)); +/** What a metrics read adds up to. */ +const shown = (metrics: MetricsUsage): Truth => ({ + requests: metrics.rows.reduce((sum, row) => sum + row.requests, 0), + tokens: metrics.rows.reduce( + (sum, row) => sum + row.tokens.input + row.tokens.output + row.tokens.cacheRead + row.tokens.cacheWrite, + 0, + ), + cost: metrics.rows.reduce((sum, row) => sum + row.cost.total, 0), + timed: metrics.rows.reduce((sum, row) => sum + row.timed, 0), + prompts: metrics.tallies.filter((t) => t.kind === "prompt").reduce((sum, t) => sum + t.count, 0), + tools: metrics.tallies.filter((t) => t.kind === "tool").reduce((sum, t) => sum + t.count, 0), +}); +const expectTotals = (actual: Truth, expected: Truth) => { + expect({ ...actual, cost: 0 }).toEqual({ ...expected, cost: 0 }); + expect(actual.cost).toBeCloseTo(expected.cost, 6); +}; +/** The app's metrics equal what the machines hold, in every zone and per machine; returns the totals. */ +const agrees = async (expected: ReadonlyArray) => { + const all = expected.reduce((sum, [, t]) => plus(sum, t), none); + for (const zone of ["UTC", "Asia/Kolkata", "Asia/Kathmandu", "America/Los_Angeles", "Pacific/Chatham"]) + expectTotals(shown(await read(everything(zone))), all); + for (const [m, t] of expected) + expectTotals(shown(await read({ ...everything(), machineId: typeof m === "string" ? m : idOf(m) })), t); + // The projects split the whole without a remainder. + const whole = await read(everything()); + let byProject = none; + for (const project of whole.projects) + byProject = plus(byProject, shown(await read({ ...everything(), projectId: project.id }))); + expectTotals(byProject, all); + return { all, sessions: whole.sessions.length, deleted: whole.sessions.filter((s) => s.deleted).length }; +}; + +const cases: Record = {}; +const describeMachines = available ? describe : describe.skip; + +describeMachines("metrics simulation across OrbStack machines", () => { + beforeAll(async () => { + for (const m of pool) await ensure(m.name); + await Promise.all(pool.map((m) => m.setup())); + const [a, b] = await Promise.all([seeded(cwA, 300, 365, 11), seeded(cwB, 120, 200, 23)]); + cases.seeded = { "cw-a": a, "cw-b": b }; + appHome = tempDir("metrics-simulation-app"); + desk = await app(appHome); + for (const m of pool) + ids.set(m.name, await desk.run((rpc) => rpc["servers.add"]({ kind: "remote", url: m.url }))); + }, 3_600_000); + + afterAll(async () => { + await Promise.all(pool.map((m) => m.net("restore").catch(() => undefined))); + await cleanup(); + }, 120_000); + + it("the first connect mirrors both histories, deleted sessions keeping their numbers", async () => { + for (const m of pool) await rolled(m); + const before = await agrees([ + [cwA, await truth(cwA)], + [cwB, await truth(cwB)], + ]); + const doomed = await cwA.sql("SELECT id FROM session ORDER BY id LIMIT 7"); + for (const { id } of doomed) + await cwA.run((rpc) => rpc["session.delete"]({ sessionId: Session.SessionSchema.ID.make(id as string) })); + await rolled(cwA); + const after = await agrees([ + [cwA, await truth(cwA)], + [cwB, await truth(cwB)], + ]); + expect(after.all).toEqual(before.all); + expect(after.deleted).toBe(before.deleted + 7); + cases.connected = { before, after }; + }, 600_000); + + it("live turns show while connected", async () => { + const at = lastSeq(appHome); + await work(cwA, 3); + await rolled(cwA); + const today = new Date().toISOString().slice(0, 10); + const day = await read({ zone: "UTC", since: today, until: today, machineId: idOf(cwA) }); + expect(shown(day).requests).toBeGreaterThanOrEqual(6); + expect(notices(appHome, Contract.syncTopics.usage, at).length).toBeGreaterThan(0); + cases.live = await agrees([ + [cwA, await truth(cwA)], + [cwB, await truth(cwB)], + ]); + }, 600_000); + + it("the app down while both machines work resumes from its cursors", async () => { + const cursors = pool.map((m) => cursorOf(appHome, idOf(m)).rollup); + await stop(desk.proc, "SIGTERM"); + await Promise.all([work(cwA, 2), work(cwB, 4)]); + const expected = await Promise.all(pool.map((m, at) => pending(m, cursors[at]!))); + const restartedAt = lastSeq(appHome); + desk = await app(appHome); + for (const m of pool) await rolled(m); + const applied = pool.map((m) => + notices(appHome, Contract.syncTopics.usage, restartedAt) + .filter((payload) => payload.machineId === idOf(m)) + .reduce((sum, payload) => sum + (payload.items as number), 0), + ); + expect(applied).toEqual(expected); + cases.resumed = { + applied, + totals: await agrees([ + [cwA, await truth(cwA)], + [cwB, await truth(cwB)], + ]), + }; + }, 600_000); + + it("a paused machine keeps its history on screen and resumes without a re-read", async () => { + const b = await truth(cwB); + await orb(["stop", cwB.name], { timeout: 300_000 }); + await work(cwA, 2); + await rolled(cwA); + const paused = await agrees([ + [cwA, await truth(cwA)], + [cwB, b], + ]); + const cursor = cursorOf(appHome, idOf(cwB)).rollup; + const at = lastSeq(appHome); + await orb(["start", cwB.name], { timeout: 300_000 }); + await cwB.start(live); + await rolled(cwB); + const reread = notices(appHome, Contract.syncTopics.usage, at).filter((p) => p.machineId === idOf(cwB)); + expect(reread.reduce((sum, p) => sum + (p.items as number), 0)).toBe(await pending(cwB, cursor)); + expect(await truth(cwB)).toEqual(b); + cases.paused = { + paused, + resumed: await agrees([ + [cwA, await truth(cwA)], + [cwB, b], + ]), + }; + }, 900_000); + + it("the app down across a machine's pause and restart, with work after it, resumes too", async () => { + await stop(desk.proc, "SIGTERM"); + const cursor = cursorOf(appHome, idOf(cwA)).rollup; + await orb(["stop", cwA.name], { timeout: 300_000 }); + await orb(["start", cwA.name], { timeout: 300_000 }); + await cwA.start(live); + await work(cwA, 2); + const expected = await pending(cwA, cursor); + const at = lastSeq(appHome); + desk = await app(appHome); + for (const m of pool) await rolled(m); + const applied = notices(appHome, Contract.syncTopics.usage, at) + .filter((p) => p.machineId === idOf(cwA)) + .reduce((sum, p) => sum + (p.items as number), 0); + expect(applied).toBe(expected); + cases.restartedAway = await agrees([ + [cwA, await truth(cwA)], + [cwB, await truth(cwB)], + ]); + }, 900_000); + + it("work during a network cut arrives once the network is back", async () => { + await cwA.net("cut"); + await cwA.drive([["session.create", { directory: `${cwA.cwHome}/work/infra`, title: "Offline work" }]]); + const [{ id }] = (await cwA.sql("SELECT id FROM session ORDER BY created_at DESC LIMIT 1")) as [Row]; + await cwA.drive([ + ["thread.prompt", { sessionId: id, text: "bash echo offline" }], + ["thread.wait", { sessionId: id }], + ]); + await cwA.net("restore"); + await rolled(cwA); + cases.cut = await agrees([ + [cwA, await truth(cwA)], + [cwB, await truth(cwB)], + ]); + }, 600_000); + + it("a backup restored after its head passed the app's cursor replaces that store's rows", async () => { + await cwA.stop("TERM"); + await cwA.sh(`cp ${cwA.database} ${cwA.cwHome}/backup.db`); + await cwA.start(live); + const lost = await work(cwA, 2); + const { rev: passed } = await rolled(cwA); + await cwA.stop("TERM"); + await cwA.sh( + `for f in ${cwA.database}-wal ${cwA.database}-shm; do [ -e $f ] && mv $f $f.lost; done; cp ${cwA.cwHome}/backup.db ${cwA.database}`, + ); + await cwA.start(live); + while ((await cwA.run((rpc) => rpc["server.info"]({}))).rev < passed) await work(cwA, 1); + // The index does not recover from this yet: its resume skips the revisions the restore rewrote + // and its batches fail on a thread whose session never came. Usage has its own stream. + await rolled(cwA, { index: false }); + const ghost = query(appHome, "SELECT COUNT(*) AS n FROM usage WHERE session_id = ?", lost.sessionId)[0]!.n; + expect(ghost).toBe(0); + cases.restored = await agrees([ + [cwA, await truth(cwA)], + [cwB, await truth(cwB)], + ]); + }, 900_000); + + it("a removed server's history stays; added again, it is counted once", async () => { + const b = await truth(cwB); + const a = await truth(cwA); + const removed = idOf(cwB); + await desk.run((rpc) => rpc["servers.remove"]({ id: removed })); + const kept = await agrees([ + [cwA, a], + [removed, b], + ]); + ids.set(cwB.name, await desk.run((rpc) => rpc["servers.add"]({ kind: "remote", url: cwB.url }))); + await rolled(cwB); + const again = await agrees([ + [cwA, a], + [cwB, b], + ]); + expect(again.all).toEqual(kept.all); + cases.readded = { kept, again }; + }, 900_000); + + it("a recreated store adds to the old one's history", async () => { + const old = await truth(cwB); + const a = await truth(cwA); + await aside(cwB); + await cwB.start(live); + await cwB.sh(`mkdir -p ${cwB.cwHome}/work/codework`); + await work(cwB, 2); + await rolled(cwB); + const fresh = await truth(cwB); + cases.recreated = await agrees([ + [cwA, a], + [cwB, plus(old, fresh)], + ]); + await snapshot("metrics.simulation", cases); + }, 900_000); +}); From ae6b8c6905ff06cab090e61caacfce14778f5c65 Mon Sep 17 00:00:00 2001 From: Sanchit Rk <0xsanchit@gmail.com> Date: Sat, 14 Nov 2026 16:27:40 +0530 Subject: [PATCH 8/8] chore: name the machine seeder as a knip entry The machine suites run it inside each OrbStack machine by path. Co-Authored-By: Claude Opus 5.5 --- knip.jsonc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/knip.jsonc b/knip.jsonc index bfd4cb69..764eaf82 100644 --- a/knip.jsonc +++ b/knip.jsonc @@ -32,8 +32,8 @@ }, }, "apps/server": { - // The machine simulation runs it on each OrbStack machine by path. - "entry": ["test/machines/drive.ts"], + // The machine suites run these on each OrbStack machine by path. + "entry": ["test/machines/drive.ts", "test/machines/seed.ts"], }, "packages/codework": { // bundle.mjs builds the SEA binary from it by path.