diff --git a/apps/contract/src/contract.ts b/apps/contract/src/contract.ts index f0198831..302d7283 100644 --- a/apps/contract/src/contract.ts +++ b/apps/contract/src/contract.ts @@ -17,6 +17,7 @@ import { Repository, RunState, SpaceInfo, + UsageItem, } from "@codeworksh/widget/codework"; import { Event, EventTimeout, Publish, Request, Synced } from "@codeworksh/widget/event"; @@ -373,11 +374,21 @@ export const ServerKind = Schema.Literals(["local", "remote"]); /** The server the app always has, which cannot be removed. */ export const LOCAL_SERVER = "local"; +/** How a server is named: the user's `override`, else the `host` name its server reports, else where it is. */ +const ServerNaming = { + /** As shown. */ + name: Schema.String, + /** As its server last reported it; none from a server that reports no name or was never reached. */ + host: Schema.NullOr(Schema.String), + /** As typed in Settings; none when the user named it nothing. */ + override: Schema.NullOr(Schema.String), +}; + /** A local server and how it is set up, with its executable and process as found when listed. */ export const LocalServerStatus = Schema.Struct({ id: Schema.String, kind: Schema.Literal("local"), - name: Schema.String, + ...ServerNaming, /** The one the app connects to. */ isDefault: Schema.Boolean, /** @@ -398,7 +409,7 @@ export type LocalServerStatus = typeof LocalServerStatus.Type; export const RemoteServerStatus = Schema.Struct({ id: Schema.String, kind: Schema.Literal("remote"), - name: Schema.String, + ...ServerNaming, isDefault: Schema.Boolean, url: Schema.String, }); @@ -421,23 +432,23 @@ export type CliAction = typeof CliAction.Type; /** * A new server. A local one needs a home of its own, since a home names one server; a remote one - * needs its WebSocket URL. + * needs its WebSocket URL. A blank or absent `name` is no override. */ export const ServerAdd = Schema.Union([ Schema.Struct({ kind: Schema.Literal("local"), - name: Schema.String, + name: Schema.optionalKey(Schema.String), executable: Schema.NullOr(Schema.String), home: Schema.String, args: Schema.String, }), - Schema.Struct({ kind: Schema.Literal("remote"), name: Schema.String, url: Schema.String }), + Schema.Struct({ kind: Schema.Literal("remote"), name: Schema.optionalKey(Schema.String), url: Schema.String }), ]); export type ServerAdd = typeof ServerAdd.Type; /** * Changes to a server; an absent field is left as it is. A `null` path goes back to the one found; - * `env` replaces every variable. `url` is for a remote server only, the rest but `name` for a local one. + * a blank `name` removes the override; `env` replaces every variable. `url` is for a remote server only, the rest but `name` for a local one. */ export const ServerUpdate = Schema.Struct({ id: Schema.String, @@ -468,6 +479,8 @@ export const syncTopics = { tracking: "sync/tracking", /** `SyncState` whenever a space daemon's mirrored state changed (git status, skills); stored. */ state: "sync/state", + /** `SyncUsage` after each applied usage batch that carried rows; stored. */ + usage: "sync/usage", }; /** @@ -499,6 +512,14 @@ export const SyncState = Schema.Struct({ }); export type SyncState = typeof SyncState.Type; +/** A machine's usage rows from one of its stores were mirrored anew; `items` counts them. */ +export const SyncUsage = Schema.Struct({ + machineId: Schema.String, + storeId: Schema.String, + items: Schema.Int, +}); +export type SyncUsage = typeof SyncUsage.Type; + /** A thread's status moved; `at` is the app server's clock. */ export const SyncStatus = Schema.Struct({ threadId: Schema.String, @@ -603,13 +624,123 @@ export type AllThreads = typeof AllThreads.Type; * `sync/delta/` streams its draft between checkpoints. * - `threads.list`: `sync/changed` naming a thread or a removed session; every `sync/status` and * `sync/tracking`. + * - `metrics.usage` and `metrics.hours`: every `sync/usage`, and every `sync/changed` for the session + * titles, deletions and project names they carry. */ export const readTopics = { sidebar: [syncTopics.changed, syncTopics.status, syncTopics.tracking], threads: [syncTopics.changed, syncTopics.status, syncTopics.tracking], line: (threadId: string) => [syncTopics.changed, syncTopics.delta(threadId)], + metrics: [syncTopics.usage, syncTopics.changed], +}; + +const [UsageWire, TallyWire] = UsageItem.members; + +/** A calendar day in the request's `zone`, `YYYY-MM-DD`. */ +const LocalDay = Schema.String; + +/** + * What the metrics reads cover: every machine and project unless one is named. `zone` is an IANA + * time zone; days and hours are its wall clock. The rollup's resolution is a UTC hour, which counts + * whole in the local hour holding its start: in a zone 30 or 45 minutes off the hour, all of it in + * the earlier one. A day a clock change shortens has 23 hours, one it lengthens 25, the repeated + * hour summed into one. + */ +const MetricsScope = { + zone: Schema.String, + machineId: Schema.optionalKey(Schema.String), + projectId: Schema.optionalKey(Schema.String), }; +/** Requests of one session on one local day to one provider, model and thinking level, summed over its threads. */ +export const MetricsRow = Schema.Struct({ + day: LocalDay, + machineId: Schema.String, + sessionId: Schema.String, + provider: Schema.String, + model: Schema.String, + thinking: Schema.String, + requests: Schema.Int, + tokens: UsageWire.fields.tokens, + cost: UsageWire.fields.cost, + duration: Schema.Finite, + timed: Schema.Int, + /** Over `edges`, as `UsageItem`'s. */ + durations: Schema.Array(Schema.Int), +}); +export type MetricsRow = typeof MetricsRow.Type; + +/** Prompts (`name` empty) or a tool's calls of one session on one local day. */ +export const MetricsTally = Schema.Struct({ + day: LocalDay, + machineId: Schema.String, + sessionId: Schema.String, + kind: TallyWire.fields.kind, + name: Schema.String, + count: Schema.Int, +}); +export type MetricsTally = typeof MetricsTally.Type; + +/** How many of a session's threads had requests or tallies on a local day. */ +export const MetricsThreads = Schema.Struct({ + day: LocalDay, + machineId: Schema.String, + sessionId: Schema.String, + threads: Schema.Int, +}); +export type MetricsThreads = typeof MetricsThreads.Type; + +/** + * A session the rows name. `title` is its main thread's, or what an untitled one is called; null + * once the session is deleted or gone from the mirror, its rows kept. Which sandbox it ran in is + * not mirrored, so it is not here. + */ +export const MetricsSession = Schema.Struct({ + id: Schema.String, + machineId: Schema.String, + /** As its rows recorded it. */ + projectId: Schema.String, + title: Schema.NullOr(Schema.String), + deleted: Schema.Boolean, +}); +export type MetricsSession = typeof MetricsSession.Type; + +/** A period's usage at the grain the Metrics page slices, with the names it shows. */ +export const MetricsUsage = Schema.Struct({ + rows: Schema.Array(MetricsRow), + tallies: Schema.Array(MetricsTally), + threads: Schema.Array(MetricsThreads), + sessions: Schema.Array(MetricsSession), + /** + * A project id is its repo's, so one on several machines is listed once, named as on the machine + * that changed it last; null when no machine has it any more. + */ + projects: Schema.Array(Schema.Struct({ id: Schema.String, name: Schema.NullOr(Schema.String) })), + machines: Schema.Array(Schema.Struct({ id: Schema.String, name: Schema.String })), + /** The upper bounds in ms of `durations`' buckets. */ + edges: Schema.Array(Schema.Int), +}); +export type MetricsUsage = typeof MetricsUsage.Type; + +/** Requests of one session in one local hour (0–23) to one provider and model. */ +export const MetricsHour = Schema.Struct({ + hour: Schema.Int, + machineId: Schema.String, + sessionId: Schema.String, + provider: Schema.String, + model: Schema.String, + requests: Schema.Int, + /** Input, output, cache read and cache write. */ + tokens: Schema.Int, + cost: Schema.Finite, +}); +export type MetricsHour = typeof MetricsHour.Type; + +/** A metrics read refused: an unknown time zone, a malformed day, or a period ending before it starts. */ +export class MetricsError extends Schema.TaggedError()("Server.MetricsError", { + message: Schema.String, +}) {} + /** A desktop edit refused: a value out of range, or a thread, session, group or organization not found. */ export class TrackingError extends Schema.TaggedError()("Server.TrackingError", { message: Schema.String, @@ -668,6 +799,18 @@ export const Api = RpcGroup.make( payload: { organizationId: Schema.String, by: ThreadsBy }, success: AllThreads, }), + /** Usage from `since` to `until`, both local days included, summed over threads and stores. Read from app.db alone. */ + Rpc.make("metrics.usage", { + payload: { ...MetricsScope, since: LocalDay, until: LocalDay }, + success: MetricsUsage, + error: MetricsError, + }), + /** One local day's usage by local hour, for its chart; its sessions and threads are in `metrics.usage`. Read from app.db alone. */ + Rpc.make("metrics.hours", { + payload: { ...MetricsScope, day: LocalDay }, + success: Schema.Array(MetricsHour), + error: MetricsError, + }), /** The models a new session can use; `hostDir` is its folder, whose project settings apply. */ Rpc.make("models.list", { payload: { hostDir: Schema.optionalKey(Schema.String) }, diff --git a/apps/server/src/cli.ts b/apps/server/src/cli.ts index 4da40b21..f6b72522 100644 --- a/apps/server/src/cli.ts +++ b/apps/server/src/cli.ts @@ -171,6 +171,8 @@ export const statuses = Effect.gen(function* () { id: row.id, kind: row.kind, name: row.name, + host: row.host, + override: row.override, isDefault: row.isDefault, home, cli, diff --git a/apps/server/src/db.ts b/apps/server/src/db.ts index 28f0d323..27997f6a 100644 --- a/apps/server/src/db.ts +++ b/apps/server/src/db.ts @@ -274,6 +274,75 @@ const migrations = { // The one organization the UI uses until auth exists. yield* sql`INSERT INTO organization ${sql.insert({ id: "personal", name: "Personal", createdAt: now, updatedAt: now })}`; }), + + // Each machine's usage rollup, as `usage.sync` sends it, keyed by store: a recreated store starts + // new rows and keeps the old ones as history, and a removed server re-added later replaces its own. + // No foreign keys: the rows outlive their machine, project, session and thread; `machine_id` is the + // machine that last sent them. + "202611140001_usage": Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + // `rollup` is the usage cursor, counted in `rollup_store` while `rollup_boot` was its latest open: + // it moves apart from `cursor` and `store_id`. + yield* sql`ALTER TABLE machine ADD COLUMN rollup INTEGER NOT NULL DEFAULT 0`; + yield* sql`ALTER TABLE machine ADD COLUMN rollup_store TEXT`; + yield* sql`ALTER TABLE machine ADD COLUMN rollup_boot TEXT`; + yield* sql` + CREATE TABLE usage ( + machine_id TEXT NOT NULL, + store_id TEXT NOT NULL, + hour INTEGER NOT NULL, + thread_id TEXT NOT NULL, + provider TEXT NOT NULL, + model TEXT NOT NULL, + thinking TEXT NOT NULL, + session_id TEXT NOT NULL, + project_id TEXT NOT NULL, + requests INTEGER NOT NULL, + input INTEGER NOT NULL, + output INTEGER NOT NULL, + cache_read INTEGER NOT NULL, + cache_write INTEGER NOT NULL, + reasoning INTEGER NOT NULL, + cost REAL NOT NULL, + cost_input REAL NOT NULL, + cost_output REAL NOT NULL, + cost_cache_read REAL NOT NULL, + cost_cache_write REAL NOT NULL, + duration REAL NOT NULL, + timed INTEGER NOT NULL, + durations TEXT NOT NULL, + rev INTEGER NOT NULL, + PRIMARY KEY (store_id, hour, thread_id, provider, model, thinking) + ) + `; + yield* sql` + CREATE TABLE tally ( + machine_id TEXT NOT NULL, + store_id TEXT NOT NULL, + hour INTEGER NOT NULL, + thread_id TEXT NOT NULL, + kind TEXT NOT NULL, + name TEXT NOT NULL, + session_id TEXT NOT NULL, + project_id TEXT NOT NULL, + count INTEGER NOT NULL, + rev INTEGER NOT NULL, + PRIMARY KEY (store_id, hour, thread_id, kind, name) + ) + `; + }), + + // A machine is named by what its server reports (`host`); `name` is only the user's override, so + // it becomes nullable. ADD + DROP + RENAME instead of rebuilding the table, which its foreign keys + // would cascade into. The built-in server's old seed name is no override. + "202611140002_host": Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* sql`ALTER TABLE machine ADD COLUMN host TEXT`; + yield* sql`ALTER TABLE machine ADD COLUMN alias TEXT`; + yield* sql`UPDATE machine SET alias = name WHERE NOT (id = 'local' AND name = 'CodeWork')`; + yield* sql`ALTER TABLE machine DROP COLUMN name`; + yield* sql`ALTER TABLE machine RENAME COLUMN alias TO name`; + }), }; const setup = Effect.gen(function* () { diff --git a/apps/server/src/handlers.ts b/apps/server/src/handlers.ts index 52e24bdb..3c41f6cb 100644 --- a/apps/server/src/handlers.ts +++ b/apps/server/src/handlers.ts @@ -18,6 +18,7 @@ import { Desk } from "./desk.ts"; import { Event } from "./event.ts"; import { Health } from "./health.ts"; import { Kv } from "./kv.ts"; +import { Metrics } from "./metrics.ts"; import { Reads } from "./reads.ts"; import { Skills } from "./skills.ts"; import { Servers } from "./servers.ts"; @@ -106,6 +107,8 @@ export const layer = Contract.Api.toLayer( "changes.diff": (input) => Changes.diff(input, sync.client(input.machineId)), "threads.list": (input) => Reads.threads(input).pipe(Effect.provideService(SqlClient.SqlClient, sql)), "threads.line": (input) => Reads.line(input).pipe(Effect.provideService(SqlClient.SqlClient, sql)), + "metrics.usage": (input) => Metrics.usage(input).pipe(Effect.provideService(SqlClient.SqlClient, sql)), + "metrics.hours": (input) => Metrics.hours(input).pipe(Effect.provideService(SqlClient.SqlClient, sql)), "host.info": Effect.fn("host.info")(function* () { return yield* call((client) => client("host.info", {})); diff --git a/apps/server/src/health.ts b/apps/server/src/health.ts index 7863f51c..1ca12248 100644 --- a/apps/server/src/health.ts +++ b/apps/server/src/health.ts @@ -71,6 +71,16 @@ export const layer = Layer.effect( yield* check.pipe(Effect.forkScoped); + // A name a server reports reaches Settings without waiting for the next check; the app's own + // edits are checked by their handlers. + yield* servers.watch.pipe( + Stream.map((rows) => rows.map((row) => `${row.id}=${row.host}`).join("\n")), + Stream.changes, + Stream.drop(1), + Stream.runForEach(() => check), + Effect.forkScoped, + ); + // Every interval; a new interval starts counting at once, and 0 waits for one. yield* Effect.forever( Effect.gen(function* () { diff --git a/apps/server/src/metrics.ts b/apps/server/src/metrics.ts new file mode 100644 index 00000000..8dbd37e4 --- /dev/null +++ b/apps/server/src/metrics.ts @@ -0,0 +1,336 @@ +import { Effect, Schema } from "effect"; +import { SqlClient } from "effect/sql"; +import { + type MetricsHour, + MetricsError, + type MetricsRow, + type MetricsSession, + type MetricsTally, + type MetricsUsage, +} from "@codeworksh/contract"; +import { edges, UNTITLED } from "@codeworksh/widget/codework"; +import { Reads } from "./reads.ts"; + +/* + * The Metrics page's reads, from the mirrored usage rollup. SQL sums each UTC hour over threads and + * stores; the hours are then bucketed into `zone`'s days and hours here, as SQLite knows no zones. + */ + +const HOUR = 3_600_000; +const DAY = 24 * HOUR; +/** Past the widest UTC offsets (−12 and +14 hours): every UTC hour that can start on a requested local day. */ +const MARGIN = 15 * HOUR; + +interface Scope { + readonly zone: string; + readonly machineId?: string; + readonly projectId?: string; +} + +/** A day's UTC midnight; a malformed or impossible day (`2026-02-30`) is refused. */ +const midnight = (day: string) => { + const parts = /^(\d{4})-(\d{2})-(\d{2})$/.exec(day); + const at = parts === null ? Number.NaN : Date.UTC(Number(parts[1]), Number(parts[2]) - 1, Number(parts[3])); + return Number.isNaN(at) || new Date(at).toISOString().slice(0, 10) !== day + ? Effect.fail(new MetricsError({ message: `"${day}" is not a day: expected YYYY-MM-DD.` })) + : Effect.succeed(at); +}; + +/** Each UTC hour's local day and hour in `zone`; each hour is formatted once. */ +const clock = (zone: string) => + Effect.try({ + try: () => + new Intl.DateTimeFormat("en-US", { + timeZone: zone, + year: "numeric", + month: "2-digit", + day: "2-digit", + hour: "2-digit", + hourCycle: "h23", + }), + catch: () => new MetricsError({ message: `"${zone}" is not a time zone.` }), + }).pipe( + Effect.map((format) => { + const seen = new Map(); + return (hour: number) => { + const known = seen.get(hour); + if (known !== undefined) return known; + const part = Object.fromEntries(format.formatToParts(hour).map(({ type, value }) => [type, value])); + const local = { day: `${part.year}-${part.month}-${part.day}`, hour: Number(part.hour) }; + seen.set(hour, local); + return local; + }; + }), + ); + +/** The local days `since` to `until` in `zone`, and the UTC hours to read for them. */ +const period = Effect.fn(function* (zone: string, since: string, until: string) { + const local = yield* clock(zone); + const from = yield* midnight(since); + const to = yield* midnight(until); + if (to < from) + return yield* new MetricsError({ message: `The period ends (${until}) before it starts (${since}).` }); + return { local, from: from - MARGIN, to: to + DAY + MARGIN }; +}); + +const where = (sql: SqlClient.SqlClient, scope: Scope, from: number, to: number) => + sql.and([ + sql`hour >= ${from}`, + sql`hour < ${to}`, + ...(scope.machineId === undefined ? [] : [sql`machine_id = ${scope.machineId}`]), + ...(scope.projectId === undefined ? [] : [sql`project_id = ${scope.projectId}`]), + ]); + +const decodeHistograms = Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Array(Schema.Array(Schema.Int)))); + +/** Each session's usage per UTC hour, provider, model and thinking level, summed over its threads and stores. */ +const hourly = (sql: SqlClient.SqlClient, scope: Scope, from: number, to: number) => + sql<{ + machineId: string; + hour: number; + sessionId: string; + provider: string; + model: string; + thinking: string; + requests: number; + input: number; + output: number; + cacheRead: number; + cacheWrite: number; + reasoning: number; + cost: number; + costInput: number; + costOutput: number; + costCacheRead: number; + costCacheWrite: number; + duration: number; + timed: number; + durations: string; + }>` + SELECT machine_id, hour, session_id, provider, model, thinking, SUM(requests) AS requests, + SUM(input) AS input, SUM(output) AS output, SUM(cache_read) AS cache_read, SUM(cache_write) AS cache_write, + SUM(reasoning) AS reasoning, SUM(cost) AS cost, SUM(cost_input) AS cost_input, SUM(cost_output) AS cost_output, + SUM(cost_cache_read) AS cost_cache_read, SUM(cost_cache_write) AS cost_cache_write, + SUM(duration) AS duration, SUM(timed) AS timed, json_group_array(json(durations)) AS durations + FROM usage WHERE ${where(sql, scope, from, to)} + GROUP BY machine_id, hour, session_id, provider, model, thinking + `; + +/** Adds `value` into the entry at `key`, starting from `value` itself. */ +const add = (into: Map, key: ReadonlyArray, value: A, plus: (sum: A, value: A) => A) => { + const id = key.join("\u0000"); + const sum = into.get(id); + into.set(id, sum === undefined ? value : plus(sum, value)); +}; +/** Entries by key, so a read answers in the same order each time. */ +const ordered = (map: Map) => + [...map].toSorted(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)).map(([, value]) => value); + +const plusRow = (sum: MetricsRow, row: MetricsRow): MetricsRow => ({ + ...sum, + requests: sum.requests + row.requests, + tokens: { + input: sum.tokens.input + row.tokens.input, + output: sum.tokens.output + row.tokens.output, + cacheRead: sum.tokens.cacheRead + row.tokens.cacheRead, + cacheWrite: sum.tokens.cacheWrite + row.tokens.cacheWrite, + reasoning: sum.tokens.reasoning + row.tokens.reasoning, + }, + cost: { + total: sum.cost.total + row.cost.total, + input: sum.cost.input + row.cost.input, + output: sum.cost.output + row.cost.output, + cacheRead: sum.cost.cacheRead + row.cost.cacheRead, + cacheWrite: sum.cost.cacheWrite + row.cost.cacheWrite, + }, + duration: sum.duration + row.duration, + timed: sum.timed + row.timed, + durations: sum.durations.map((count, bucket) => count + (row.durations[bucket] ?? 0)), +}); + +export const usage = Effect.fn("Metrics.usage")(function* ( + request: Scope & { readonly since: string; readonly until: string }, +) { + const { local, from, to } = yield* period(request.zone, request.since, request.until); + const inPeriod = (hour: number) => { + const { day } = local(hour); + return day >= request.since && day <= request.until ? day : undefined; + }; + return yield* Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const filter = where(sql, request, from, to); + + const rows = new Map(); + for (const row of yield* hourly(sql, request, from, to)) { + const day = inPeriod(row.hour); + if (day === undefined) continue; + const empty = Array.from({ length: edges.length + 1 }, () => 0); + const durations = (yield* decodeHistograms(row.durations)).reduce( + (sum, list) => sum.map((count, bucket) => count + (list[bucket] ?? 0)), + empty, + ); + const key = [day, row.machineId, row.sessionId, row.provider, row.model, row.thinking]; + add( + rows, + key, + { + day, + machineId: row.machineId, + sessionId: row.sessionId, + provider: row.provider, + model: row.model, + thinking: row.thinking, + requests: row.requests, + tokens: { + input: row.input, + output: row.output, + cacheRead: row.cacheRead, + cacheWrite: row.cacheWrite, + reasoning: row.reasoning, + }, + cost: { + total: row.cost, + input: row.costInput, + output: row.costOutput, + cacheRead: row.costCacheRead, + cacheWrite: row.costCacheWrite, + }, + duration: row.duration, + timed: row.timed, + durations, + }, + plusRow, + ); + } + + const tallies = new Map(); + const counted = yield* sql<{ + machineId: string; + hour: number; + sessionId: string; + kind: MetricsTally["kind"]; + name: string; + count: number; + }>` + SELECT machine_id, hour, session_id, kind, name, SUM(count) AS count FROM tally WHERE ${filter} + GROUP BY machine_id, hour, session_id, kind, name + `; + for (const row of counted) { + const day = inPeriod(row.hour); + if (day === undefined) continue; + const { machineId, sessionId, kind, name, count } = row; + const tally = { day, machineId, sessionId, kind, name, count }; + add(tallies, [day, machineId, sessionId, kind, name], tally, (sum, next) => ({ + ...sum, + count: sum.count + next.count, + })); + } + + // Threads are summed away above, so the ones each session worked on a day are counted apart. + const worked = yield* sql<{ + machineId: string; + hour: number; + sessionId: string; + threadId: string; + projectId: string; + }>` + SELECT machine_id, hour, session_id, thread_id, project_id FROM usage WHERE ${filter} + UNION SELECT machine_id, hour, session_id, thread_id, project_id FROM tally WHERE ${filter} + `; + const threads = new Map }>(); + const sessions = new Map(); + for (const row of worked) { + const day = inPeriod(row.hour); + if (day === undefined) continue; + const ids = new Set([row.threadId]); + add( + threads, + [day, row.machineId, row.sessionId], + { day, machineId: row.machineId, sessionId: row.sessionId, ids }, + (sum, next) => { + for (const id of next.ids) sum.ids.add(id); + return sum; + }, + ); + sessions.set(row.sessionId, { id: row.sessionId, machineId: row.machineId, projectId: row.projectId }); + } + + const ids = [...sessions.keys()]; + const projectIds = [...new Set([...sessions.values()].map((session) => session.projectId))]; + const machineIds = [...new Set([...sessions.values()].map((session) => session.machineId))]; + const named = + ids.length === 0 + ? [] + : yield* sql<{ id: string; deletedAt: number | null; title: string | null }>` + SELECT session.id, session.deleted_at, thread.title FROM session + LEFT JOIN thread ON thread.session_id = session.id AND thread.origin_kind = 'main' + WHERE session.id IN ${sql.in(ids)} + `; + const titles = new Map(named.map((row) => [row.id, row])); + const projects = + projectIds.length === 0 + ? [] + : yield* sql<{ id: string; name: string }>` + SELECT id, name FROM project WHERE id IN ${sql.in(projectIds)} ORDER BY updated_at DESC, machine_id + `; + const projectNames = new Map(); + for (const project of projects) if (!projectNames.has(project.id)) projectNames.set(project.id, project.name); + const machines = + machineIds.length === 0 + ? [] + : yield* sql` + SELECT machine.id, ${sql.literal(Reads.MACHINE)} FROM machine WHERE machine.id IN ${sql.in(machineIds)} + ORDER BY machine.id + `; + + return { + rows: ordered(rows), + tallies: ordered(tallies), + threads: ordered(threads).map(({ ids, ...row }) => ({ ...row, threads: ids.size })), + sessions: ordered(sessions).map((owner): MetricsSession => { + const session = titles.get(owner.id); + const deleted = session === undefined || session.deletedAt !== null; + return { ...owner, title: deleted ? null : (session.title ?? UNTITLED), deleted }; + }), + projects: projectIds.toSorted().map((id) => ({ id, name: projectNames.get(id) ?? null })), + machines: machines.map((machine) => ({ id: machine.id, name: Reads.machineName(machine) })), + edges: [...edges], + } satisfies MetricsUsage; + }).pipe(Effect.orDie); +}); + +export const hours = Effect.fn("Metrics.hours")(function* (request: Scope & { readonly day: string }) { + const { local, from, to } = yield* period(request.zone, request.day, request.day); + return yield* Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const hours = new Map(); + for (const row of yield* hourly(sql, request, from, to)) { + const { day, hour } = local(row.hour); + if (day !== request.day) continue; + // A clock change's repeated hour is one wall-clock hour: both of its UTC hours sum into it. + add( + hours, + [String(hour).padStart(2, "0"), row.machineId, row.sessionId, row.provider, row.model], + { + hour, + machineId: row.machineId, + sessionId: row.sessionId, + provider: row.provider, + model: row.model, + requests: row.requests, + tokens: row.input + row.output + row.cacheRead + row.cacheWrite, + cost: row.cost, + }, + (sum, next) => ({ + ...sum, + requests: sum.requests + next.requests, + tokens: sum.tokens + next.tokens, + cost: sum.cost + next.cost, + }), + ); + } + return ordered(hours); + }).pipe(Effect.orDie); +}); + +export * as Metrics from "./metrics.ts"; diff --git a/apps/server/src/mirror.ts b/apps/server/src/mirror.ts index 4a95ee5b..102fb619 100644 --- a/apps/server/src/mirror.ts +++ b/apps/server/src/mirror.ts @@ -18,6 +18,7 @@ import { type IndexRow = Exclude; type ThreadRow = Exclude; +type UsageRow = Exclude; type Append = (input: Publish) => Effect.Effect; /** The rows a batch wrote, for its `SyncChanged` notice. */ @@ -274,6 +275,51 @@ export const unseen = (machineId: string, seen: Changes, now: number) => `; }); +/** Applies one `usage.sync` batch of a machine's store: each row replaces the one at its key, running totals included. */ +export const usage = (machineId: string, storeId: string, rows: ReadonlyArray) => + Effect.forEach( + rows, + (row) => { + const owner = { + machineId, + storeId, + hour: row.hour, + threadId: row.threadId, + sessionId: row.sessionId, + projectId: row.projectId, + rev: row.rev, + }; + if (row._tag === "Tally") + return upsert("tally", ["store_id", "hour", "thread_id", "kind", "name"], { + ...owner, + kind: row.kind, + name: row.name, + count: row.count, + }); + return upsert("usage", ["store_id", "hour", "thread_id", "provider", "model", "thinking"], { + ...owner, + provider: row.provider, + model: row.model, + thinking: row.thinking, + requests: row.requests, + input: row.tokens.input, + output: row.tokens.output, + cacheRead: row.tokens.cacheRead, + cacheWrite: row.tokens.cacheWrite, + reasoning: row.tokens.reasoning, + cost: row.cost.total, + costInput: row.cost.input, + costOutput: row.cost.output, + costCacheRead: row.cost.cacheRead, + costCacheWrite: row.cost.cacheWrite, + duration: row.duration, + timed: row.timed, + durations: JSON.stringify(row.durations), + }); + }, + { discard: true }, + ); + /** Stores entries of a thread's line, each under the thread that wrote it, with its parts. */ export const entries = (list: ReadonlyArray, changes: Changes) => Effect.gen(function* () { diff --git a/apps/server/src/reads.ts b/apps/server/src/reads.ts index b411a0cc..2b4f68f1 100644 --- a/apps/server/src/reads.ts +++ b/apps/server/src/reads.ts @@ -17,6 +17,7 @@ import { type ThreadStatus, } from "@codeworksh/contract"; import { HistoryEntry, Part, QueuedPrompt, RunError, UNTITLED } from "@codeworksh/widget/codework"; +import { Servers } from "./servers.ts"; /* * What the UI reads from app.db: the sidebar and a thread's line. Neither calls CodeWork, so both @@ -35,6 +36,18 @@ const decodeThread = Schema.decodeUnknownEffect(LineThread); const model = (provider: string | null, id: string | null) => provider === null || id === null ? {} : { model: { provider, id } }; +/** The machine columns a read selects to name it by {@link Servers.nameOf}. */ +export const MACHINE = + "machine.kind AS machine_kind, machine.name AS machine_override, machine.host AS machine_host, machine.url AS machine_url"; +export interface MachineColumns { + readonly machineKind: string; + readonly machineOverride: string | null; + readonly machineHost: string | null; + readonly machineUrl: string | null; +} +export const machineName = (row: MachineColumns) => + Servers.nameOf({ kind: row.machineKind, override: row.machineOverride, host: row.machineHost, url: row.machineUrl }); + /** The machine filter of `sidebar.list`. */ const onMachines = (sql: SqlClient.SqlClient, machines: "default" | "all") => machines === "all" ? sql.literal("") : sql`AND machine.is_default = 1`; @@ -47,16 +60,17 @@ export const sidebar = Effect.fn("Reads.sidebar")(function* ({ readonly machines: "default" | "all"; }) { const sql = yield* SqlClient.SqlClient; - const projects = yield* sql<{ - machineId: string; - machineName: string; - id: string; - name: string; - location: string | null; - createdAt: number; - updatedAt: number; - }>` - SELECT project.machine_id, machine.name AS machine_name, project.id, project.name, project.created_at, project.updated_at, + const projects = yield* sql< + MachineColumns & { + machineId: string; + id: string; + name: string; + location: string | null; + createdAt: number; + updatedAt: number; + } + >` + SELECT project.machine_id, ${sql.literal(MACHINE)}, project.id, project.name, project.created_at, project.updated_at, (SELECT location FROM space WHERE space.machine_id = project.machine_id AND space.project_id = project.id ORDER BY space.kind = 'primary' DESC, space.created_at LIMIT 1) AS location FROM project JOIN machine ON machine.id = project.machine_id @@ -122,7 +136,7 @@ export const sidebar = Effect.fn("Reads.sidebar")(function* ({ .map((project): SidebarProject => { const list = sessionsOf(project); return { - machine: { id: project.machineId, name: project.machineName }, + machine: { id: project.machineId, name: machineName(project) }, id: project.id, name: project.name, // A worktree session's folder is not the project's, so new threads start in the primary space. @@ -177,26 +191,27 @@ export const threads = Effect.fn("Reads.threads")(function* ({ readonly by: ThreadsBy; }) { const sql = yield* SqlClient.SqlClient; - const rows = yield* sql<{ - id: string; - sessionId: string; - originKind: ListedThread["originKind"]; - originThreadId: string | null; - title: string | null; - machineId: string; - machineName: string; - projectId: string; - projectName: string; - runState: ListedThread["runState"]; - status: ThreadStatus; - priority: Priority; - groupingId: string | null; - ordering: number; - color: string | null; - updatedAt: number; - }>` + const rows = yield* sql< + MachineColumns & { + id: string; + sessionId: string; + originKind: ListedThread["originKind"]; + originThreadId: string | null; + title: string | null; + machineId: string; + projectId: string; + projectName: string; + runState: ListedThread["runState"]; + status: ThreadStatus; + priority: Priority; + groupingId: string | null; + ordering: number; + color: string | null; + updatedAt: number; + } + >` SELECT thread.id, thread.session_id, thread.origin_kind, thread.origin_thread_id, thread.title, - machine.id AS machine_id, machine.name AS machine_name, project.id AS project_id, project.name AS project_name, thread.run_state, thread.status, thread.priority, + machine.id AS machine_id, ${sql.literal(MACHINE)}, project.id AS project_id, project.name AS project_name, thread.run_state, thread.status, thread.priority, thread.grouping_id, thread.ordering, COALESCE(thread.color, session.color) AS color, thread.updated_at FROM thread JOIN session ON session.id = thread.session_id @@ -217,7 +232,7 @@ export const threads = Effect.fn("Reads.threads")(function* ({ originKind: row.originKind, originThreadId: row.originThreadId, title: titleOf(row) ?? UNTITLED, - machine: { id: row.machineId, name: row.machineName }, + machine: { id: row.machineId, name: machineName(row) }, project: { id: row.projectId, name: row.projectName }, runState: row.runState, status: row.status, diff --git a/apps/server/src/servers.ts b/apps/server/src/servers.ts index a644122e..8d54b3c2 100644 --- a/apps/server/src/servers.ts +++ b/apps/server/src/servers.ts @@ -20,12 +20,37 @@ export type StoredEnv = typeof StoredEnv.Type; export const splitArgs = (input: string) => [...input.matchAll(/"([^"]*)"|'([^']*)'|(\S+)/g)].map((match) => match[1] ?? match[2] ?? match[3] ?? ""); +/** + * What a machine is called: the user's override, else the name its server reports, else where it is. + * The one rule every machine name shown goes through. + */ +export const nameOf = (machine: { + readonly kind: string; + readonly override: string | null; + readonly host: string | null; + readonly url: string | null; +}) => + machine.override ?? + machine.host ?? + (machine.kind === "remote" && machine.url !== null && URL.canParse(machine.url) + ? new URL(machine.url).host + : "This Machine"); + +/** How a machine is named, as stored and as shown. */ +interface Naming { + /** As shown: {@link nameOf}. */ + readonly name: string; + /** As its server last reported it. */ + readonly host: string | null; + /** As the user named it in Settings. */ + readonly override: string | null; +} + /** A local server as stored. A `null` executable or home is the one found. An executable is a path, or a * command whose first word is one (`node --conditions=development …/index.ts`, running from source). */ -export interface LocalRow { +export interface LocalRow extends Naming { readonly id: string; readonly kind: "local"; - readonly name: string; readonly executable: string | null; readonly home: string | null; readonly args: string; @@ -34,10 +59,9 @@ export interface LocalRow { } /** A remote server as stored: where the app reaches it. */ -export interface RemoteRow { +export interface RemoteRow extends Naming { readonly id: string; readonly kind: "remote"; - readonly name: string; readonly url: string; readonly isDefault: boolean; } @@ -136,6 +160,8 @@ export class Service extends Context.Service< readonly homeChanges: Stream.Stream; /** Every server now, then again after each change to any of them. */ readonly watch: Stream.Stream>; + /** Stores the name a server reports, when it is not the one stored. */ + readonly setHost: (id: string, host: string | null) => Effect.Effect; readonly add: (input: ServerAdd) => Effect.Effect; readonly update: (input: ServerUpdate) => Effect.Effect; readonly remove: (id: string) => Effect.Effect; @@ -151,23 +177,30 @@ export const layer = Layer.effect( const fallback = yield* cliHome.pipe(Effect.orDie); const list = sql< - Omit & { + Omit & { kind: string; env: string; url: string | null; isDefault: number; } >` - SELECT id, kind, name, executable, home, args, env, url, is_default FROM machine ORDER BY pos + SELECT id, kind, name AS override, host, executable, home, args, env, url, is_default FROM machine ORDER BY pos `.pipe( Effect.flatMap((rows) => - Effect.forEach(rows, ({ kind, url, env, isDefault, ...row }) => - kind === "remote" && url !== null - ? Effect.succeed({ id: row.id, kind, name: row.name, url, isDefault: isDefault === 1 }) + Effect.forEach(rows, ({ kind, url, env, isDefault, ...row }) => { + const naming = { name: nameOf({ kind, url, ...row }), host: row.host, override: row.override }; + return kind === "remote" && url !== null + ? Effect.succeed({ id: row.id, kind, ...naming, url, isDefault: isDefault === 1 }) : decodeEnv(env).pipe( - Effect.map((env): Row => ({ ...row, kind: "local", env, isDefault: isDefault === 1 })), - ), - ), + Effect.map((env): Row => ({ + ...row, + ...naming, + kind: "local", + env, + isDefault: isDefault === 1, + })), + ); + }), ), Effect.orDie, ); @@ -218,9 +251,10 @@ export const layer = Layer.effect( }), ); - const name = (input: string) => { - const trimmed = input.trim(); - return trimmed === "" ? Effect.fail(fail("A server needs a name.")) : Effect.succeed(trimmed); + /** A name typed in Settings; blank is no override. */ + const override = (input: string | undefined) => { + const trimmed = input?.trim() ?? ""; + return trimmed === "" ? null : trimmed; }; const transaction = (effect: Effect.Effect) => @@ -234,6 +268,12 @@ export const layer = Layer.effect( defaultHome, homeChanges: SubscriptionRef.changes(current).pipe(Stream.drop(1)), watch: SubscriptionRef.changes(machines), + setHost: Effect.fn("Servers.setHost")(function* (id, host) { + const changed = yield* sql` + UPDATE machine SET host = ${host} WHERE id = ${id} AND host IS NOT ${host} RETURNING id + `.pipe(Effect.orDie); + if (changed.length > 0) yield* announce; + }), add: Effect.fn("Servers.add")(function* (input) { const id = yield* Effect.sync(() => crypto.randomUUID().slice(0, 8)); @@ -250,7 +290,7 @@ export const layer = Layer.effect( yield* sql`INSERT INTO machine ${sql.insert({ id, kind: input.kind, - name: yield* name(input.name), + name: override(input.name), ...route, namespace: "codework", pos: (last?.pos ?? 0) + 1, @@ -267,7 +307,7 @@ export const layer = Layer.effect( Effect.gen(function* () { const row = yield* get(id); const fields: Record = {}; - if (patch.name !== undefined) fields.name = yield* name(patch.name); + if (patch.name !== undefined) fields.name = override(patch.name); if (row.kind === "remote") { if ([patch.executable, patch.home, patch.args, patch.env].some((value) => value !== undefined)) return yield* fail("A remote server has no executable, home, arguments or variables."); @@ -292,7 +332,7 @@ export const layer = Layer.effect( yield* announce; }), - // Everything synced from the server goes with it: its rows cascade. + // What was synced from the server goes with it, its rows cascading, except its usage: history. remove: Effect.fn("Servers.remove")(function* (id) { const row = yield* get(id); if (row.id === LOCAL_SERVER) return yield* fail("The built-in local server cannot be removed."); diff --git a/apps/server/src/sync.ts b/apps/server/src/sync.ts index eb12c276..404000eb 100644 --- a/apps/server/src/sync.ts +++ b/apps/server/src/sync.ts @@ -34,6 +34,12 @@ type IndexSynced = Extract; type IndexRow = Exclude; type ThreadSynced = Extract; type ThreadRow = Exclude; +type UsageSynced = Extract; +type UsageRow = Exclude; + +/** The RPC server's answer to a call it has no handler for: a server older than `usage.sync`. */ +const unserved = (cause: Cause.Cause) => + cause.reasons.some((reason) => Cause.isDieReason(reason) && reason.defect === "Unknown request tag: usage.sync"); /** A machine whose index is current: its connection and the threads whose detail it syncs. */ interface Live { @@ -62,7 +68,7 @@ export interface Interface { /** * Keeps app.db a copy of every machine's CodeWork: each machine over its own connection, its index - * through `index.sync` from `machine.cursor`, and the detail of opened threads through `thread.sync` + * through `index.sync` from `machine.cursor`, its usage rollup through `usage.sync` from `machine.rollup`, and the detail of opened threads through `thread.sync` * from `thread.cursor`. A new or restored store is read again from zero. */ export class Service extends Context.Service()("@codeworksh/server/sync/Service") {} @@ -303,23 +309,29 @@ export const layer = Layer.effect( Effect.forever, ); + /** Waits forever when another machine already syncs `storeId`: the same server added twice. */ + const unclaimed = (machineId: string, storeId: string) => + Effect.gen(function* () { + const [owner] = yield* sql<{ id: string }>` + SELECT id FROM machine WHERE store_id = ${storeId} AND id <> ${machineId} + `; + if (owner === undefined) return; + yield* Effect.logWarning( + `Not syncing machine ${machineId}: its store is already synced as machine ${owner.id}`, + ); + return yield* Effect.never; + }); + /** One connection's sync: decides between resume and full sync, then follows the index. */ const connected = (machineId: string, client: Connection) => Effect.gen(function* () { const info = yield* client("server.info", {}); + yield* servers.setHost(machineId, info.host?.name ?? null); const [machine] = yield* sql<{ storeId: string | null; cursor: number; namespace: string }>` SELECT store_id, cursor, namespace FROM machine WHERE id = ${machineId} `; if (machine === undefined) return yield* Effect.never; - const [owner] = yield* sql<{ id: string }>` - SELECT id FROM machine WHERE store_id = ${info.storeId} AND id <> ${machineId} - `; - if (owner !== undefined) { - yield* Effect.logWarning( - `Not syncing machine ${machineId}: its store is already synced as machine ${owner.id}`, - ); - return yield* Effect.never; - } + yield* unclaimed(machineId, info.storeId); let full = info.storeId !== machine.storeId || info.rev < machine.cursor; const live: Live = { client, threads: yield* FiberMap.make() }; yield* Effect.addFinalizer(() => @@ -372,6 +384,80 @@ export const layer = Layer.effect( return yield* Effect.fail("The index stream ended"); }).pipe(Effect.scoped); + /** + * One connection's usage rollup: from `machine.rollup` while the store is the one it counts in, + * else from zero. A read from zero is the store's whole state, so its first batch replaces the + * store's rows, in one transaction so a failed re-read leaves the old ones: a restored backup, or + * a removed server added again, leaves nothing stale. A resume that left the store's line (by + * `rollup_boot`) comes back from zero with `reset`; a server older than boots is read from zero + * only while its head is behind the cursor. + */ + const syncUsage = (machineId: string, client: Connection) => + Effect.gen(function* () { + const info = yield* client("server.info", {}); + const [machine] = yield* sql<{ rollup: number; rollupStore: string | null; rollupBoot: string | null }>` + SELECT rollup, rollup_store, rollup_boot FROM machine WHERE id = ${machineId} + `; + if (machine === undefined) return yield* Effect.never; + yield* unclaimed(machineId, info.storeId); + const resumed = info.storeId === machine.rollupStore && info.rev >= machine.rollup; + let whole = !resumed; + yield* batches( + client("usage.sync", { + after: resumed ? machine.rollup : 0, + ...(resumed && machine.rollupBoot !== null ? { boot: machine.rollupBoot } : {}), + }), + (item) => item._tag === "Synced", + () => Effect.succeed(true), + (rows, synced) => + Effect.gen(function* () { + if (synced.storeId !== info.storeId) return yield* Effect.fail("The usage stream changed store"); + const replaced = whole || synced.reset === true; + yield* transact((append) => + Effect.gen(function* () { + if (replaced) { + yield* sql`DELETE FROM usage WHERE store_id = ${synced.storeId}`; + yield* sql`DELETE FROM tally WHERE store_id = ${synced.storeId}`; + } + yield* Mirror.usage(machineId, synced.storeId, rows); + yield* sql` + UPDATE machine SET rollup = ${synced.rev}, rollup_store = ${synced.storeId}, + rollup_boot = ${synced.boot ?? null} + WHERE id = ${machineId} + `; + if (rows.length > 0 || replaced) + yield* append({ + topic: syncTopics.usage, + source: "sync", + payload: { machineId, storeId: synced.storeId, items: rows.length }, + }); + }), + ); + whole = false; + }), + ); + return yield* Effect.fail("The usage stream ended"); + }); + + /** + * {@link syncUsage} for as long as the connection lasts, apart from the index: either may fail + * and retry without the other. A server older than `usage.sync` is asked again only after a while. + */ + const followUsage = (machineId: string, client: Connection) => + syncUsage(machineId, client).pipe( + Effect.catchCause((cause) => + Cause.hasInterruptsOnly(cause) + ? Effect.interrupt + : unserved(cause) + ? Effect.logInfo(`Machine ${machineId} does not serve usage.sync`).pipe( + Effect.andThen(Effect.sleep("1 minute")), + ) + : Effect.logWarning(`Usage sync of machine ${machineId} stopped`, cause), + ), + Effect.andThen(Effect.sleep("1 second")), + Effect.forever, + ); + /** A machine for as long as it is configured: connected, and reconnected after any drop. */ const follow = (row: Servers.Row) => Effect.gen(function* () { @@ -382,6 +468,7 @@ export const layer = Layer.effect( const client = Context.get(context, Client); clients.set(row.id, client); yield* Effect.addFinalizer(() => Effect.sync(() => clients.delete(row.id))); + yield* Effect.forkScoped(followUsage(row.id, client)); yield* connected(row.id, client).pipe( Effect.catchCause((cause) => Effect.logWarning("Machine sync stopped; reconnecting", cause).pipe( diff --git a/apps/server/test/__artifacts__/host.migrated.json b/apps/server/test/__artifacts__/host.migrated.json new file mode 100644 index 00000000..c937d927 --- /dev/null +++ b/apps/server/test/__artifacts__/host.migrated.json @@ -0,0 +1,30 @@ +{ + "stored": [ + { + "id": "", + "kind": "local", + "name": null, + "host": null + }, + { + "id": "", + "kind": "remote", + "name": "Box", + "host": null + } + ], + "servers": [ + { + "id": "", + "name": "This Machine", + "host": null, + "override": null + }, + { + "id": "", + "name": "Box", + "host": null, + "override": "Box" + } + ] +} diff --git a/apps/server/test/__artifacts__/host.names.json b/apps/server/test/__artifacts__/host.names.json new file mode 100644 index 00000000..56b7283e --- /dev/null +++ b/apps/server/test/__artifacts__/host.names.json @@ -0,0 +1,178 @@ +{ + "reported": [ + { + "stored": { + "id": "", + "kind": "local", + "name": null, + "host": "Studio Mac" + }, + "id": "", + "name": "Studio Mac", + "host": "Studio Mac", + "override": null, + "sidebar": "Studio Mac", + "allThreads": "Studio Mac" + }, + { + "stored": { + "id": "", + "kind": "remote", + "name": null, + "host": "ubuntu" + }, + "id": "", + "name": "ubuntu", + "host": "ubuntu", + "override": null, + "sidebar": "ubuntu", + "allThreads": "ubuntu" + }, + { + "stored": { + "id": "", + "kind": "remote", + "name": null, + "host": null + }, + "id": "", + "name": "127.0.0.1:", + "host": null, + "override": null, + "sidebar": "127.0.0.1:", + "allThreads": "127.0.0.1:" + } + ], + "overridden": [ + { + "stored": { + "id": "", + "kind": "local", + "name": null, + "host": "Studio Mac" + }, + "id": "", + "name": "Studio Mac", + "host": "Studio Mac", + "override": null, + "sidebar": "Studio Mac", + "allThreads": "Studio Mac" + }, + { + "stored": { + "id": "", + "kind": "remote", + "name": "Build Box", + "host": "ubuntu" + }, + "id": "", + "name": "Build Box", + "host": "ubuntu", + "override": "Build Box", + "sidebar": "Build Box", + "allThreads": "Build Box" + }, + { + "stored": { + "id": "", + "kind": "remote", + "name": null, + "host": null + }, + "id": "", + "name": "127.0.0.1:", + "host": null, + "override": null, + "sidebar": "127.0.0.1:", + "allThreads": "127.0.0.1:" + } + ], + "cleared": [ + { + "stored": { + "id": "", + "kind": "local", + "name": null, + "host": "Studio Mac" + }, + "id": "", + "name": "Studio Mac", + "host": "Studio Mac", + "override": null, + "sidebar": "Studio Mac", + "allThreads": "Studio Mac" + }, + { + "stored": { + "id": "", + "kind": "remote", + "name": null, + "host": "ubuntu" + }, + "id": "", + "name": "ubuntu", + "host": "ubuntu", + "override": null, + "sidebar": "ubuntu", + "allThreads": "ubuntu" + }, + { + "stored": { + "id": "", + "kind": "remote", + "name": null, + "host": null + }, + "id": "", + "name": "127.0.0.1:", + "host": null, + "override": null, + "sidebar": "127.0.0.1:", + "allThreads": "127.0.0.1:" + } + ], + "renamed": [ + { + "stored": { + "id": "", + "kind": "local", + "name": null, + "host": "Studio Mac" + }, + "id": "", + "name": "Studio Mac", + "host": "Studio Mac", + "override": null, + "sidebar": "Studio Mac", + "allThreads": "Studio Mac" + }, + { + "stored": { + "id": "", + "kind": "remote", + "name": null, + "host": "ubuntu-2" + }, + "id": "", + "name": "ubuntu-2", + "host": "ubuntu-2", + "override": null, + "sidebar": "ubuntu-2", + "allThreads": "ubuntu-2" + }, + { + "stored": { + "id": "", + "kind": "remote", + "name": null, + "host": null + }, + "id": "", + "name": "127.0.0.1:", + "host": null, + "override": null, + "sidebar": "127.0.0.1:", + "allThreads": "127.0.0.1:" + } + ] +} diff --git a/apps/server/test/__artifacts__/metrics.filters.json b/apps/server/test/__artifacts__/metrics.filters.json new file mode 100644 index 00000000..10ca22f6 --- /dev/null +++ b/apps/server/test/__artifacts__/metrics.filters.json @@ -0,0 +1,415 @@ +{ + "Host A": { + "rows": [ + { + "day": "1970-01-01", + "machineId": "", + "sessionId": "", + "provider": "openai", + "model": "gpt-5.6-luna", + "thinking": "medium", + "requests": 2, + "tokens": { + "input": 2, + "output": 2, + "cacheRead": 0, + "cacheWrite": 0, + "reasoning": 0 + }, + "cost": { + "total": 0, + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "duration": 0, + "timed": 2, + "durations": [ + 2, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + }, + { + "day": "1970-01-01", + "machineId": "", + "sessionId": "", + "provider": "openai", + "model": "gpt-5.6-luna", + "thinking": "medium", + "requests": 2, + "tokens": { + "input": 2, + "output": 2, + "cacheRead": 0, + "cacheWrite": 0, + "reasoning": 0 + }, + "cost": { + "total": 0, + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "duration": 0, + "timed": 2, + "durations": [ + 2, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + } + ], + "tallies": [ + { + "day": "1970-01-01", + "machineId": "", + "sessionId": "", + "kind": "tool", + "name": "bash", + "count": 1 + }, + { + "day": "1970-01-01", + "machineId": "", + "sessionId": "", + "kind": "tool", + "name": "bash", + "count": 1 + }, + { + "day": "", + "machineId": "", + "sessionId": "", + "kind": "prompt", + "name": "", + "count": 1 + }, + { + "day": "", + "machineId": "", + "sessionId": "", + "kind": "prompt", + "name": "", + "count": 1 + } + ], + "threads": [ + { + "day": "1970-01-01", + "machineId": "", + "sessionId": "", + "threads": 1 + }, + { + "day": "1970-01-01", + "machineId": "", + "sessionId": "", + "threads": 1 + }, + { + "day": "", + "machineId": "", + "sessionId": "", + "threads": 1 + }, + { + "day": "", + "machineId": "", + "sessionId": "", + "threads": 1 + } + ], + "sessions": [ + { + "id": "", + "machineId": "", + "projectId": "", + "title": "bash echo alpha", + "deleted": false + }, + { + "id": "", + "machineId": "", + "projectId": "", + "title": "bash echo beta", + "deleted": false + } + ], + "projects": [ + { + "id": "", + "name": "Alpha" + }, + { + "id": "", + "name": "Beta" + } + ], + "machines": [ + { + "id": "", + "name": "Host A" + } + ], + "edges": [ + 250, + 374, + 561, + 839, + 1257, + 1882, + 2818, + 4220, + 6318, + 9461, + 14167, + 21213, + 31764, + 47563, + 71221, + 106645, + 159688, + 239115, + 358046, + 536133, + 802797, + 1202096, + 1800000 + ] + }, + "Host B": { + "rows": [ + { + "day": "1970-01-01", + "machineId": "", + "sessionId": "", + "provider": "openai", + "model": "gpt-5.6-luna", + "thinking": "medium", + "requests": 2, + "tokens": { + "input": 2, + "output": 2, + "cacheRead": 0, + "cacheWrite": 0, + "reasoning": 0 + }, + "cost": { + "total": 0, + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "duration": 0, + "timed": 2, + "durations": [ + 2, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + } + ], + "tallies": [ + { + "day": "1970-01-01", + "machineId": "", + "sessionId": "", + "kind": "tool", + "name": "bash", + "count": 1 + }, + { + "day": "", + "machineId": "", + "sessionId": "", + "kind": "prompt", + "name": "", + "count": 1 + } + ], + "threads": [ + { + "day": "1970-01-01", + "machineId": "", + "sessionId": "", + "threads": 1 + }, + { + "day": "", + "machineId": "", + "sessionId": "", + "threads": 1 + } + ], + "sessions": [ + { + "id": "", + "machineId": "", + "projectId": "", + "title": "bash echo gamma", + "deleted": false + } + ], + "projects": [ + { + "id": "", + "name": "Gamma" + } + ], + "machines": [ + { + "id": "", + "name": "Host B" + } + ], + "edges": [ + 250, + 374, + 561, + 839, + 1257, + 1882, + 2818, + 4220, + 6318, + 9461, + 14167, + 21213, + 31764, + 47563, + 71221, + 106645, + 159688, + 239115, + 358046, + 536133, + 802797, + 1202096, + 1800000 + ] + }, + "all": { + "rows": 3, + "machines": 2, + "projects": [ + { + "id": "", + "name": "Alpha" + }, + { + "id": "", + "name": "Beta" + }, + { + "id": "", + "name": "Gamma" + } + ] + }, + "alpha": { + "sessions": [ + { + "id": "", + "machineId": "", + "projectId": "", + "title": "bash echo alpha", + "deleted": false + } + ], + "projects": [ + { + "id": "", + "name": "Alpha" + } + ] + }, + "hoursA": [ + { + "hour": 0, + "machineId": "", + "sessionId": "", + "provider": "openai", + "model": "gpt-5.6-luna", + "requests": 2, + "tokens": 4, + "cost": 0 + }, + { + "hour": 0, + "machineId": "", + "sessionId": "", + "provider": "openai", + "model": "gpt-5.6-luna", + "requests": 2, + "tokens": 4, + "cost": 0 + } + ] +} diff --git a/apps/server/test/__artifacts__/metrics.simulation.json b/apps/server/test/__artifacts__/metrics.simulation.json new file mode 100644 index 00000000..2612b80d --- /dev/null +++ b/apps/server/test/__artifacts__/metrics.simulation.json @@ -0,0 +1,176 @@ +{ + "seeded": { + "cw-a": { + "sessions": 300, + "prompts": 2277, + "requests": 7321, + "aborted": 196, + "failed": 137, + "calls": 7778 + }, + "cw-b": { + "sessions": 120, + "prompts": 881, + "requests": 2839, + "aborted": 98, + "failed": 60, + "calls": 3042 + } + }, + "connected": { + "before": { + "all": { + "requests": 10160, + "tokens": 496714759, + "cost": 228.254801875, + "timed": 9866, + "prompts": 3158, + "tools": 10315 + }, + "sessions": 420, + "deleted": 0 + }, + "after": { + "all": { + "requests": 10160, + "tokens": 496714759, + "cost": 228.254801875, + "timed": 9866, + "prompts": 3158, + "tools": 10315 + }, + "sessions": 420, + "deleted": 7 + } + }, + "live": { + "all": { + "requests": 10166, + "tokens": 496714771, + "cost": 228.254801875, + "timed": 9872, + "prompts": 3161, + "tools": 10318 + }, + "sessions": 421, + "deleted": 7 + }, + "resumed": { + "applied": [ + 3, + 3 + ], + "totals": { + "all": { + "requests": 10178, + "tokens": 496714795, + "cost": 228.254801875, + "timed": 9884, + "prompts": 3167, + "tools": 10324 + }, + "sessions": 423, + "deleted": 7 + } + }, + "paused": { + "paused": { + "all": { + "requests": 10182, + "tokens": 496714803, + "cost": 228.254801875, + "timed": 9888, + "prompts": 3169, + "tools": 10326 + }, + "sessions": 424, + "deleted": 7 + }, + "resumed": { + "all": { + "requests": 10182, + "tokens": 496714803, + "cost": 228.254801875, + "timed": 9888, + "prompts": 3169, + "tools": 10326 + }, + "sessions": 424, + "deleted": 7 + } + }, + "restartedAway": { + "all": { + "requests": 10186, + "tokens": 496714811, + "cost": 228.254801875, + "timed": 9892, + "prompts": 3171, + "tools": 10328 + }, + "sessions": 425, + "deleted": 7 + }, + "cut": { + "all": { + "requests": 10188, + "tokens": 496714815, + "cost": 228.254801875, + "timed": 9894, + "prompts": 3172, + "tools": 10329 + }, + "sessions": 426, + "deleted": 7 + }, + "restored": { + "all": { + "requests": 10192, + "tokens": 496714823, + "cost": 228.254801875, + "timed": 9898, + "prompts": 3174, + "tools": 10331 + }, + "sessions": 428, + "deleted": 9 + }, + "readded": { + "kept": { + "all": { + "requests": 10192, + "tokens": 496714823, + "cost": 228.254801875, + "timed": 9898, + "prompts": 3174, + "tools": 10331 + }, + "sessions": 428, + "deleted": 130 + }, + "again": { + "all": { + "requests": 10192, + "tokens": 496714823, + "cost": 228.254801875, + "timed": 9898, + "prompts": 3174, + "tools": 10331 + }, + "sessions": 428, + "deleted": 9 + } + }, + "recreated": { + "all": { + "requests": 10196, + "tokens": 496714831, + "cost": 228.254801875, + "timed": 9902, + "prompts": 3176, + "tools": 10333 + }, + "sessions": 429, + "deleted": 130 + } +} diff --git a/apps/server/test/__artifacts__/metrics.stores.json b/apps/server/test/__artifacts__/metrics.stores.json new file mode 100644 index 00000000..eeb50460 --- /dev/null +++ b/apps/server/test/__artifacts__/metrics.stores.json @@ -0,0 +1,223 @@ +{ + "rows": [ + { + "day": "1970-01-01", + "machineId": "", + "sessionId": "", + "provider": "openai", + "model": "gpt-5.6-luna", + "thinking": "medium", + "requests": 2, + "tokens": { + "input": 2, + "output": 2, + "cacheRead": 0, + "cacheWrite": 0, + "reasoning": 0 + }, + "cost": { + "total": 0, + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "duration": 0, + "timed": 2, + "durations": [ + 2, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + }, + { + "day": "1970-01-01", + "machineId": "", + "sessionId": "", + "provider": "openai", + "model": "gpt-5.6-luna", + "thinking": "medium", + "requests": 2, + "tokens": { + "input": 2, + "output": 2, + "cacheRead": 0, + "cacheWrite": 0, + "reasoning": 0 + }, + "cost": { + "total": 0, + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "duration": 0, + "timed": 2, + "durations": [ + 2, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + } + ], + "tallies": [ + { + "day": "1970-01-01", + "machineId": "", + "sessionId": "", + "kind": "tool", + "name": "bash", + "count": 1 + }, + { + "day": "1970-01-01", + "machineId": "", + "sessionId": "", + "kind": "tool", + "name": "bash", + "count": 1 + }, + { + "day": "", + "machineId": "", + "sessionId": "", + "kind": "prompt", + "name": "", + "count": 1 + }, + { + "day": "", + "machineId": "", + "sessionId": "", + "kind": "prompt", + "name": "", + "count": 1 + } + ], + "threads": [ + { + "day": "1970-01-01", + "machineId": "", + "sessionId": "", + "threads": 1 + }, + { + "day": "1970-01-01", + "machineId": "", + "sessionId": "", + "threads": 1 + }, + { + "day": "", + "machineId": "", + "sessionId": "", + "threads": 1 + }, + { + "day": "", + "machineId": "", + "sessionId": "", + "threads": 1 + } + ], + "sessions": [ + { + "id": "", + "machineId": "", + "projectId": "", + "title": null, + "deleted": true + }, + { + "id": "", + "machineId": "", + "projectId": "", + "title": "bash echo new", + "deleted": false + } + ], + "projects": [ + { + "id": "", + "name": "New" + }, + { + "id": "", + "name": "Old" + } + ], + "machines": [ + { + "id": "", + "name": "Fixture" + } + ], + "edges": [ + 250, + 374, + 561, + 839, + 1257, + 1882, + 2818, + 4220, + 6318, + 9461, + 14167, + 21213, + 31764, + 47563, + 71221, + 106645, + 159688, + 239115, + 358046, + 536133, + 802797, + 1202096, + 1800000 + ] +} diff --git a/apps/server/test/__artifacts__/metrics.usage.json b/apps/server/test/__artifacts__/metrics.usage.json new file mode 100644 index 00000000..6ed95683 --- /dev/null +++ b/apps/server/test/__artifacts__/metrics.usage.json @@ -0,0 +1,271 @@ +{ + "notice": { + "machineId": "", + "storeId": "", + "items": 1 + }, + "reported": [ + { + "cost": 0, + "input": 2, + "output": 2, + "cacheRead": 0, + "cacheWrite": 0 + }, + { + "cost": 0, + "input": 2, + "output": 2, + "cacheRead": 0, + "cacheWrite": 0 + }, + { + "cost": 0, + "input": 2, + "output": 2, + "cacheRead": 0, + "cacheWrite": 0 + } + ], + "after": { + "rows": [ + { + "day": "1970-01-01", + "machineId": "", + "sessionId": "", + "provider": "openai", + "model": "gpt-5.6-luna", + "thinking": "medium", + "requests": 6, + "tokens": { + "input": 6, + "output": 6, + "cacheRead": 0, + "cacheWrite": 0, + "reasoning": 0 + }, + "cost": { + "total": 0, + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "duration": 0, + "timed": 6, + "durations": [ + 6, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + }, + { + "day": "1970-01-01", + "machineId": "", + "sessionId": "", + "provider": "openai", + "model": "gpt-5.6-luna", + "thinking": "medium", + "requests": 2, + "tokens": { + "input": 2, + "output": 2, + "cacheRead": 0, + "cacheWrite": 0, + "reasoning": 0 + }, + "cost": { + "total": 0, + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "duration": 0, + "timed": 2, + "durations": [ + 2, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + } + ], + "tallies": [ + { + "day": "1970-01-01", + "machineId": "", + "sessionId": "", + "kind": "tool", + "name": "bash", + "count": 2 + }, + { + "day": "1970-01-01", + "machineId": "", + "sessionId": "", + "kind": "tool", + "name": "bash", + "count": 1 + }, + { + "day": "", + "machineId": "", + "sessionId": "", + "kind": "prompt", + "name": "", + "count": 2 + }, + { + "day": "", + "machineId": "", + "sessionId": "", + "kind": "prompt", + "name": "", + "count": 1 + } + ], + "threads": [ + { + "day": "1970-01-01", + "machineId": "", + "sessionId": "", + "threads": 3 + }, + { + "day": "1970-01-01", + "machineId": "", + "sessionId": "", + "threads": 1 + }, + { + "day": "", + "machineId": "", + "sessionId": "", + "threads": 2 + }, + { + "day": "", + "machineId": "", + "sessionId": "", + "threads": 1 + } + ], + "sessions": [ + { + "id": "", + "machineId": "", + "projectId": "", + "title": "bash echo one", + "deleted": false + }, + { + "id": "", + "machineId": "", + "projectId": "", + "title": "bash echo two", + "deleted": false + } + ], + "projects": [ + { + "id": "", + "name": "Alpha" + }, + { + "id": "", + "name": "Beta" + } + ], + "machines": [ + { + "id": "", + "name": "Metrics Host" + } + ], + "edges": [ + 250, + 374, + 561, + 839, + 1257, + 1882, + 2818, + 4220, + 6318, + 9461, + 14167, + 21213, + 31764, + 47563, + 71221, + 106645, + 159688, + 239115, + 358046, + 536133, + 802797, + 1202096, + 1800000 + ] + }, + "deleted": { + "sessions": [ + { + "id": "", + "machineId": "", + "projectId": "", + "title": "bash echo one", + "deleted": false + }, + { + "id": "", + "machineId": "", + "projectId": "", + "title": null, + "deleted": true + } + ] + } +} diff --git a/apps/server/test/__artifacts__/metrics.zones.json b/apps/server/test/__artifacts__/metrics.zones.json new file mode 100644 index 00000000..9aff028f --- /dev/null +++ b/apps/server/test/__artifacts__/metrics.zones.json @@ -0,0 +1,410 @@ +{ + "kolkata": { + "both": { + "2026-03-09": 1, + "2026-03-10": 2 + }, + "first": { + "2026-03-09": 1 + }, + "second": { + "2026-03-10": 2 + }, + "tallies": [ + { + "day": "2026-03-10", + "machineId": "local", + "sessionId": "india", + "kind": "prompt", + "name": "", + "count": 1 + } + ], + "hoursFirst": [ + [ + 23, + 1 + ] + ], + "hoursSecond": [ + [ + 0, + 2 + ] + ] + }, + "kathmandu": { + "both": { + "2026-03-09": 1, + "2026-03-10": 2 + }, + "hours": [ + [ + 0, + 2 + ] + ] + }, + "utc": { + "2026-03-09": 3 + }, + "losAngeles": { + "both": { + "2026-03-09": 1, + "2026-03-10": 2 + }, + "second": { + "2026-03-10": 2 + }, + "hoursFirst": [ + [ + 23, + 1 + ] + ], + "hoursSecond": [ + [ + 0, + 2 + ] + ] + }, + "fallBack": { + "days": { + "2026-10-31": 1, + "2026-11-01": 25, + "2026-11-02": 1 + }, + "hours": [ + [ + 0, + 1 + ], + [ + 1, + 2 + ], + [ + 2, + 1 + ], + [ + 3, + 1 + ], + [ + 4, + 1 + ], + [ + 5, + 1 + ], + [ + 6, + 1 + ], + [ + 7, + 1 + ], + [ + 8, + 1 + ], + [ + 9, + 1 + ], + [ + 10, + 1 + ], + [ + 11, + 1 + ], + [ + 12, + 1 + ], + [ + 13, + 1 + ], + [ + 14, + 1 + ], + [ + 15, + 1 + ], + [ + 16, + 1 + ], + [ + 17, + 1 + ], + [ + 18, + 1 + ], + [ + 19, + 1 + ], + [ + 20, + 1 + ], + [ + 21, + 1 + ], + [ + 22, + 1 + ], + [ + 23, + 1 + ] + ] + }, + "springForward": { + "days": { + "2026-03-07": 1, + "2026-03-08": 23, + "2026-03-09": 1 + }, + "hours": [ + [ + 0, + 1 + ], + [ + 1, + 1 + ], + [ + 3, + 1 + ], + [ + 4, + 1 + ], + [ + 5, + 1 + ], + [ + 6, + 1 + ], + [ + 7, + 1 + ], + [ + 8, + 1 + ], + [ + 9, + 1 + ], + [ + 10, + 1 + ], + [ + 11, + 1 + ], + [ + 12, + 1 + ], + [ + 13, + 1 + ], + [ + 14, + 1 + ], + [ + 15, + 1 + ], + [ + 16, + 1 + ], + [ + 17, + 1 + ], + [ + 18, + 1 + ], + [ + 19, + 1 + ], + [ + 20, + 1 + ], + [ + 21, + 1 + ], + [ + 22, + 1 + ], + [ + 23, + 1 + ] + ] + }, + "stores": { + "rows": [ + { + "day": "2026-05-01", + "machineId": "local", + "sessionId": "stores", + "provider": "scripted", + "model": "model", + "thinking": "off", + "requests": 7, + "tokens": { + "input": 70, + "output": 35, + "cacheRead": 0, + "cacheWrite": 0, + "reasoning": 0 + }, + "cost": { + "total": 3.5, + "input": 1.75, + "output": 1.75, + "cacheRead": 0, + "cacheWrite": 0 + }, + "duration": 2100, + "timed": 7, + "durations": [ + 0, + 7, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + } + ], + "tallies": [], + "threads": [ + { + "day": "2026-05-01", + "machineId": "local", + "sessionId": "stores", + "threads": 2 + } + ], + "sessions": [ + { + "id": "stores", + "machineId": "local", + "projectId": "prj_stores", + "title": null, + "deleted": true + } + ], + "projects": [ + { + "id": "prj_stores", + "name": null + } + ], + "machines": [ + { + "id": "local", + "name": "This Machine" + } + ], + "edges": [ + 250, + 374, + 561, + 839, + 1257, + 1882, + 2818, + 4220, + 6318, + 9461, + 14167, + 21213, + 31764, + 47563, + 71221, + 106645, + 159688, + 239115, + 358046, + 536133, + 802797, + 1202096, + 1800000 + ] + }, + "refusals": { + "zone": { + "tag": "Server.MetricsError", + "message": "\"Mars/Olympus\" is not a time zone." + }, + "backwards": { + "tag": "Server.MetricsError", + "message": "The period ends (2026-03-09) before it starts (2026-03-10)." + }, + "malformed": { + "tag": "Server.MetricsError", + "message": "\"2026-3-9\" is not a day: expected YYYY-MM-DD." + }, + "impossible": { + "tag": "Server.MetricsError", + "message": "\"2026-02-30\" is not a day: expected YYYY-MM-DD." + }, + "hoursZone": { + "tag": "Server.MetricsError", + "message": "\"\" is not a time zone." + } + } +} diff --git a/apps/server/test/__artifacts__/reads.machines.json b/apps/server/test/__artifacts__/reads.machines.json index 87b8ffa4..112e8e85 100644 --- a/apps/server/test/__artifacts__/reads.machines.json +++ b/apps/server/test/__artifacts__/reads.machines.json @@ -3,7 +3,7 @@ { "machine": { "id": "", - "name": "CodeWork" + "name": "Fixture" }, "id": "", "name": "Local", @@ -30,7 +30,7 @@ { "machine": { "id": "", - "name": "CodeWork" + "name": "Fixture" }, "id": "", "name": "Local", diff --git a/apps/server/test/__artifacts__/schema.json b/apps/server/test/__artifacts__/schema.json index f5da9b3b..7ce0f74e 100644 --- a/apps/server/test/__artifacts__/schema.json +++ b/apps/server/test/__artifacts__/schema.json @@ -70,7 +70,7 @@ "type": "table", "name": "machine", "tbl_name": "machine", - "sql": "CREATE TABLE machine ( id TEXT PRIMARY KEY, kind TEXT NOT NULL, name TEXT NOT NULL, executable TEXT, home TEXT, args TEXT NOT NULL DEFAULT '', env TEXT NOT NULL DEFAULT '[]', url TEXT, namespace TEXT NOT NULL, is_default INTEGER NOT NULL DEFAULT 0, pos INTEGER NOT NULL, store_id TEXT UNIQUE, cursor INTEGER NOT NULL DEFAULT 0, created_at INTEGER NOT NULL )" + "sql": "CREATE TABLE machine ( id TEXT PRIMARY KEY, kind TEXT NOT NULL, executable TEXT, home TEXT, args TEXT NOT NULL DEFAULT '', env TEXT NOT NULL DEFAULT '[]', url TEXT, namespace TEXT NOT NULL, is_default INTEGER NOT NULL DEFAULT 0, pos INTEGER NOT NULL, store_id TEXT UNIQUE, cursor INTEGER NOT NULL DEFAULT 0, created_at INTEGER NOT NULL , rollup INTEGER NOT NULL DEFAULT 0, rollup_store TEXT, rollup_boot TEXT, host TEXT, name TEXT)" }, { "type": "table", @@ -150,6 +150,12 @@ "tbl_name": "sqlite_sequence", "sql": "CREATE TABLE sqlite_sequence(name,seq)" }, + { + "type": "table", + "name": "tally", + "tbl_name": "tally", + "sql": "CREATE TABLE tally ( machine_id TEXT NOT NULL, store_id TEXT NOT NULL, hour INTEGER NOT NULL, thread_id TEXT NOT NULL, kind TEXT NOT NULL, name TEXT NOT NULL, session_id TEXT NOT NULL, project_id TEXT NOT NULL, count INTEGER NOT NULL, rev INTEGER NOT NULL, PRIMARY KEY (store_id, hour, thread_id, kind, name) )" + }, { "type": "table", "name": "thread", @@ -192,6 +198,12 @@ "tbl_name": "thread", "sql": "CREATE INDEX thread_status_idx ON thread (organization_id, status, ordering) WHERE deleted_at IS NULL" }, + { + "type": "table", + "name": "usage", + "tbl_name": "usage", + "sql": "CREATE TABLE usage ( machine_id TEXT NOT NULL, store_id TEXT NOT NULL, hour INTEGER NOT NULL, thread_id TEXT NOT NULL, provider TEXT NOT NULL, model TEXT NOT NULL, thinking TEXT NOT NULL, session_id TEXT NOT NULL, project_id TEXT NOT NULL, requests INTEGER NOT NULL, input INTEGER NOT NULL, output INTEGER NOT NULL, cache_read INTEGER NOT NULL, cache_write INTEGER NOT NULL, reasoning INTEGER NOT NULL, cost REAL NOT NULL, cost_input REAL NOT NULL, cost_output REAL NOT NULL, cost_cache_read REAL NOT NULL, cost_cache_write REAL NOT NULL, duration REAL NOT NULL, timed INTEGER NOT NULL, durations TEXT NOT NULL, rev INTEGER NOT NULL, PRIMARY KEY (store_id, hour, thread_id, provider, model, thinking) )" + }, { "type": "table", "name": "widget", @@ -210,7 +222,6 @@ { "id": "local", "kind": "local", - "name": "CodeWork", "executable": null, "home": null, "args": "", @@ -221,7 +232,12 @@ "pos": 0, "store_id": null, "cursor": 0, - "created_at": "( throw new Error(`timed out waiting for ${what}`); }; -/** A CodeWork server of the fixture in `home` on `port`, with its scripted model. */ -export const codework = async (home: string, port: number) => { - const proc = spawn(process.execPath, ["--conditions=development", fixture, home, String(port)], { +/** + * A CodeWork server of the fixture in `home` on `port`, with its scripted model; `older` lacks + * `usage.sync` and a host name. `host` is the host name it reports (`null` for none), else `Fixture`. + */ +export const codework = async ( + home: string, + port: number, + options: { readonly older?: boolean; readonly host?: string | null } = {}, +) => { + const args = [fixture, home, String(port), ...(options.older === true ? ["127.0.0.1", "older"] : [])]; + const proc = spawn(process.execPath, ["--conditions=development", ...args], { + env: options.host === undefined ? process.env : { ...process.env, CODEWORK_FIXTURE_HOST: options.host ?? "" }, stdio: ["pipe", "pipe", "inherit"], }); processes.add(proc); @@ -108,6 +117,9 @@ export const app = async (home: string) => { stdio: ["ignore", "pipe", "inherit"], }); processes.add(proc); + // Its stdout, the log included, line by line. + const lines: Array = []; + createInterface({ input: proc.stdout! }).on("line", (line) => lines.push(line)); const { port } = await ready(proc); const protocol = RpcClient.layerProtocolSocket().pipe( Layer.provide(Socket.layerWebSocket(`ws://127.0.0.1:${port}/rpc`)), @@ -116,6 +128,7 @@ export const app = async (home: string) => { ); return { proc, + lines, run: (use: (rpc: AppRpc) => Effect.Effect) => Effect.gen(function* () { return yield* use(yield* appClient); @@ -143,7 +156,8 @@ export const query = (home: string, statement: string, ...params: Array) => { - const db = new DatabaseSync(join(home, "app.db")); + // Waits out the app's own writes, as a startup's. + const db = new DatabaseSync(join(home, "app.db"), { timeout: 5_000 }); try { db.prepare(statement).run(...params); } finally { @@ -174,6 +188,40 @@ export const synced = async (home: string, machineId: string, server: Codework) return { storeId, rev }; }; +export const cursorOf = (home: string, machineId: string) => + query(home, "SELECT rollup, rollup_store FROM machine WHERE id = ?", machineId)[0] as { + rollup: number; + rollup_store: string | null; + }; + +/** Waits until the app's usage cursor for `machineId` reached the server's head. */ +export const rolled = async (home: string, machineId: string, server: Codework) => { + const { storeId, rev } = await server.run((rpc) => rpc["server.info"]({})); + await waitFor(() => { + const row = cursorOf(home, machineId); + return row.rollup_store === storeId && row.rollup >= rev ? row : undefined; + }, `usage of machine ${machineId} at rev ${rev}`); + return { storeId, rev }; +}; + +/** Events on `topics` from the moment it resolves, collected until `stop`. */ +export const follow = async (desk: Awaited>, topics: ReadonlyArray) => { + const events: Array<{ topic: string; payload: Row }> = []; + let subscribed = false; + const background = desk.background((rpc) => + rpc["event.subscribe"]({ topics }).pipe( + Stream.runForEach((item) => + Effect.sync(() => { + if ("_tag" in item) subscribed = true; + else events.push({ topic: item.topic, payload: item.payload as Row }); + }), + ), + ), + ); + await waitFor(() => (subscribed ? true : undefined), `a subscription to ${topics.join(", ")}`); + return { events, stop: background.stop }; +}; + /** The payloads of the sync's bus events after `after` (an event seq). */ export const notices = (home: string, topic: string, after = 0) => query(home, "SELECT payload FROM event WHERE topic = ? AND seq > ? ORDER BY seq", topic, after).map( diff --git a/apps/server/test/fixtures/codework.ts b/apps/server/test/fixtures/codework.ts index 8d81816f..2ab7670d 100644 --- a/apps/server/test/fixtures/codework.ts +++ b/apps/server/test/fixtures/codework.ts @@ -1,13 +1,23 @@ /* @effect-diagnostics nodeBuiltinImport:off -- a CodeWork server run as its own process by the sync suite. */ import { createAssistantMessageEventStream, type Message } from "@codeworksh/aikit"; -import { NodeRuntime } from "@effect/platform-node"; +import { NodeHttpServer, NodeRuntime } from "@effect/platform-node"; import { Effect, Layer } from "effect"; +import { HttpRouter } from "effect/http"; +import { RpcSerialization, RpcServer } from "effect/rpc"; import { existsSync, mkdirSync, writeFileSync } from "node:fs"; +import { createServer } from "node:http"; import { join } from "node:path"; import { createInterface } from "node:readline"; -import { assistant } from "../../../../packages/harness/test/fixtures/llm.ts"; +import { assistant as scriptedAssistant } from "../../../../packages/harness/test/fixtures/llm.ts"; import { testModel } from "../../../../packages/harness/test/fixtures/model.ts"; import type * as LLM from "../../../../packages/harness/src/runner/llm.ts"; +import * as Clones from "../../../../packages/codework/src/server/clones.ts"; +import * as Contract from "../../../../packages/codework/src/server/contract.ts"; +import * as EventFeed from "../../../../packages/codework/src/server/feed.ts"; +import * as Handlers from "../../../../packages/codework/src/server/handlers.ts"; +import * as Host from "../../../../packages/codework/src/server/host.ts"; +import * as Identity from "../../../../packages/codework/src/server/identity.ts"; +import * as OAuth from "../../../../packages/codework/src/server/oauth.ts"; import * as Server from "../../../../packages/codework/src/server/server.ts"; import { Control, Harness } from "../../../../packages/harness/src/effect.ts"; import { bashPlugin } from "../../../../packages/harness/src/plugin/builtin/tool/bash.ts"; @@ -25,6 +35,9 @@ import * as DaytonaDriver from "../../../../packages/harness/src/sandboxes/dayto * Each line `release` on stdin lets the oldest held reply take its next step. * With `sweep`, it only ends the runs the last server left open, as a start does before it listens, * prints `swept` and exits: the store can then be read as no client has seen it. + * With `older`, it serves every call but `usage.sync`, and names no host, as a server built before it did. + * It reports its host as `CODEWORK_FIXTURE_HOST` (none when blank), else `Fixture`: never this computer's name, + * unless `CODEWORK_FIXTURE_HOST_OS=1`, which a live machine sets to report its own name as a real server does. */ const [home, port, host = "127.0.0.1", mode] = process.argv.slice(2); @@ -66,6 +79,16 @@ const bashOf = (input: LLM.Input) => { return /^bash (.+)$/s.exec(promptOf(input))?.[1]; }; +// With `CODEWORK_FIXTURE_NOW=1` a reply took the last 1.5 s of real time, so usage lands in today's +// hours; otherwise its times are its index, in the first hour of 1970, the same on every run. +const assistant: typeof scriptedAssistant = (input, index, overrides) => + scriptedAssistant(input, index, { + ...(process.env.CODEWORK_FIXTURE_NOW === "1" + ? { time: { created: Date.now() - 1_500, completed: Date.now() } } + : {}), + ...overrides, + }); + // Ids unique across restarts of this process: entry ids are the reply's message ids. let count = 0; const scripted: LLM.Open = (input, signal) => @@ -147,6 +170,31 @@ const sandboxes = process.env.DAYTONA_API_KEY?.trim() ? [DaytonaDriver.make()] : // Only the bash tool: the scripted model's `bash ` prompts call it, and nothing else varies the prompt. const harness = { home, hostCwd: home, plugins: [bashPlugin], llm: scripted, sandboxes }; +const named = process.env.CODEWORK_FIXTURE_HOST ?? "Fixture"; +const identity = + process.env.CODEWORK_FIXTURE_HOST_OS === "1" + ? Identity.layer + : Layer.succeed( + Identity.Service, + Identity.Service.of(named === "" ? Identity.current : { ...Identity.current, host: { name: named } }), + ); + +/** The RPC server alone, without the calls a server older than this build lacks. */ +const older = HttpRouter.serve( + RpcServer.layerHttp({ group: Contract.Api.omit("usage.sync"), path: Contract.path, protocol: "websocket" }).pipe( + Layer.provide(Handlers.layer.pipe(Layer.provide(EventFeed.layer))), + ), + { disableListenLog: true }, +).pipe( + Layer.provide(OAuth.layer({ home })), + Layer.provide(Host.layer), + Layer.provide(Clones.layer), + Layer.provide(NodeHttpServer.layer(createServer, { host, port: Number(port) })), + Layer.provide(RpcSerialization.layerNdjson), + Layer.provide(Layer.succeed(Identity.Service, Identity.current)), + Layer.provide(Harness.layer(harness)), +); + NodeRuntime.runMain( mode === "sweep" ? Effect.gen(function* () { @@ -156,7 +204,9 @@ NodeRuntime.runMain( yield* Effect.sync(() => process.stdout.write(`${JSON.stringify({ type: "swept" })}\n`)); }) : Effect.gen(function* () { - yield* Layer.build(Server.layer({ host, port: Number(port), harness })); + yield* Layer.build( + mode === "older" ? older : Server.layer({ host, port: Number(port), harness, identity }), + ); yield* Effect.sync(() => process.stdout.write(`${JSON.stringify({ type: "ready" })}\n`)); return yield* Effect.never; }).pipe(Effect.scoped), diff --git a/apps/server/test/host.test.ts b/apps/server/test/host.test.ts new file mode 100644 index 00000000..849b6179 --- /dev/null +++ b/apps/server/test/host.test.ts @@ -0,0 +1,158 @@ +/* @effect-diagnostics nodeBuiltinImport:off -- the suite registers CodeWork servers in temp dirs. */ +import { Effect, Option, Stream } from "effect"; +import { mkdirSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { afterEach, expect, it } from "vite-plus/test"; +import { Contract } from "@codeworksh/contract"; +import { + type AppRpc, + app, + cleanup, + type Codework, + codework, + freePort, + normalizer, + query, + snapshot as match, + stop, + synced, + tempDir, + waitFor, + write, +} from "./fixtures/apps.ts"; + +afterEach(cleanup); + +const snapshot = (name: string, artifact: unknown) => match(`host.${name}`, artifact); + +/** Registers `server` in `home` as its default local server would be, before the app starts. */ +const registerLocal = (appHome: string, server: Codework, port: number) => { + const cli = join(appHome, "cli"); + mkdirSync(cli, { recursive: true }); + writeFileSync( + join(cli, "service.json"), + JSON.stringify({ pid: server.proc.pid, version: "0.0.0", url: server.url, port }), + ); +}; + +const project = (server: Codework, directory: string, name: string) => + server.run((rpc) => rpc["session.create"]({ directory, project: { name } })); + +/** What each read that names a machine shows, per server as Settings lists them. */ +const names = (rpc: AppRpc) => + Effect.gen(function* () { + const sidebar = yield* rpc["sidebar.list"]({ organizationId: Contract.PERSONAL_ORGANIZATION, machines: "all" }); + const threads = yield* rpc["threads.list"]({ organizationId: Contract.PERSONAL_ORGANIZATION, by: "status" }); + yield* rpc["servers.check"](undefined); + const statuses = yield* rpc["servers.watch"](undefined).pipe(Stream.runHead, Effect.map(Option.getOrThrow)); + const listed = threads.lists.flatMap((list) => list.threads); + return statuses.map(({ id, name, host, override }) => ({ + id, + name, + host, + override, + sidebar: sidebar.find((project) => project.machine.id === id)?.machine.name, + allThreads: listed.find((thread) => thread.machine.id === id)?.machine.name, + })); + }); + +const stored = (appHome: string) => query(appHome, "SELECT id, kind, name, host FROM machine ORDER BY pos"); + +/** The stored row and every read of each machine. */ +const read = async (appHome: string, desk: Awaited>) => { + const rows = stored(appHome); + const reads = await desk.run(names); + return reads.map((machine) => ({ stored: rows.find((row) => row.id === machine.id), ...machine })); +}; +const shown = (state: Awaited>, id: string) => { + const machine = state.find((machine) => machine.id === id); + return [machine?.name, machine?.sidebar, machine?.allThreads]; +}; + +it("a machine is named by its server's host unless the user names it, else by where it is", async () => { + const localHome = tempDir("host-local"); + const remoteHome = tempDir("host-remote"); + const olderHome = tempDir("host-older"); + const appHome = tempDir("host-app"); + const localPort = await freePort(); + const remotePort = await freePort(); + const olderPort = await freePort(); + const local = await codework(localHome, localPort, { host: "Studio Mac" }); + let remote = await codework(remoteHome, remotePort, { host: "ubuntu" }); + const older = await codework(olderHome, olderPort, { older: true }); + await project(local, localHome, "Local"); + await project(remote, remoteHome, "Remote"); + await project(older, olderHome, "Older"); + + registerLocal(appHome, local, localPort); + const desk = await app(appHome); + const remoteId = await desk.run((rpc) => rpc["servers.add"]({ kind: "remote", url: remote.url })); + const olderId = await desk.run((rpc) => rpc["servers.add"]({ kind: "remote", name: " ", url: older.url })); + await synced(appHome, Contract.LOCAL_SERVER, local); + await synced(appHome, remoteId, remote); + await synced(appHome, olderId, older); + + const reported = await read(appHome, desk); + expect(shown(reported, Contract.LOCAL_SERVER)).toEqual(Array(3).fill("Studio Mac")); + expect(shown(reported, remoteId)).toEqual(Array(3).fill("ubuntu")); + expect(shown(reported, olderId)).toEqual(Array(3).fill(`127.0.0.1:${olderPort}`)); + + await desk.run((rpc) => rpc["servers.update"]({ id: remoteId, name: " Build Box " })); + const overridden = await read(appHome, desk); + expect(shown(overridden, remoteId)).toEqual(Array(3).fill("Build Box")); + + await desk.run((rpc) => rpc["servers.update"]({ id: remoteId, name: "" })); + const cleared = await read(appHome, desk); + expect(shown(cleared, remoteId)).toEqual(Array(3).fill("ubuntu")); + + // The same server comes back under a new name. + await stop(remote.proc); + remote = await codework(remoteHome, remotePort, { host: "ubuntu-2" }); + await waitFor( + () => (stored(appHome).find((row) => row.id === remoteId)?.host === "ubuntu-2" ? true : undefined), + "the new host name", + ); + const renamed = await read(appHome, desk); + expect(shown(renamed, remoteId)).toEqual(Array(3).fill("ubuntu-2")); + + const steps = { reported, overridden, cleared, renamed }; + expect(JSON.stringify(steps)).not.toContain("CodeWork"); + await snapshot( + "names", + normalizer( + [localHome, remoteHome, olderHome, appHome], + [ + [remoteId, ""], + [olderId, ""], + [`127.0.0.1:${olderPort}`, "127.0.0.1:"], + ], + )(steps), + ); +}, 90_000); + +it("the built-in server's old seed name is no override after the migration", async () => { + const appHome = tempDir("host-migrate"); + const first = await app(appHome); + const boxId = await first.run((rpc) => + rpc["servers.add"]({ kind: "remote", name: "Box", url: "wss://box.example.test/rpc" }), + ); + await stop(first.proc); + // app.db as it was before `host`: the seed's name on the built-in server, a name on every row. + write(appHome, "ALTER TABLE machine DROP COLUMN host"); + write(appHome, "UPDATE machine SET name = 'CodeWork' WHERE id = 'local'"); + write(appHome, "DELETE FROM effect_sql_migrations WHERE migration_id = 202611140002"); + + const second = await app(appHome); + const servers = await second.run((rpc) => + rpc["servers.watch"](undefined).pipe(Stream.runHead, Effect.map(Option.getOrThrow)), + ); + const migrated = { + stored: stored(appHome), + servers: servers.map(({ id, name, host, override }) => ({ id, name, host, override })), + }; + expect(migrated.stored).toEqual([ + { id: Contract.LOCAL_SERVER, kind: "local", name: null, host: null }, + { id: boxId, kind: "remote", name: "Box", host: null }, + ]); + await snapshot("migrated", normalizer([appHome], [[boxId, ""]])(migrated)); +}, 60_000); diff --git a/apps/server/test/machines/seed.ts b/apps/server/test/machines/seed.ts new file mode 100644 index 00000000..f95932ad --- /dev/null +++ b/apps/server/test/machines/seed.ts @@ -0,0 +1,268 @@ +/* @effect-diagnostics nodeBuiltinImport:off -- a one-shot script run inside an OrbStack machine. */ +/** + * Fills every session of a stopped CodeWork store with a history of turns across the past `days` + * days, through the thread store's own writes, so the usage rollup, thread totals and revisions come + * out as a live run makes them: + * node --conditions=development seed.ts [] + * Each session gets a start day (a little busier recently, quieter at weekends) and 2-14 turns on its days; + * each turn is a prompt and 1-6 requests to one model of its project's mix, the last answering and + * the others calling tools. A few requests fail, a few are interrupted (aborted, never timed), and a + * few calls error or are skipped. Prints one JSON line of what it wrote. + */ +import { Effect, Layer } from "effect"; +import { randomUUID } from "node:crypto"; +import { Database, Event, EventRegistry, Usage } from "../../../../packages/harness/src/effect.ts"; +import * as Document from "../../../../packages/harness/src/document/document.ts"; +import * as ThreadSchema from "../../../../packages/harness/src/thread/schema.ts"; +import * as Thread from "../../../../packages/harness/src/thread/thread.ts"; +import type { ToolStatus } from "../../../../packages/harness/src/db/schema.sql.ts"; + +const [database, daysArg = "365", seedArg = "1", untilArg] = process.argv.slice(2); +if (database === undefined) throw new Error("usage: seed.ts []"); +const DAYS = Number(daysArg); +const UNTIL = untilArg === undefined ? Date.now() : Number(untilArg); +const DAY = 86_400_000; + +// mulberry32: the same seed writes the same history. +let state = Number(seedArg) >>> 0; +const random = () => { + state = (state + 0x6d2b79f5) >>> 0; + let t = state; + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4_294_967_296; +}; +const int = (low: number, high: number) => low + Math.floor(random() * (high - low + 1)); +const pick = (items: ReadonlyArray): A => items[Math.floor(random() * items.length)]!; +const weighted = (items: ReadonlyArray): A => { + let left = random() * items.reduce((sum, [, weight]) => sum + weight, 0); + for (const [item, weight] of items) if ((left -= weight) <= 0) return item; + return items.at(-1)![0]; +}; +/** Right-skewed, like real response times: median `median`, rarely 10x. */ +const skewed = (median: number) => Math.round(median * Math.exp((random() + random() + random() - 1.5) * 1.6)); + +interface Model { + readonly provider: string; + readonly name: string; + readonly model: string; + /** USD per million tokens: input, output, cache read, cache write; zero for an unpriced one. */ + readonly price: readonly [number, number, number, number]; + readonly speed: number; +} +const MODELS: ReadonlyArray = [ + { provider: "anthropic", name: "Anthropic", model: "claude-opus-5-5", price: [5, 25, 0.5, 6.25], speed: 55 }, + { provider: "anthropic", name: "Anthropic", model: "claude-sonnet-5-5", price: [3, 15, 0.3, 3.75], speed: 80 }, + { provider: "anthropic", name: "Anthropic", model: "claude-haiku-5-5", price: [1, 5, 0.1, 1.25], speed: 140 }, + { provider: "openai", name: "OpenAI", model: "gpt-6-luna", price: [1.25, 10, 0.125, 0], speed: 70 }, + { provider: "openai-codex", name: "OpenAI Codex", model: "gpt-6-luna", price: [0, 0, 0, 0], speed: 65 }, + { provider: "github-copilot", name: "GitHub Copilot", model: "kimi-k3", price: [0.6, 2.5, 0.15, 0], speed: 45 }, + { provider: "openrouter", name: "OpenRouter", model: "crouton-alpha", price: [0, 0, 0, 0], speed: 90 }, +]; +/** Priced models are the common case; unpriced ones show up now and then. */ +const pickModel = () => weighted(MODELS.map((model) => [model, model.price[0] === 0 ? 1 : 3] as const)); +const THINKING = [ + ["off", 2], + ["minimal", 1], + ["low", 3], + ["medium", 4], + ["high", 5], + ["xhigh", 1], +] as const; +const TOOLS = [ + ["bash", 30], + ["read", 25], + ["edit", 14], + ["grep", 10], + ["write", 6], + ["glob", 5], + ["todo", 4], + ["web_fetch", 2], + ["task", 2], +] as const; +const PROMPTS = [ + "Fix the failing test in the sync suite", + "Add a retry with backoff to the uploader", + "Explain how the cursor resumes after a restart", + "Refactor the settings loader into its own module", + "Why does the sidebar flicker when a thread finishes?", + "Write a migration that adds an index on hour", + "Review this diff for race conditions", + "Make the chart tooltip follow the pointer", +]; + +const layer = Thread.layer.pipe( + Layer.provideMerge(Usage.layer), + Layer.provideMerge(Document.layer), + Layer.provideMerge(Event.layer), + Layer.provideMerge(EventRegistry.layer()), + Layer.provideMerge(Database.layer(database)), +); + +/** A start day across the period, leaning recent, a third as busy at weekends. */ +const startDay = () => { + for (;;) { + const ago = Math.floor(DAYS * random() ** 1.25); + const weekday = new Date(UNTIL - ago * DAY).getUTCDay(); + if (weekday !== 0 && weekday !== 6) return ago; + if (random() < 0.33) return ago; + } +}; + +const program = Effect.gen(function* () { + const sql = yield* Database.SqlClient.SqlClient; + const threads = yield* Thread.Service; + const sessions = yield* sql<{ readonly id: string; readonly threadId: string; readonly projectId: string }>` + SELECT s.id, t.id AS thread_id, sp.project_id FROM session s + JOIN thread t ON t.session_id = s.id JOIN space sp ON sp.id = s.space_id + ORDER BY s.id + `; + // Each project leans on two or three models. + const mixes = new Map>(); + const mixOf = (projectId: string) => { + const known = mixes.get(projectId); + if (known !== undefined) return known; + const mix = Array.from({ length: int(2, 3) }, (_, rank) => [pickModel(), 6 - rank * 2] as const); + mixes.set(projectId, mix); + return mix; + }; + const wrote = { sessions: 0, prompts: 0, requests: 0, aborted: 0, failed: 0, calls: 0 }; + + for (const session of sessions) { + const threadId = ThreadSchema.ID.make(session.threadId); + const [head] = yield* sql<{ readonly seq: number }>`SELECT seq FROM sequence WHERE aggregate_id = ${session.id}`; + let seq = head?.seq ?? 1; + let day = startDay(); + const mix = mixOf(session.projectId); + const model = weighted(mix); + const thinking = weighted(THINKING); + let context = int(8_000, 30_000); + for (let turn = 0, turns = int(2, 14); turn < turns && day >= 0; turn++) { + // Work hours, 08:00-20:00 UTC, so most zones see a working day. + let at = UNTIL - day * DAY - (UNTIL % DAY) + int(8, 19) * 3_600_000 + int(0, 3_599_999); + if (at > UNTIL) at = UNTIL - int(60_000, 3_600_000); + const promptId = randomUUID(); + yield* threads.append({ + id: promptId, + threadId, + seq: ++seq, + type: "user", + data: JSON.stringify({ messageId: promptId, role: "user", time: { created: at } }), + parts: [{ type: "text", data: JSON.stringify({ type: "text", text: pick(PROMPTS) }) }], + }); + wrote.prompts++; + const requests = int(1, 6); + for (let step = 0; step < requests; step++) { + const last = step === requests - 1; + const fails = random() < 0.02; + const interrupted = !fails && random() < 0.03; + const input = fails ? int(200, 2_000) : int(300, 4_000); + const cacheRead = fails ? 0 : Math.round(context * (0.55 + random() * 0.4)); + const cacheWrite = fails ? 0 : int(0, 3_000); + const output = fails ? 0 : last ? int(150, 2_500) : int(40, 600); + const reasoning = thinking === "off" || fails ? 0 : Math.round(output * random() * 0.6); + const duration = skewed(1_200 + (output / model.speed) * 1_000); + const [pIn, pOut, pRead, pWrite] = model.price.map((perMillion) => perMillion / 1_000_000) as [ + number, + number, + number, + number, + ]; + const cost = { + input: input * pIn, + output: output * pOut, + cacheRead: cacheRead * pRead, + cacheWrite: cacheWrite * pWrite, + total: input * pIn + output * pOut + cacheRead * pRead + cacheWrite * pWrite, + }; + const calls = + last || fails + ? [] + : Array.from( + { + length: weighted([ + [1, 6], + [2, 3], + [3, 1], + ] as const), + }, + () => weighted(TOOLS), + ); + const messageId = randomUUID(); + const envelope = { + messageId, + role: "assistant", + protocol: model.provider, + provider: { env: [], id: model.provider, name: model.name, source: "custom" }, + model: model.model, + thinkingLevel: thinking, + usage: { + input, + output, + cacheRead, + cacheWrite, + reasoning, + totalTokens: input + output + cacheRead + cacheWrite, + cost, + }, + stopReason: fails ? "error" : calls.length > 0 ? "toolUse" : interrupted ? "aborted" : "stop", + time: { created: at, completed: at + duration }, + ...(fails ? { errorMessage: "provider returned 529: overloaded" } : {}), + }; + const parts = [ + ...(output > 0 + ? [{ type: "text" as const, data: JSON.stringify({ type: "text", text: "Working on it." }) }] + : []), + ...calls.map((name, index) => { + const callId = `call_${messageId}_${index}`; + const status: ToolStatus = interrupted + ? "aborted" + : random() < 0.06 + ? "error" + : random() < 0.02 + ? "skipped" + : "completed"; + return { + type: "toolCall" as const, + callId, + toolName: name, + status, + data: JSON.stringify({ + type: "toolCall", + callID: callId, + name, + arguments: {}, + status, + time: { start: at + duration, end: at + duration + int(50, 4_000) }, + ...(status === "completed" || status === "error" + ? { result: { content: [{ type: "text", text: "ok" }], isError: status === "error" } } + : {}), + }), + }; + }), + ]; + const entry = { id: messageId, threadId, data: JSON.stringify(envelope), parts }; + if (interrupted) { + yield* threads.append({ ...entry, seq: ++seq, type: "assistant", state: "draft" }); + yield* threads.abort({ ...entry, seq: ++seq }); + wrote.aborted++; + } else yield* threads.append({ ...entry, seq: ++seq, type: "assistant" }); + wrote.requests++; + if (fails) wrote.failed++; + wrote.calls += parts.filter((part) => part.type === "toolCall").length; + context += input + output; + at += duration + int(500, 20_000); + if (interrupted || fails) break; + } + // The next turn: the same day, or a later one. + if (random() < 0.55) day -= int(1, 9); + } + // Live events of this session continue after the history. + yield* sql`UPDATE sequence SET seq = ${seq} WHERE aggregate_id = ${session.id}`; + wrote.sessions++; + } + return wrote; +}); + +const wrote = await Effect.runPromise(program.pipe(Effect.provide(layer))); +console.log(JSON.stringify(wrote)); diff --git a/apps/server/test/metrics.simulation.live.test.ts b/apps/server/test/metrics.simulation.live.test.ts new file mode 100644 index 00000000..294ff336 --- /dev/null +++ b/apps/server/test/metrics.simulation.live.test.ts @@ -0,0 +1,411 @@ +/* @effect-diagnostics nodeBuiltinImport:off -- the suite runs CodeWork servers on OrbStack Linux machines. */ +import { Effect, Stream } from "effect"; +import { afterAll, beforeAll, describe, expect, it } from "vite-plus/test"; +import { Contract, type MetricsUsage } from "@codeworksh/contract"; +import { Session } from "../../../packages/harness/src/effect.ts"; +import { + type AppRpc, + app, + cleanup, + cursorOf, + lastSeq, + machineRow, + notices, + query, + type Row, + snapshot, + stop, + tempDir, + waitFor, +} from "./fixtures/apps.ts"; +import { available, ensure, machine, type Machine, orb } from "./machines/orb.ts"; + +/* + * A long-lived desktop against two seeded machines: cw-a holds a year of history (300 sessions over + * six projects), cw-b 200 days (120 sessions), both written through the thread store by + * `machines/seed.ts`. Every case reads the totals inside each machine with sqlite3 and checks the + * app's metrics reads against them: the first connect, live turns, the app down while work goes on, + * a machine paused and resumed (and the app down across it), a network cut, a backup restored after + * its head passed the app's cursor, a server removed and added again, and a store recreated. + * `metrics.simulation.json` records each case's totals. + */ + +const cwA = machine("cw-a"); +const cwB = machine("cw-b"); +const pool = [cwA, cwB]; +type Desk = Awaited>; + +let appHome = ""; +let desk: Desk; +const ids = new Map(); +const idOf = (m: Machine) => ids.get(m.name)!; +const DAY = 86_400_000; +const until = () => new Date(Date.now() + 2 * DAY).toISOString().slice(0, 10); +const everything = (zone = "UTC") => ({ zone, since: "1970-01-01", until: until() }); +const PROJECTS = ["codework", "billing-api", "mobile-app", "infra", "docs-site", "ml-pipeline"]; +const live = { CODEWORK_FIXTURE_HOST_OS: "1", CODEWORK_FIXTURE_NOW: "1" }; + +/** The machine's server stopped and its home moved aside: the next start makes a new store. */ +const aside = async (m: Machine) => { + await m.net("restore"); + await m.stop("KILL"); + await m.sh(`[ -d ${m.cwHome} ] && mv ${m.cwHome} ${m.cwHome}.sim-$(date +%s%N); mkdir -p ${m.cwHome}/work`); +}; + +/** A new store on `m` with `count` sessions and `days` of history, served again. */ +const seeded = async (m: Machine, count: number, days: number, seed: number) => { + await aside(m); + await m.start(live); + await m.sh(`mkdir -p ${PROJECTS.map((project) => `${m.cwHome}/work/${project}`).join(" ")}`); + for (let i = 0; i < count; i++) { + const project = PROJECTS[(i * 7 + seed) % PROJECTS.length]!; + await m.run((rpc) => + rpc["session.create"]({ + directory: `${m.cwHome}/work/${project}`, + title: `Session ${i + 1} in ${project}`, + project: { name: project }, + }), + ); + } + await m.stop("TERM"); + const wrote = await m.sh( + `cd ~/codework && ~/.local/node/bin/node --conditions=development apps/server/test/machines/seed.ts ${m.database} ${days} ${seed}`, + ); + await m.start(live); + return JSON.parse(wrote.trim().split("\n").at(-1)!) as Record; +}; + +type Ref = { readonly sessionId: string }; +const turn = (m: Machine, ref: Ref, text: string) => + m.run((rpc) => { + const sessionId = Session.SessionSchema.ID.make(ref.sessionId); + return rpc["thread.prompt"]({ sessionId, text }).pipe(Effect.andThen(rpc["thread.wait"]({ sessionId }))); + }); +/** New work on `m`: a new session in a project with `turns` tool turns (two requests each). */ +const work = async (m: Machine, turns: number) => { + const created = await m.run((rpc) => + rpc["session.create"]({ + directory: `${m.cwHome}/work/codework`, + title: "Live work", + project: { name: "codework" }, + }), + ); + for (let i = 0; i < turns; i++) await turn(m, { sessionId: created.session.id }, `bash echo live ${i}`); + return { sessionId: created.session.id as string }; +}; + +/** The app's latest warnings, for a wait that timed out. */ +const warnings = () => { + const last = desk.lines.findLastIndex((line) => line.includes("WARN") || line.includes("ERROR")); + return last === -1 + ? "" + : desk.lines + .slice(last, last + 12) + .join("\n") + .slice(0, 3_000); +}; + +/** Waits until the app's usage and index cursors for `m` reached its head, which `m` must be serving. */ +const rolled = (m: Machine, { index = true, timeout = 180_000 } = {}) => + caught(m, index, timeout).catch((error: Error) => { + throw new Error(`${error.message}\n${warnings()}`); + }); +const caught = async (m: Machine, index: boolean, timeout: number) => { + const { storeId, rev } = await m.run((rpc) => rpc["server.info"]({})); + await waitFor( + () => { + const row = cursorOf(appHome, idOf(m)); + return row.rollup_store === storeId && row.rollup >= rev ? row : undefined; + }, + `${m.name} rolled up at rev ${rev}`, + timeout, + ); + if (index) + await waitFor( + () => { + const row = machineRow(appHome, idOf(m)); + return row?.store_id === storeId && row.cursor >= rev ? row : undefined; + }, + `${m.name} indexed at rev ${rev}`, + timeout, + ); + return { storeId, rev }; +}; + +/** What `usage.sync` past `after` sends up to its first `Synced`: what a resume should apply. */ +const pending = async (m: Machine, after: number) => + ( + await m.run((rpc) => + rpc["usage.sync"]({ after }).pipe( + Stream.takeUntil((item) => item._tag === "Synced"), + Stream.runCollect, + ), + ) + ).filter((item) => item._tag !== "Synced").length; + +interface Truth { + readonly requests: number; + readonly tokens: number; + readonly cost: number; + readonly timed: number; + readonly prompts: number; + readonly tools: number; +} +const none: Truth = { requests: 0, tokens: 0, cost: 0, timed: 0, prompts: 0, tools: 0 }; +const plus = (a: Truth, b: Truth): Truth => ({ + requests: a.requests + b.requests, + tokens: a.tokens + b.tokens, + cost: a.cost + b.cost, + timed: a.timed + b.timed, + prompts: a.prompts + b.prompts, + tools: a.tools + b.tools, +}); +/** The machine's rollup totals, read inside it. */ +const truth = async (m: Machine): Promise => { + const [usage] = await m.sql(` + SELECT COALESCE(SUM(requests), 0) AS requests, COALESCE(SUM(input + output + cache_read + cache_write), 0) AS tokens, + COALESCE(SUM(cost), 0) AS cost, COALESCE(SUM(timed), 0) AS timed FROM usage + `); + const [tally] = await m.sql(` + SELECT COALESCE(SUM(CASE kind WHEN 'prompt' THEN count END), 0) AS prompts, + COALESCE(SUM(CASE kind WHEN 'tool' THEN count END), 0) AS tools FROM tally + `); + return { + requests: usage!.requests as number, + tokens: usage!.tokens as number, + cost: usage!.cost as number, + timed: usage!.timed as number, + prompts: tally!.prompts as number, + tools: tally!.tools as number, + }; +}; +const read = (request: Parameters[0]) => desk.run((rpc) => rpc["metrics.usage"](request)); +/** What a metrics read adds up to. */ +const shown = (metrics: MetricsUsage): Truth => ({ + requests: metrics.rows.reduce((sum, row) => sum + row.requests, 0), + tokens: metrics.rows.reduce( + (sum, row) => sum + row.tokens.input + row.tokens.output + row.tokens.cacheRead + row.tokens.cacheWrite, + 0, + ), + cost: metrics.rows.reduce((sum, row) => sum + row.cost.total, 0), + timed: metrics.rows.reduce((sum, row) => sum + row.timed, 0), + prompts: metrics.tallies.filter((t) => t.kind === "prompt").reduce((sum, t) => sum + t.count, 0), + tools: metrics.tallies.filter((t) => t.kind === "tool").reduce((sum, t) => sum + t.count, 0), +}); +const expectTotals = (actual: Truth, expected: Truth) => { + expect({ ...actual, cost: 0 }).toEqual({ ...expected, cost: 0 }); + expect(actual.cost).toBeCloseTo(expected.cost, 6); +}; +/** The app's metrics equal what the machines hold, in every zone and per machine; returns the totals. */ +const agrees = async (expected: ReadonlyArray) => { + const all = expected.reduce((sum, [, t]) => plus(sum, t), none); + for (const zone of ["UTC", "Asia/Kolkata", "Asia/Kathmandu", "America/Los_Angeles", "Pacific/Chatham"]) + expectTotals(shown(await read(everything(zone))), all); + for (const [m, t] of expected) + expectTotals(shown(await read({ ...everything(), machineId: typeof m === "string" ? m : idOf(m) })), t); + // The projects split the whole without a remainder. + const whole = await read(everything()); + let byProject = none; + for (const project of whole.projects) + byProject = plus(byProject, shown(await read({ ...everything(), projectId: project.id }))); + expectTotals(byProject, all); + return { all, sessions: whole.sessions.length, deleted: whole.sessions.filter((s) => s.deleted).length }; +}; + +const cases: Record = {}; +const describeMachines = available ? describe : describe.skip; + +describeMachines("metrics simulation across OrbStack machines", () => { + beforeAll(async () => { + for (const m of pool) await ensure(m.name); + await Promise.all(pool.map((m) => m.setup())); + const [a, b] = await Promise.all([seeded(cwA, 300, 365, 11), seeded(cwB, 120, 200, 23)]); + cases.seeded = { "cw-a": a, "cw-b": b }; + appHome = tempDir("metrics-simulation-app"); + desk = await app(appHome); + for (const m of pool) + ids.set(m.name, await desk.run((rpc) => rpc["servers.add"]({ kind: "remote", url: m.url }))); + }, 3_600_000); + + afterAll(async () => { + await Promise.all(pool.map((m) => m.net("restore").catch(() => undefined))); + await cleanup(); + }, 120_000); + + it("the first connect mirrors both histories, deleted sessions keeping their numbers", async () => { + for (const m of pool) await rolled(m); + const before = await agrees([ + [cwA, await truth(cwA)], + [cwB, await truth(cwB)], + ]); + const doomed = await cwA.sql("SELECT id FROM session ORDER BY id LIMIT 7"); + for (const { id } of doomed) + await cwA.run((rpc) => rpc["session.delete"]({ sessionId: Session.SessionSchema.ID.make(id as string) })); + await rolled(cwA); + const after = await agrees([ + [cwA, await truth(cwA)], + [cwB, await truth(cwB)], + ]); + expect(after.all).toEqual(before.all); + expect(after.deleted).toBe(before.deleted + 7); + cases.connected = { before, after }; + }, 600_000); + + it("live turns show while connected", async () => { + const at = lastSeq(appHome); + await work(cwA, 3); + await rolled(cwA); + const today = new Date().toISOString().slice(0, 10); + const day = await read({ zone: "UTC", since: today, until: today, machineId: idOf(cwA) }); + expect(shown(day).requests).toBeGreaterThanOrEqual(6); + expect(notices(appHome, Contract.syncTopics.usage, at).length).toBeGreaterThan(0); + cases.live = await agrees([ + [cwA, await truth(cwA)], + [cwB, await truth(cwB)], + ]); + }, 600_000); + + it("the app down while both machines work resumes from its cursors", async () => { + const cursors = pool.map((m) => cursorOf(appHome, idOf(m)).rollup); + await stop(desk.proc, "SIGTERM"); + await Promise.all([work(cwA, 2), work(cwB, 4)]); + const expected = await Promise.all(pool.map((m, at) => pending(m, cursors[at]!))); + const restartedAt = lastSeq(appHome); + desk = await app(appHome); + for (const m of pool) await rolled(m); + const applied = pool.map((m) => + notices(appHome, Contract.syncTopics.usage, restartedAt) + .filter((payload) => payload.machineId === idOf(m)) + .reduce((sum, payload) => sum + (payload.items as number), 0), + ); + expect(applied).toEqual(expected); + cases.resumed = { + applied, + totals: await agrees([ + [cwA, await truth(cwA)], + [cwB, await truth(cwB)], + ]), + }; + }, 600_000); + + it("a paused machine keeps its history on screen and resumes without a re-read", async () => { + const b = await truth(cwB); + await orb(["stop", cwB.name], { timeout: 300_000 }); + await work(cwA, 2); + await rolled(cwA); + const paused = await agrees([ + [cwA, await truth(cwA)], + [cwB, b], + ]); + const cursor = cursorOf(appHome, idOf(cwB)).rollup; + const at = lastSeq(appHome); + await orb(["start", cwB.name], { timeout: 300_000 }); + await cwB.start(live); + await rolled(cwB); + const reread = notices(appHome, Contract.syncTopics.usage, at).filter((p) => p.machineId === idOf(cwB)); + expect(reread.reduce((sum, p) => sum + (p.items as number), 0)).toBe(await pending(cwB, cursor)); + expect(await truth(cwB)).toEqual(b); + cases.paused = { + paused, + resumed: await agrees([ + [cwA, await truth(cwA)], + [cwB, b], + ]), + }; + }, 900_000); + + it("the app down across a machine's pause and restart, with work after it, resumes too", async () => { + await stop(desk.proc, "SIGTERM"); + const cursor = cursorOf(appHome, idOf(cwA)).rollup; + await orb(["stop", cwA.name], { timeout: 300_000 }); + await orb(["start", cwA.name], { timeout: 300_000 }); + await cwA.start(live); + await work(cwA, 2); + const expected = await pending(cwA, cursor); + const at = lastSeq(appHome); + desk = await app(appHome); + for (const m of pool) await rolled(m); + const applied = notices(appHome, Contract.syncTopics.usage, at) + .filter((p) => p.machineId === idOf(cwA)) + .reduce((sum, p) => sum + (p.items as number), 0); + expect(applied).toBe(expected); + cases.restartedAway = await agrees([ + [cwA, await truth(cwA)], + [cwB, await truth(cwB)], + ]); + }, 900_000); + + it("work during a network cut arrives once the network is back", async () => { + await cwA.net("cut"); + await cwA.drive([["session.create", { directory: `${cwA.cwHome}/work/infra`, title: "Offline work" }]]); + const [{ id }] = (await cwA.sql("SELECT id FROM session ORDER BY created_at DESC LIMIT 1")) as [Row]; + await cwA.drive([ + ["thread.prompt", { sessionId: id, text: "bash echo offline" }], + ["thread.wait", { sessionId: id }], + ]); + await cwA.net("restore"); + await rolled(cwA); + cases.cut = await agrees([ + [cwA, await truth(cwA)], + [cwB, await truth(cwB)], + ]); + }, 600_000); + + it("a backup restored after its head passed the app's cursor replaces that store's rows", async () => { + await cwA.stop("TERM"); + await cwA.sh(`cp ${cwA.database} ${cwA.cwHome}/backup.db`); + await cwA.start(live); + const lost = await work(cwA, 2); + const { rev: passed } = await rolled(cwA); + await cwA.stop("TERM"); + await cwA.sh( + `for f in ${cwA.database}-wal ${cwA.database}-shm; do [ -e $f ] && mv $f $f.lost; done; cp ${cwA.cwHome}/backup.db ${cwA.database}`, + ); + await cwA.start(live); + while ((await cwA.run((rpc) => rpc["server.info"]({}))).rev < passed) await work(cwA, 1); + // The index does not recover from this yet: its resume skips the revisions the restore rewrote + // and its batches fail on a thread whose session never came. Usage has its own stream. + await rolled(cwA, { index: false }); + const ghost = query(appHome, "SELECT COUNT(*) AS n FROM usage WHERE session_id = ?", lost.sessionId)[0]!.n; + expect(ghost).toBe(0); + cases.restored = await agrees([ + [cwA, await truth(cwA)], + [cwB, await truth(cwB)], + ]); + }, 900_000); + + it("a removed server's history stays; added again, it is counted once", async () => { + const b = await truth(cwB); + const a = await truth(cwA); + const removed = idOf(cwB); + await desk.run((rpc) => rpc["servers.remove"]({ id: removed })); + const kept = await agrees([ + [cwA, a], + [removed, b], + ]); + ids.set(cwB.name, await desk.run((rpc) => rpc["servers.add"]({ kind: "remote", url: cwB.url }))); + await rolled(cwB); + const again = await agrees([ + [cwA, a], + [cwB, b], + ]); + expect(again.all).toEqual(kept.all); + cases.readded = { kept, again }; + }, 900_000); + + it("a recreated store adds to the old one's history", async () => { + const old = await truth(cwB); + const a = await truth(cwA); + await aside(cwB); + await cwB.start(live); + await cwB.sh(`mkdir -p ${cwB.cwHome}/work/codework`); + await work(cwB, 2); + await rolled(cwB); + const fresh = await truth(cwB); + cases.recreated = await agrees([ + [cwA, a], + [cwB, plus(old, fresh)], + ]); + await snapshot("metrics.simulation", cases); + }, 900_000); +}); diff --git a/apps/server/test/metrics.test.ts b/apps/server/test/metrics.test.ts new file mode 100644 index 00000000..2d93518b --- /dev/null +++ b/apps/server/test/metrics.test.ts @@ -0,0 +1,486 @@ +/* @effect-diagnostics nodeBuiltinImport:off -- the suite makes project folders and removes CodeWork stores in temp dirs. */ +import { Effect } from "effect"; +import { mkdirSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { afterEach, expect, it } from "vite-plus/test"; +import { Contract, type MetricsHour, type MetricsRow, type MetricsUsage } from "@codeworksh/contract"; +import { edges } from "@codeworksh/widget/codework"; +import type * as Client from "../../../packages/codework/src/server/client.ts"; +import { + app, + type AppRpc, + cleanup, + type Codework, + codework, + follow, + freePort, + normalizer, + query, + rolled, + snapshot as match, + stop, + synced, + tempDir, + waitFor, + write, +} from "./fixtures/apps.ts"; + +afterEach(cleanup); + +const snapshot = (name: string, artifact: unknown) => match(`metrics.${name}`, artifact); + +type Desk = Awaited>; +type Scope = Omit[0], "since" | "until">; + +// The scripted model's requests land in the first hour of 1970 and prompts in the real one, so a +// period covering every row runs from then to tomorrow. +const DAY = 86_400_000; +const today = new Date().toISOString().slice(0, 10); +const everything = { zone: "UTC", since: "1970-01-01", until: new Date(Date.now() + DAY).toISOString().slice(0, 10) }; + +const read = (desk: Desk, request: Parameters[0]) => + desk.run((rpc) => rpc["metrics.usage"](request)); +const hoursOf = (desk: Desk, request: Parameters[0]) => + desk.run((rpc) => rpc["metrics.hours"](request)); + +/** What the rows add up to. */ +const sum = (rows: ReadonlyArray) => + rows.reduce( + (total, row) => ({ + requests: total.requests + row.requests, + input: total.input + row.tokens.input, + output: total.output + row.tokens.output, + cacheRead: total.cacheRead + row.tokens.cacheRead, + cacheWrite: total.cacheWrite + row.tokens.cacheWrite, + cost: total.cost + row.cost.total, + }), + { requests: 0, input: 0, output: 0, cacheRead: 0, cacheWrite: 0, cost: 0 }, + ); +const counts = (metrics: MetricsUsage) => { + const by = new Map(); + for (const tally of metrics.tallies) + by.set(`${tally.kind}:${tally.name}`, (by.get(`${tally.kind}:${tally.name}`) ?? 0) + tally.count); + return Object.fromEntries(by); +}; + +/** Projects by name: their ids are their folders' hashes, which differ each run. */ +const named = (metrics: MetricsUsage) => ({ + ...metrics, + projects: metrics.projects.toSorted((a, b) => (a.name ?? "").localeCompare(b.name ?? "")), +}); + +/** A session in `directory` of project `name` whose prompt runs one `bash` call. */ +const session = async (server: Codework, directory: string, name: string, command: string) => { + mkdirSync(directory, { recursive: true }); + const created = await server.run((rpc) => rpc["session.create"]({ directory, project: { name } })); + await turn(server, { sessionId: created.session.id }, command); + return created; +}; +const turn = (server: Codework, ref: Parameters[0], command: string) => + server.run((rpc) => + rpc["thread.prompt"]({ ...ref, text: `bash ${command}` }).pipe(Effect.andThen(rpc["thread.wait"](ref))), + ); + +/** Adds `server` with no name of its own: it is named by the host it reports. */ +const addServer = async (home: string, server: Codework, desk: Desk) => { + const id = await desk.run((rpc) => rpc["servers.add"]({ kind: "remote", url: server.url })); + await synced(home, id, server); + return id; +}; + +/** Each project id a machine's store holds, by name, for the artifacts' labels. */ +const projectLabels = (home: string) => + query(home, "SELECT DISTINCT id, name FROM project ORDER BY name").map( + (row) => [row.id as string, ``] as const, + ); + +it("a period's usage sums each session's threads, counts its threads per day and names its sessions, projects and machine; a deleted session keeps its rows", async () => { + const cwHome = tempDir("metrics-cw"); + const appHome = tempDir("metrics-app"); + const server = await codework(cwHome, await freePort(), { host: "Metrics Host" }); + const desk = await app(appHome); + const machineId = await addServer(appHome, server, desk); + const topics = await follow(desk, Contract.readTopics.metrics); + + // A session with a fork that runs a turn of its own, and a second session in another project. + const forked = await session(server, join(cwHome, "alpha"), "Alpha", "echo one"); + const fork = await server.run((rpc) => + Effect.gen(function* () { + const ref = { sessionId: forked.session.id }; + const reply = (yield* rpc["thread.history"](ref)).entries.at(-1)!; + return (yield* rpc["thread.fork"]({ ...ref, entryId: reply.id })).thread; + }), + ); + await turn(server, { sessionId: forked.session.id, threadId: fork.id }, "echo fork"); + const other = await session(server, join(cwHome, "beta"), "Beta", "echo two"); + const { storeId } = await rolled(appHome, machineId, server); + const notice = await waitFor( + () => topics.events.find((event) => event.topic === Contract.syncTopics.usage), + "a usage notice", + ); + topics.stop(); + + const reported = await server.run((rpc) => + Effect.forEach( + [ + { sessionId: forked.session.id, threadId: forked.id }, + { sessionId: forked.session.id, threadId: fork.id }, + { sessionId: other.session.id, threadId: other.id }, + ], + (ref) => rpc["thread.info"](ref).pipe(Effect.map((thread) => thread.usage)), + ), + ); + const before = await read(desk, everything); + const totals = sum(before.rows); + expect(totals).toEqual({ + requests: 6, + input: reported.reduce((all, usage) => all + usage.input, 0), + output: reported.reduce((all, usage) => all + usage.output, 0), + cacheRead: reported.reduce((all, usage) => all + usage.cacheRead, 0), + cacheWrite: reported.reduce((all, usage) => all + usage.cacheWrite, 0), + cost: reported.reduce((all, usage) => all + usage.cost, 0), + }); + expect(counts(before)).toEqual({ "prompt:": 3, "tool:bash": 3 }); + + // CodeWork starts no subagents yet, so the rollup row is written as the sync would copy one: the fork's, under a thread of its own. + const columns = query(appHome, "SELECT name FROM pragma_table_info('usage')").map((row) => row.name as string); + write( + appHome, + `INSERT INTO usage (${columns.join(", ")}) + SELECT ${columns.map((name) => (name === "thread_id" ? "'thr_subagent'" : name)).join(", ")} FROM usage WHERE thread_id = ?`, + fork.id, + ); + const after = await read(desk, everything); + const threadsOf = (metrics: MetricsUsage, sessionId: string) => + metrics.threads.filter((row) => row.sessionId === sessionId).map((row) => [row.day, row.threads]); + // Requests and tool calls in 1970, prompts today. + expect(threadsOf(after, forked.session.id)).toEqual([ + ["1970-01-01", 3], + [today, 2], + ]); + expect(threadsOf(after, other.session.id)).toEqual([ + ["1970-01-01", 1], + [today, 1], + ]); + expect(sum(after.rows).requests).toBe(8); + expect(after.rows.filter((row) => row.sessionId === forked.session.id)).toHaveLength(1); + expect(after.machines).toEqual([{ id: machineId, name: "Metrics Host" }]); + expect(named(after).projects.map((project) => project.name)).toEqual(["Alpha", "Beta"]); + expect( + after.sessions + .map(({ id, title, deleted }) => ({ id, title, deleted })) + .toSorted((a, b) => a.id.localeCompare(b.id)), + ).toEqual( + [ + { id: forked.session.id, title: "bash echo one", deleted: false }, + { id: other.session.id, title: "bash echo two", deleted: false }, + ].toSorted((a, b) => a.id.localeCompare(b.id)), + ); + expect(after.edges).toEqual([...edges]); + + // A deleted session: its rows stay, its title goes, its project is still named. + await server.run((rpc) => rpc["session.delete"]({ sessionId: other.session.id })); + await synced(appHome, machineId, server); + const deleted = await read(desk, everything); + expect(deleted.rows).toEqual(after.rows); + expect(deleted.sessions.find((entry) => entry.id === other.session.id)).toMatchObject({ + title: null, + deleted: true, + }); + expect(deleted.projects).toEqual(after.projects); + + await snapshot( + "usage", + normalizer( + [cwHome, appHome], + [...projectLabels(appHome), [today, ""], [storeId, ""]], + )({ + notice: notice.payload, + reported, + after: named(after), + deleted: { sessions: deleted.sessions }, + }), + ); +}, 120_000); + +/** A usage row as `usage.sync` would leave it: `requests` requests, each of 10 input and 5 output tokens and $0.5. */ +const put = ( + home: string, + row: { + hour: number; + project: string; + session: string; + thread?: string; + store?: string; + model?: string; + requests?: number; + }, +) => { + const requests = row.requests ?? 1; + const durations = Array.from({ length: edges.length + 1 }, (_, bucket) => (bucket === 1 ? requests : 0)); + write( + home, + `INSERT INTO usage (machine_id, store_id, hour, thread_id, provider, model, thinking, session_id, project_id, + requests, input, output, cache_read, cache_write, reasoning, cost, cost_input, cost_output, cost_cache_read, + cost_cache_write, duration, timed, durations, rev) + VALUES ('local', ?, ?, ?, 'scripted', ?, 'off', ?, ?, ?, ?, ?, 0, 0, 0, ?, ?, ?, 0, 0, ?, ?, ?, 1)`, + row.store ?? "store_one", + row.hour, + row.thread ?? `thr_${row.session}`, + row.model ?? "model", + row.session, + row.project, + requests, + requests * 10, + requests * 5, + requests * 0.5, + requests * 0.25, + requests * 0.25, + requests * 300, + requests, + JSON.stringify(durations), + ); +}; +const prompt = (home: string, row: { hour: number; project: string; session: string; thread: string }) => + write( + home, + `INSERT INTO tally (machine_id, store_id, hour, thread_id, kind, name, session_id, project_id, count, rev) + VALUES ('local', 'store_one', ?, ?, 'prompt', '', ?, ?, 1, 1)`, + row.hour, + row.thread, + row.session, + row.project, + ); + +const utc = (month: number, day: number, hour: number) => Date.UTC(2026, month - 1, day, hour); +const days = (metrics: MetricsUsage) => + Object.fromEntries( + [...new Set(metrics.rows.map((row) => row.day))].map((day) => [ + day, + sum(metrics.rows.filter((row) => row.day === day)).requests, + ]), + ); +const hourly = (hours: ReadonlyArray) => hours.map((row) => [row.hour, row.requests]); + +it("usage lands on the zone's local days and hours: half-hour offsets, both sides of midnight, clock changes, and stores summed", async () => { + const appHome = tempDir("metrics-zones-app"); + const desk = await app(appHome); + + // 18:00 UTC is 23:30 in Kolkata and 23:45 in Kathmandu; 19:00 UTC is 00:30 and 00:45 the next day. + put(appHome, { hour: utc(3, 9, 18), project: "prj_india", session: "india", requests: 1 }); + put(appHome, { hour: utc(3, 9, 19), project: "prj_india", session: "india", requests: 2 }); + // 06:00 UTC is 23:00 the day before in Los Angeles (PDT), 07:00 UTC its midnight. + put(appHome, { hour: utc(3, 10, 6), project: "prj_la", session: "la", requests: 1 }); + put(appHome, { hour: utc(3, 10, 7), project: "prj_la", session: "la", requests: 2 }); + // Every UTC hour from the last before Los Angeles' 1 November to the first after: it is 25 hours long. + for (let hour = utc(11, 1, 6); hour <= utc(11, 2, 8); hour += 3_600_000) + put(appHome, { hour, project: "prj_fall", session: "fall" }); + // And around 8 March, which is 23 hours long. + for (let hour = utc(3, 8, 7); hour <= utc(3, 9, 7); hour += 3_600_000) + put(appHome, { hour, project: "prj_spring", session: "spring" }); + // One hour of a session from two stores of the machine, and a second thread in the newer one. + put(appHome, { hour: utc(5, 1, 12), project: "prj_stores", session: "stores", store: "store_old", requests: 1 }); + put(appHome, { hour: utc(5, 1, 12), project: "prj_stores", session: "stores", store: "store_new", requests: 2 }); + put(appHome, { + hour: utc(5, 1, 13), + project: "prj_stores", + session: "stores", + store: "store_new", + thread: "thr_second", + requests: 4, + }); + prompt(appHome, { hour: utc(3, 9, 19), project: "prj_india", session: "india", thread: "thr_india" }); + + const india = { projectId: "prj_india" }; + const kolkata = { ...india, zone: "Asia/Kolkata" }; + const la = { projectId: "prj_la", zone: "America/Los_Angeles" }; + const fall = { projectId: "prj_fall", zone: "America/Los_Angeles" }; + const spring = { projectId: "prj_spring", zone: "America/Los_Angeles" }; + const span = (scope: Scope, since: string, until = since) => read(desk, { ...scope, since, until }); + const results = { + kolkata: { + both: days(await span(kolkata, "2026-03-09", "2026-03-10")), + first: days(await span(kolkata, "2026-03-09")), + second: days(await span(kolkata, "2026-03-10")), + tallies: (await span(kolkata, "2026-03-10")).tallies, + hoursFirst: hourly(await hoursOf(desk, { ...kolkata, day: "2026-03-09" })), + hoursSecond: hourly(await hoursOf(desk, { ...kolkata, day: "2026-03-10" })), + }, + kathmandu: { + both: days(await span({ ...india, zone: "Asia/Kathmandu" }, "2026-03-09", "2026-03-10")), + hours: hourly(await hoursOf(desk, { ...india, zone: "Asia/Kathmandu", day: "2026-03-10" })), + }, + utc: days(await span({ ...india, zone: "UTC" }, "2026-03-09", "2026-03-10")), + losAngeles: { + both: days(await span(la, "2026-03-09", "2026-03-10")), + second: days(await span(la, "2026-03-10")), + hoursFirst: hourly(await hoursOf(desk, { ...la, day: "2026-03-09" })), + hoursSecond: hourly(await hoursOf(desk, { ...la, day: "2026-03-10" })), + }, + fallBack: { + days: days(await span(fall, "2026-10-31", "2026-11-02")), + hours: hourly(await hoursOf(desk, { ...fall, day: "2026-11-01" })), + }, + springForward: { + days: days(await span(spring, "2026-03-07", "2026-03-09")), + hours: hourly(await hoursOf(desk, { ...spring, day: "2026-03-08" })), + }, + stores: await span({ projectId: "prj_stores", zone: "UTC" }, "2026-05-01"), + }; + + expect(results.kolkata).toMatchObject({ + both: { "2026-03-09": 1, "2026-03-10": 2 }, + first: { "2026-03-09": 1 }, + second: { "2026-03-10": 2 }, + hoursFirst: [[23, 1]], + hoursSecond: [[0, 2]], + }); + expect(results.kolkata.tallies.map((tally) => [tally.day, tally.kind, tally.count])).toEqual([ + ["2026-03-10", "prompt", 1], + ]); + expect(results.kathmandu).toEqual({ both: { "2026-03-09": 1, "2026-03-10": 2 }, hours: [[0, 2]] }); + expect(results.utc).toEqual({ "2026-03-09": 3 }); + expect(results.losAngeles).toEqual({ + both: { "2026-03-09": 1, "2026-03-10": 2 }, + second: { "2026-03-10": 2 }, + hoursFirst: [[23, 1]], + hoursSecond: [[0, 2]], + }); + // 27 UTC hours: one on each neighbour, and all 25 of the long day, whose 1 AM came twice. + expect(results.fallBack.days).toEqual({ "2026-10-31": 1, "2026-11-01": 25, "2026-11-02": 1 }); + expect(results.fallBack.hours).toEqual(Array.from({ length: 24 }, (_, hour) => [hour, hour === 1 ? 2 : 1])); + // 25 UTC hours: one on each neighbour, and the short day's 23, with no 2 AM. + expect(results.springForward.days).toEqual({ "2026-03-07": 1, "2026-03-08": 23, "2026-03-09": 1 }); + expect(results.springForward.hours).toEqual( + Array.from({ length: 24 }, (_, hour) => hour) + .filter((hour) => hour !== 2) + .map((hour) => [hour, 1]), + ); + expect(results.stores.rows).toHaveLength(1); + expect(results.stores.rows[0]).toMatchObject({ requests: 7, timed: 7, tokens: { input: 70 } }); + expect(results.stores.rows[0]!.durations[1]).toBe(7); + expect(results.stores.threads).toEqual([{ day: "2026-05-01", machineId: "local", sessionId: "stores", threads: 2 }]); + expect(results.stores.sessions).toEqual([ + { id: "stores", machineId: "local", projectId: "prj_stores", title: null, deleted: true }, + ]); + expect(results.stores.projects).toEqual([{ id: "prj_stores", name: null }]); + expect(results.stores.machines).toEqual([{ id: "local", name: "This Machine" }]); + + // Bad input is refused with the reason. + const refused = async ( + effect: (rpc: AppRpc) => Effect.Effect, + ) => { + const error = await desk.run((rpc) => effect(rpc).pipe(Effect.flip)); + return { tag: error._tag, message: error.message }; + }; + const refusals = { + zone: await refused((rpc) => + rpc["metrics.usage"]({ zone: "Mars/Olympus", since: "2026-03-09", until: "2026-03-09" }), + ), + backwards: await refused((rpc) => + rpc["metrics.usage"]({ zone: "UTC", since: "2026-03-10", until: "2026-03-09" }), + ), + malformed: await refused((rpc) => rpc["metrics.usage"]({ zone: "UTC", since: "2026-3-9", until: "2026-03-09" })), + impossible: await refused((rpc) => rpc["metrics.hours"]({ zone: "UTC", day: "2026-02-30" })), + hoursZone: await refused((rpc) => rpc["metrics.hours"]({ zone: "", day: "2026-03-09" })), + }; + expect(Object.values(refusals).map((refusal) => refusal.tag)).toEqual(Array(5).fill("Server.MetricsError")); + + await snapshot("zones", { ...results, refusals }); +}, 60_000); + +it("filters by machine and by project; without a filter every machine and project is summed", async () => { + const homeA = tempDir("metrics-filter-a"); + const homeB = tempDir("metrics-filter-b"); + const appHome = tempDir("metrics-filter-app"); + const serverA = await codework(homeA, await freePort(), { host: "Host A" }); + const serverB = await codework(homeB, await freePort(), { host: "Host B" }); + const desk = await app(appHome); + const idA = await addServer(appHome, serverA, desk); + const idB = await addServer(appHome, serverB, desk); + await session(serverA, join(homeA, "alpha"), "Alpha", "echo alpha"); + await session(serverA, join(homeA, "beta"), "Beta", "echo beta"); + await session(serverB, join(homeB, "gamma"), "Gamma", "echo gamma"); + await rolled(appHome, idA, serverA); + await rolled(appHome, idB, serverB); + + const projectId = (name: string) => query(appHome, "SELECT id FROM project WHERE name = ?", name)[0]!.id as string; + const all = await read(desk, everything); + const onA = await read(desk, { ...everything, machineId: idA }); + const onB = await read(desk, { ...everything, machineId: idB }); + const alpha = await read(desk, { ...everything, projectId: projectId("Alpha") }); + const alphaOnB = await read(desk, { ...everything, machineId: idB, projectId: projectId("Alpha") }); + const hoursA = await hoursOf(desk, { zone: "UTC", day: "1970-01-01", machineId: idA }); + + expect(new Set(onA.rows.map((row) => row.machineId))).toEqual(new Set([idA])); + expect(new Set(onB.rows.map((row) => row.machineId))).toEqual(new Set([idB])); + expect(sum(all.rows).requests).toBe(6); + expect(sum(onA.rows).requests + sum(onB.rows).requests).toBe(sum(all.rows).requests); + expect(counts(onA)).toEqual({ "prompt:": 2, "tool:bash": 2 }); + expect(all.machines.map((machine) => machine.name).toSorted()).toEqual(["Host A", "Host B"]); + expect(named(all).projects.map((project) => project.name)).toEqual(["Alpha", "Beta", "Gamma"]); + expect(named(onA).projects.map((project) => project.name)).toEqual(["Alpha", "Beta"]); + expect(alpha.projects.map((project) => project.name)).toEqual(["Alpha"]); + expect(alpha.sessions).toHaveLength(1); + expect(sum(alpha.rows).requests).toBe(2); + expect(alphaOnB).toEqual({ + rows: [], + tallies: [], + threads: [], + sessions: [], + projects: [], + machines: [], + edges: [...edges], + }); + expect(new Set(hoursA.map((row) => row.machineId))).toEqual(new Set([idA])); + expect(hoursA.reduce((all, row) => all + row.requests, 0)).toBe(4); + + await snapshot( + "filters", + normalizer( + [homeA, homeB, appHome], + [...projectLabels(appHome), [today, ""]], + )({ + // Machine ids are random, and rows go by them: each machine's read is kept under its name. + "Host A": named(onA), + "Host B": named(onB), + all: { rows: all.rows.length, machines: all.machines.length, projects: named(all).projects }, + alpha: { sessions: alpha.sessions, projects: alpha.projects }, + hoursA, + }), + ); +}, 120_000); + +it("a machine's old and new stores read as one machine", async () => { + const cwHome = tempDir("metrics-store-cw"); + const appHome = tempDir("metrics-store-app"); + const port = await freePort(); + let server = await codework(cwHome, port); + const desk = await app(appHome); + const machineId = await addServer(appHome, server, desk); + await session(server, join(cwHome, "old"), "Old", "echo old"); + const old = await rolled(appHome, machineId, server); + + await stop(server.proc, "SIGTERM"); + for (const suffix of ["", "-wal", "-shm"]) rmSync(`${server.database}${suffix}`, { force: true }); + server = await codework(cwHome, port); + await session(server, join(cwHome, "new"), "New", "echo new"); + const fresh = await rolled(appHome, machineId, server); + expect(fresh.storeId).not.toBe(old.storeId); + expect(query(appHome, "SELECT DISTINCT store_id FROM usage")).toHaveLength(2); + + const metrics = await read(desk, everything); + expect(metrics.machines).toEqual([{ id: machineId, name: "Fixture" }]); + expect(new Set(metrics.rows.map((row) => row.machineId))).toEqual(new Set([machineId])); + expect(metrics.sessions).toHaveLength(2); + expect(sum(metrics.rows).requests).toBe(4); + expect(counts(metrics)).toEqual({ "prompt:": 2, "tool:bash": 2 }); + + await snapshot( + "stores", + normalizer( + [cwHome, appHome], + [...projectLabels(appHome), [today, ""], [old.storeId, ""], [fresh.storeId, ""]], + )(named(metrics)), + ); +}, 120_000); diff --git a/apps/server/test/reads.test.ts b/apps/server/test/reads.test.ts index c7e49aab..2b11277a 100644 --- a/apps/server/test/reads.test.ts +++ b/apps/server/test/reads.test.ts @@ -11,6 +11,7 @@ import { cleanup, type Codework, codework, + follow, freePort, normalizer, query, @@ -77,24 +78,6 @@ const registerLocal = (appHome: string, server: Codework, port: number) => { const sidebar = (rpc: AppRpc, machines: "default" | "all" = "all") => rpc["sidebar.list"]({ organizationId: Contract.PERSONAL_ORGANIZATION, machines }); -/** Events on `topics` from the moment it resolves, collected until `stop`. */ -const follow = async (desk: Awaited>, topics: ReadonlyArray) => { - const events: Array<{ topic: string; payload: Row }> = []; - let subscribed = false; - const background = desk.background((rpc) => - rpc["event.subscribe"]({ topics }).pipe( - Stream.runForEach((item) => - Effect.sync(() => { - if ("_tag" in item) subscribed = true; - else events.push({ topic: item.topic, payload: item.payload as Row }); - }), - ), - ), - ); - await waitFor(() => (subscribed ? true : undefined), `a subscription to ${topics.join(", ")}`); - return { events, stop: background.stop }; -}; - it("the sidebar reads app.db alone: a run started before the app shows running at launch, a fork's activity moves its session up, and it stays while CodeWork is stopped", async () => { const cwHome = tempDir("sidebar-cw"); const appHome = tempDir("sidebar-app"); diff --git a/apps/server/test/schema.test.ts b/apps/server/test/schema.test.ts index 012e30f4..3298d512 100644 --- a/apps/server/test/schema.test.ts +++ b/apps/server/test/schema.test.ts @@ -90,11 +90,21 @@ const insertTree = (db: DatabaseSync, machine: string, suffix: string, groupingI `INSERT INTO inbox (id, thread_id, delivery, text, admitted_seq, created_at) VALUES (?, ?, 'followUp', 'next', 2, ?)`, ).run(`msg_${suffix}`, `thr_${suffix}`, now); + db.prepare( + `INSERT INTO usage (machine_id, store_id, hour, thread_id, provider, model, thinking, session_id, project_id, + requests, input, output, cache_read, cache_write, reasoning, cost, cost_input, cost_output, cost_cache_read, + cost_cache_write, duration, timed, durations, rev) + VALUES (?, 'str', 0, ?, 'openai', 'gpt', 'off', ?, 'prj', 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, '[1]', 1)`, + ).run(machine, `thr_${suffix}`, `ses_${suffix}`); + db.prepare( + `INSERT INTO tally (machine_id, store_id, hour, thread_id, kind, name, session_id, project_id, count, rev) + VALUES (?, 'str', 0, ?, 'tool', 'bash', ?, 'prj', 1, 1)`, + ).run(machine, `thr_${suffix}`, `ses_${suffix}`); }; -const TREE = ["project", "space", "session", "thread", "entry", "part", "inbox"] as const; +const TREE = ["project", "space", "session", "thread", "entry", "part", "inbox", "usage", "tally"] as const; -it("creates the sync tables, seeds them, and cascades a removed machine", async () => { +it("creates the sync tables, seeds them, and cascades a removed machine, keeping its usage", async () => { const db = new DatabaseSync(database); db.exec("PRAGMA busy_timeout = 5000"); db.exec("PRAGMA foreign_keys = ON"); @@ -169,7 +179,9 @@ it("creates the sync tables, seeds them, and cascades a removed machine", async db.close(); const steps = { schema, seeded, servers }; - expect(servers.cascade.after).toEqual(Object.fromEntries(TREE.map((table) => [table, 1]))); + expect(servers.cascade.after).toEqual( + Object.fromEntries(TREE.map((table) => [table, table === "usage" || table === "tally" ? 2 : 1])), + ); expect(servers.ungrouped).toEqual([{ id: "thr_local", grouping_id: null }]); const text = JSON.stringify(steps, null, "\t").replaceAll(servers.row.id as string, ""); await expect(`${text}\n`).toMatchFileSnapshot("./__artifacts__/schema.json"); diff --git a/apps/server/test/usage.machines.live.test.ts b/apps/server/test/usage.machines.live.test.ts new file mode 100644 index 00000000..a2a0d43f --- /dev/null +++ b/apps/server/test/usage.machines.live.test.ts @@ -0,0 +1,537 @@ +/* @effect-diagnostics nodeBuiltinImport:off -- the suite runs CodeWork servers on OrbStack Linux machines. */ +import { Effect, Option, Stream } from "effect"; +import { userInfo } from "node:os"; +import { afterAll, beforeAll, describe, expect, it } from "vite-plus/test"; +import { Contract, type MetricsRow, type MetricsUsage } from "@codeworksh/contract"; +import { Session } from "../../../packages/harness/src/effect.ts"; +import { + type AppRpc, + app, + cleanup, + cursorOf, + lastSeq, + machineRow, + normalizer, + notices, + query, + type Row, + snapshot as match, + stop, + tempDir, + waitFor, +} from "./fixtures/apps.ts"; +import { available, ensure, machine, type Machine } from "./machines/orb.ts"; + +/* + * The usage rollup across real computers: the fixture CodeWork server (scripted model) runs on the + * OrbStack machines cw-a, cw-b and cw-c, each reporting its own host name, all mirrored into one app + * server on the Mac. Each machine's rollup is read inside it with sqlite3 and compared with app.db and + * the metrics reads, across a network cut, an app restart and a deleted session. + * `usage.machines.rollup.json` records the names, rows and totals. + */ + +const userHome = `/home/${userInfo().username}`; +const cwA = machine("cw-a"); +const cwB = machine("cw-b"); +const cwC = machine("cw-c"); +const pool = [cwA, cwB, cwC]; + +type Desk = Awaited>; +type Rollup = { readonly usage: Array; readonly tally: Array }; + +let appHome = ""; +let desk: Desk; +const ids = new Map(); +const stores = new Map(); +const idOf = (m: Machine) => ids.get(m.name)!; + +const work = (m: Machine) => `${m.cwHome}/work`; +const DAY = 86_400_000; +const today = new Date().toISOString().slice(0, 10); +const everything = { zone: "UTC", since: "1970-01-01", until: new Date(Date.now() + DAY).toISOString().slice(0, 10) }; + +/** A machine with a fresh store and its server running, named as the OS names the machine. */ +const fresh = async (m: Machine) => { + await m.net("restore"); + await m.net("fast"); + await m.stop("KILL"); + await m.wipe(); + await m.sh(`mkdir -p ${work(m)}`); + await m.start({ CODEWORK_FIXTURE_HOST_OS: "1" }); +}; + +/** Waits until the app's index cursor for `m` reached its head. */ +const caughtUp = async (m: Machine, timeout = 120_000) => { + const { storeId, rev } = await m.run((rpc) => rpc["server.info"]({})); + await waitFor( + () => { + const row = machineRow(appHome, idOf(m)); + return row?.store_id === storeId && row.cursor >= rev ? row : undefined; + }, + `${m.name} indexed at rev ${rev}`, + timeout, + ); +}; +/** Waits until the app's usage cursor for `m` reached its head. */ +const rolledUp = async (m: Machine, timeout = 120_000) => { + const { storeId, rev } = await m.run((rpc) => rpc["server.info"]({})); + await waitFor( + () => { + const row = cursorOf(appHome, idOf(m)); + return row.rollup_store === storeId && row.rollup >= rev ? row : undefined; + }, + `${m.name} rolled up at rev ${rev}`, + timeout, + ); + stores.set(m.name, storeId); +}; +const settled = async () => { + for (const m of pool) { + await caughtUp(m); + await rolledUp(m); + } +}; + +type Ref = { readonly sessionId: string; readonly threadId?: string }; +const branded = (ref: Ref) => ({ + sessionId: Session.SessionSchema.ID.make(ref.sessionId), + ...(ref.threadId === undefined ? {} : { threadId: Session.ThreadSchema.ID.make(ref.threadId) }), +}); +const turn = (m: Machine, ref: Ref, text: string) => + m.run((rpc) => + rpc["thread.prompt"]({ ...branded(ref), text }).pipe(Effect.andThen(rpc["thread.wait"](branded(ref)))), + ); +const session = async (m: Machine, text: string) => { + const created = await m.run((rpc) => rpc["session.create"]({ directory: work(m) })); + await turn(m, { sessionId: created.session.id }, text); + return { sessionId: created.session.id as string, threadId: created.id as string }; +}; +const fork = async (m: Machine, ref: Ref) => { + const forked = await m.run((rpc) => + Effect.gen(function* () { + const reply = (yield* rpc["thread.history"](branded(ref))).entries.at(-1)!; + return (yield* rpc["thread.fork"]({ ...branded(ref), entryId: reply.id })).thread; + }), + ); + return { sessionId: ref.sessionId, threadId: forked.id as string }; +}; + +// Rows keyed as both stores key them, sorted so each side lines up. +const keyOf = (row: Row) => + [row.hour, row.thread_id, row.provider ?? row.kind, row.model ?? row.name, row.thinking].map(String).join("|"); +const byKey = (rows: ReadonlyArray) => rows.toSorted((a, b) => keyOf(a).localeCompare(keyOf(b))); +const byRev = (rows: ReadonlyArray) => + rows.toSorted((a, b) => (a.rev as number) - (b.rev as number) || keyOf(a).localeCompare(keyOf(b))); + +/** The machine's own rollup, read in it. */ +const harness = async (m: Machine): Promise => ({ + usage: await m.sql("SELECT * FROM usage"), + tally: await m.sql("SELECT * FROM tally"), +}); +/** What app.db mirrors of `m`, without the machine and store it is keyed by. */ +const mirrored = (m: Machine): Rollup & { readonly stores: ReadonlyArray } => { + const strip = (rows: Array) => rows.map(({ machine_id: _m, store_id: _s, ...row }) => row); + const usage = query(appHome, "SELECT * FROM usage WHERE machine_id = ?", idOf(m)); + const tally = query(appHome, "SELECT * FROM tally WHERE machine_id = ?", idOf(m)); + return { + usage: strip(usage), + tally: strip(tally), + stores: [...new Set([...usage, ...tally].map((row) => row.store_id))], + }; +}; +const whole = () => ({ + usage: query(appHome, "SELECT * FROM usage ORDER BY machine_id, rev, thread_id, provider, model"), + tally: query(appHome, "SELECT * FROM tally ORDER BY machine_id, rev, thread_id, kind, name"), +}); + +/** app.db holds exactly the machine's rollup, from its current store. */ +const expectMirror = async (m: Machine) => { + const theirs = await harness(m); + const mine = mirrored(m); + expect(byKey(mine.usage)).toEqual(byKey(theirs.usage)); + expect(byKey(mine.tally)).toEqual(byKey(theirs.tally)); + expect(mine.stores).toEqual([stores.get(m.name)]); + return theirs; +}; + +type Totals = { requests: number; input: number; output: number; cacheRead: number; cacheWrite: number; cost: number }; +const zero: Totals = { requests: 0, input: 0, output: 0, cacheRead: 0, cacheWrite: 0, cost: 0 }; +const add = (a: Totals, b: Totals): Totals => ({ + requests: a.requests + b.requests, + input: a.input + b.input, + output: a.output + b.output, + cacheRead: a.cacheRead + b.cacheRead, + cacheWrite: a.cacheWrite + b.cacheWrite, + cost: a.cost + b.cost, +}); +const ofRows = (rows: ReadonlyArray) => + rows.reduce( + (total, row) => + add(total, { + requests: row.requests as number, + input: row.input as number, + output: row.output as number, + cacheRead: row.cache_read as number, + cacheWrite: row.cache_write as number, + cost: row.cost as number, + }), + zero, + ); +const ofMetrics = (rows: ReadonlyArray) => + rows.reduce( + (total, row) => + add(total, { + requests: row.requests, + input: row.tokens.input, + output: row.tokens.output, + cacheRead: row.tokens.cacheRead, + cacheWrite: row.tokens.cacheWrite, + cost: row.cost.total, + }), + zero, + ); +const tallies = (rows: ReadonlyArray) => { + const by: Record = {}; + for (const row of rows) + by[`${String(row.kind)}:${String(row.name)}`] = + (by[`${String(row.kind)}:${String(row.name)}`] ?? 0) + (row.count as number); + return by; +}; +const metricsTallies = (metrics: MetricsUsage) => + tallies(metrics.tallies.map((tally) => ({ kind: tally.kind, name: tally.name, count: tally.count }))); + +const read = (request: Parameters[0]) => desk.run((rpc) => rpc["metrics.usage"](request)); + +/** What each read that names a machine shows, per machine as Settings lists them. */ +const names = async () => { + const rows = query(appHome, "SELECT id, kind, name, host FROM machine ORDER BY pos"); + const reads = await desk.run((rpc) => + Effect.gen(function* () { + const sidebar = yield* rpc["sidebar.list"]({ + organizationId: Contract.PERSONAL_ORGANIZATION, + machines: "all", + }); + const threads = yield* rpc["threads.list"]({ organizationId: Contract.PERSONAL_ORGANIZATION, by: "status" }); + const statuses = yield* rpc["servers.watch"](undefined).pipe(Stream.runHead, Effect.map(Option.getOrThrow)); + const listed = threads.lists.flatMap((list) => list.threads); + return statuses.map(({ id, name, host, override }) => ({ + id, + name, + host, + override, + sidebar: sidebar.find((project) => project.machine.id === id)?.machine.name, + allThreads: listed.find((thread) => thread.machine.id === id)?.machine.name, + })); + }), + ); + return reads.map((read) => ({ stored: rows.find((row) => row.id === read.id), ...read })); +}; +const shown = (state: Awaited>, m: Machine) => { + const found = state.find((read) => read.id === idOf(m)); + return [found?.name, found?.sidebar, found?.allThreads]; +}; + +// Kept for the artifact as the cases go. +const artifact: Record = {}; +let workload: Record }> = {}; + +const describeMachines = available ? describe : describe.skip; + +describeMachines("usage across OrbStack machines", () => { + beforeAll(async () => { + for (const m of pool) await ensure(m.name); + await Promise.all(pool.map((m) => m.setup())); + await Promise.all(pool.map((m) => fresh(m))); + appHome = tempDir("usage-machines-app"); + desk = await app(appHome); + // No name of its own: each is named by the host its server reports. + for (const m of pool) + ids.set(m.name, await desk.run((rpc) => rpc["servers.add"]({ kind: "remote", url: m.url }))); + + // Different work on each: a turn with a tool call (two requests); two turns; a turn and a fork's turn. + const a = await session(cwA, "bash echo alpha"); + const b = await session(cwB, "first"); + await turn(cwB, b, "second"); + const c = await session(cwC, "bash echo gamma"); + const forked = await fork(cwC, c); + await turn(cwC, forked, "in the fork"); + workload = { "cw-a": { sessions: [a] }, "cw-b": { sessions: [b] }, "cw-c": { sessions: [c, forked] } }; + await settled(); + }, 1_800_000); + + afterAll(async () => { + await Promise.all( + pool.map((m) => + m + .net("fast") + .then(() => m.net("restore")) + .then(() => m.stop("TERM")) + .catch(() => undefined), + ), + ); + await cleanup(); + }, 120_000); + + it("1. each machine is named by its own host name; an override wins and clears back to it", async () => { + const reported = await waitFor(async () => { + const state = await names(); + return pool.every((m) => state.find((read) => read.id === idOf(m))?.stored?.host === m.name) + ? state + : undefined; + }, "every machine's host name"); + for (const m of pool) { + const found = reported.find((read) => read.id === idOf(m))!; + expect(found.stored).toMatchObject({ kind: "remote", name: null, host: m.name }); + expect(found).toMatchObject({ host: m.name, override: null }); + expect(shown(reported, m)).toEqual(Array(3).fill(m.name)); + } + + await desk.run((rpc) => rpc["servers.update"]({ id: idOf(cwB), name: " Build Box " })); + const overridden = await names(); + expect(shown(overridden, cwB)).toEqual(Array(3).fill("Build Box")); + expect(shown(overridden, cwA)).toEqual(Array(3).fill("cw-a")); + const overriddenMetrics = (await read(everything)).machines.find((entry) => entry.id === idOf(cwB)); + expect(overriddenMetrics?.name).toBe("Build Box"); + + await desk.run((rpc) => rpc["servers.update"]({ id: idOf(cwB), name: "" })); + const cleared = await names(); + expect(shown(cleared, cwB)).toEqual(Array(3).fill("cw-b")); + expect(cleared.find((read) => read.id === idOf(cwB))?.stored).toMatchObject({ name: null, host: "cw-b" }); + + const steps = { reported, overridden, overriddenMetrics, cleared }; + expect(JSON.stringify(steps)).not.toContain("CodeWork"); + artifact.names = steps; + }, 300_000); + + it("2. each machine's mirrored usage and tallies are its own rollup, and sum to its threads' usage", async () => { + const rollups: Record = {}; + for (const m of pool) { + const theirs = await expectMirror(m); + const threads = await m.sql("SELECT id, session_id FROM thread ORDER BY created_at, created_seq"); + const reported = await m.run((rpc) => + Effect.forEach(threads, (thread) => + rpc["thread.info"]( + branded({ sessionId: thread.session_id as string, threadId: thread.id as string }), + ).pipe(Effect.map((info) => ({ id: thread.id as string, usage: info.usage }))), + ), + ); + const app = mirrored(m); + for (const { id, usage } of reported) { + const rows = app.usage.filter((row) => row.thread_id === id); + expect(ofRows(rows)).toMatchObject({ + input: usage.input, + output: usage.output, + cacheRead: usage.cacheRead, + cacheWrite: usage.cacheWrite, + cost: usage.cost, + }); + } + rollups[m.name] = { + threads: reported.length, + totals: ofRows(theirs.usage), + tallies: tallies(theirs.tally), + usage: byRev(app.usage), + tally: byRev(app.tally), + }; + } + expect( + Object.fromEntries( + Object.entries(rollups).map(([name, rollup]) => [name, (rollup as { totals: Totals }).totals.requests]), + ), + ).toEqual({ + "cw-a": 2, + "cw-b": 2, + "cw-c": 3, + }); + expect( + Object.fromEntries( + Object.entries(rollups).map(([name, rollup]) => [name, (rollup as { tallies: unknown }).tallies]), + ), + ).toEqual({ + "cw-a": { "prompt:": 1, "tool:bash": 1 }, + "cw-b": { "prompt:": 2 }, + "cw-c": { "prompt:": 2, "tool:bash": 1 }, + }); + artifact.rollup = rollups; + }, 300_000); + + it("3. metrics.usage sums every machine or one, names each machine, and metrics.hours holds the day's totals", async () => { + const own = new Map(); + for (const m of pool) own.set(m.name, await harness(m)); + const all = await read(everything); + const total = pool.reduce((sum, m) => add(sum, ofRows(own.get(m.name)!.usage)), zero); + expect(ofMetrics(all.rows)).toEqual(total); + expect(all.machines.toSorted((a, b) => a.name.localeCompare(b.name))).toEqual( + pool.map((m) => ({ id: idOf(m), name: m.name })), + ); + + const perMachine: Record = {}; + for (const m of pool) { + const one = await read({ ...everything, machineId: idOf(m) }); + const theirs = own.get(m.name)!; + expect(new Set(one.rows.map((row) => row.machineId))).toEqual(new Set([idOf(m)])); + expect(new Set(one.tallies.map((row) => row.machineId))).toEqual(new Set([idOf(m)])); + expect(ofMetrics(one.rows)).toEqual(ofRows(theirs.usage)); + expect(metricsTallies(one)).toEqual(tallies(theirs.tally)); + expect(one.machines).toEqual([{ id: idOf(m), name: m.name }]); + perMachine[m.name] = { totals: ofMetrics(one.rows), tallies: metricsTallies(one), machines: one.machines }; + } + + // The scripted model stamps its requests with its own clock, so they land on the day of its hours; + // prompts land on today's. + const requestDays = [ + ...new Set( + [...own.values()].flatMap((rollup) => + rollup.usage.map((row) => new Date(row.hour as number).toISOString().slice(0, 10)), + ), + ), + ].toSorted(); + const hours = []; + for (const day of requestDays) + hours.push(...(await desk.run((rpc) => rpc["metrics.hours"]({ zone: "UTC", day })))); + const tokens = total.input + total.output + total.cacheRead + total.cacheWrite; + expect({ + requests: hours.reduce((sum, row) => sum + row.requests, 0), + tokens: hours.reduce((sum, row) => sum + row.tokens, 0), + cost: hours.reduce((sum, row) => sum + row.cost, 0), + }).toEqual({ requests: total.requests, tokens, cost: total.cost }); + const onDays = await read({ zone: "UTC", since: requestDays[0]!, until: requestDays.at(-1)! }); + expect(ofMetrics(onDays.rows)).toEqual(total); + const onToday = await read({ zone: "UTC", since: today, until: today }); + const prompts = pool.reduce((sum, m) => sum + (tallies(own.get(m.name)!.tally)["prompt:"] ?? 0), 0); + expect(metricsTallies(onToday)["prompt:"]).toBe(prompts); + + artifact.metrics = { + all: { + totals: ofMetrics(all.rows), + tallies: metricsTallies(all), + // Listed by id, which is random: kept by name. + machines: all.machines.toSorted((a, b) => a.name.localeCompare(b.name)), + }, + perMachine, + requestDays, + hours: hours.toSorted( + (a, b) => + a.hour - b.hour || a.machineId.localeCompare(b.machineId) || a.sessionId.localeCompare(b.sessionId), + ).length, + todayPrompts: metricsTallies(onToday)["prompt:"], + }; + }, 300_000); + + it("4. a cut machine catches up from its cursor with nothing lost or doubled; an app restart changes nothing", async () => { + const [a] = workload["cw-a"]!.sessions; + const cursor = cursorOf(appHome, idOf(cwB)).rollup; + + const cutAt = performance.now(); + await cwB.net("cut"); + // cw-b keeps working over its own loopback, in a new session so its older rows stay before the + // cursor; the others over the network. + const [created] = (await cwB.drive([["session.create", { directory: work(cwB) }]])) as [ + { session: { id: string }; id: string }, + ]; + await cwB.drive([ + ["thread.prompt", { sessionId: created.session.id, text: "bash echo while cut" }], + ["thread.wait", { sessionId: created.session.id }], + ]); + await turn(cwA, a!, "bash echo during the cut"); + await session(cwC, "during the cut"); + await rolledUp(cwA); + await rolledUp(cwC); + // Past the client's ping timeout, so the app has dropped the connection. + const held = 15_000 - (performance.now() - cutAt); + if (held > 0) await new Promise((resolve) => setTimeout(resolve, held)); + expect(cursorOf(appHome, idOf(cwB)).rollup).toBe(cursor); + + const at = lastSeq(appHome); + await cwB.net("restore"); + await rolledUp(cwB); + const changed = notices(appHome, Contract.syncTopics.usage, at).filter( + (notice) => notice.machineId === idOf(cwB), + ); + const since = await harness(cwB); + const pending = [...since.usage, ...since.tally].filter((row) => (row.rev as number) > cursor).length; + const sent = changed.reduce((sum, notice) => sum + (notice.items as number), 0); + expect(sent).toBe(pending); + expect(pending).toBeLessThan(since.usage.length + since.tally.length); + await expectMirror(cwB); + + await turn(cwB, { sessionId: created.session.id }, "after the cut"); + await settled(); + for (const m of pool) await expectMirror(m); + const totals = Object.fromEntries( + await Promise.all(pool.map(async (m) => [m.name, ofRows((await harness(m)).usage)] as const)), + ); + expect(Object.fromEntries(pool.map((m) => [m.name, ofRows(mirrored(m).usage)]))).toEqual(totals); + + // The app restarts: it resumes every machine from its cursor and holds the same rows. + const before = whole(); + const metricsBefore = await read(everything); + await stop(desk.proc); + const restartedAt = lastSeq(appHome); + desk = await app(appHome); + await settled(); + const resent = notices(appHome, Contract.syncTopics.usage, restartedAt).reduce( + (sum, notice) => sum + (notice.items as number), + 0, + ); + expect(whole()).toEqual(before); + expect(resent).toBe(0); + const metricsAfter = await read(everything); + expect(ofMetrics(metricsAfter.rows)).toEqual(ofMetrics(metricsBefore.rows)); + for (const m of pool) await expectMirror(m); + + artifact.resilience = { + cut: "cw-b", + resumed: { sent, pending, notices: changed.length > 0 }, + totals, + restart: { resent, rowsUnchanged: true, metrics: ofMetrics(metricsAfter.rows) }, + }; + }, 300_000); + + it("5. a session deleted on its machine keeps its mirrored usage, marked deleted", async () => { + const [c] = workload["cw-c"]!.sessions; + const before = whole(); + const metricsBefore = await read(everything); + await cwC.run((rpc) => rpc["session.delete"](branded({ sessionId: c!.sessionId }))); + await caughtUp(cwC); + await rolledUp(cwC); + expect(query(appHome, "SELECT deleted_at FROM session WHERE id = ?", c!.sessionId)[0]?.deleted_at).not.toBeNull(); + expect(whole()).toEqual(before); + const after = await read(everything); + expect(after.rows).toEqual(metricsBefore.rows); + const entry = after.sessions.find((one) => one.id === c!.sessionId); + expect(entry).toMatchObject({ machineId: idOf(cwC), title: null, deleted: true }); + expect(after.sessions.filter((one) => one.deleted).map((one) => one.id)).toEqual([c!.sessionId]); + artifact.deleted = { + session: entry, + rows: after.rows.filter((row) => row.sessionId === c!.sessionId).length, + totals: ofMetrics(after.rows.filter((row) => row.sessionId === c!.sessionId)), + }; + }, 300_000); + + it("the artifact: names, rollups, totals and recoveries", async () => { + // Requests carry the scripted model's own hours, prompts the real one. + const hours = (value: unknown, key?: string): unknown => { + if (key === "hour" && typeof value === "number") return value < DAY ? "" : ""; + if (Array.isArray(value)) return value.map((item) => hours(item)); + if (typeof value === "object" && value !== null) + return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, hours(v, k)])); + return value; + }; + const projects = query(appHome, "SELECT DISTINCT id, name FROM project ORDER BY name").map( + (row) => [row.id as string, ``] as const, + ); + await match( + "usage.machines.rollup", + normalizer( + [appHome, userHome], + [ + ...pool.map((m) => [idOf(m), ``] as const), + ...pool.map((m) => [stores.get(m.name)!, ``] as const), + ...projects, + [today, ""], + ], + )(hours(artifact)), + ); + }); +}); diff --git a/apps/server/test/usage.test.ts b/apps/server/test/usage.test.ts new file mode 100644 index 00000000..4b03d15d --- /dev/null +++ b/apps/server/test/usage.test.ts @@ -0,0 +1,432 @@ +/* @effect-diagnostics nodeBuiltinImport:off -- the suite stops, copies and removes CodeWork stores in temp dirs. */ +import { Effect, Stream } from "effect"; +import { copyFileSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { afterEach, expect, it } from "vite-plus/test"; +import { Contract } from "@codeworksh/contract"; +import { Codework as Wire } from "@codeworksh/widget/codework"; +import type * as Client from "../../../packages/codework/src/server/client.ts"; +import { edges, type UsageItem } from "../../../packages/codework/src/server/contract.ts"; +import { + addMachine, + app, + cleanup, + type Codework, + codework, + cursorOf, + freePort, + lastSeq, + normalizer, + notices, + query, + rolled, + type Row, + snapshot as match, + stop, + synced, + tempDir, + waitFor, +} from "./fixtures/apps.ts"; + +afterEach(cleanup); + +type UsageRow = Exclude; +type SessionId = Parameters[0]["sessionId"]; + +const snapshot = (name: string, artifact: unknown) => match(`usage.${name}`, artifact); + +/** A session in `home` whose prompt runs one `bash` call: two model requests and a tool call. */ +const session = async (server: Codework, directory: string, command: string) => { + const created = await server.run((rpc) => rpc["session.create"]({ directory })); + await turn(server, created.session.id, command); + return created; +}; +const turn = (server: Codework, sessionId: SessionId, command: string) => + server.run((rpc) => + rpc["thread.prompt"]({ sessionId, text: `bash ${command}` }).pipe( + Effect.andThen(rpc["thread.wait"]({ sessionId })), + ), + ); + +/** What `usage.sync` from `after` sends up to its first `Synced`, read from CodeWork itself. */ +const catchUp = async (server: Codework, after = 0) => { + const items = await server.run((rpc) => + rpc["usage.sync"]({ after }).pipe( + Stream.takeUntil((item) => item._tag === "Synced"), + Stream.runCollect, + ), + ); + const synced = items.at(-1); + if (synced?._tag !== "Synced") throw new Error("usage.sync ended before Synced"); + return { rows: items.filter((item): item is UsageRow => item._tag !== "Synced"), synced }; +}; + +/** A wire row as app.db holds it for `machineId` and `storeId`. */ +const stored = (machineId: string, storeId: string, row: UsageRow): Row => { + const owner = { + machine_id: machineId, + store_id: storeId, + hour: row.hour, + thread_id: row.threadId, + session_id: row.sessionId, + project_id: row.projectId, + rev: row.rev, + }; + if (row._tag === "Tally") return { ...owner, kind: row.kind, name: row.name, count: row.count }; + return { + ...owner, + provider: row.provider, + model: row.model, + thinking: row.thinking, + requests: row.requests, + input: row.tokens.input, + output: row.tokens.output, + cache_read: row.tokens.cacheRead, + cache_write: row.tokens.cacheWrite, + reasoning: row.tokens.reasoning, + cost: row.cost.total, + cost_input: row.cost.input, + cost_output: row.cost.output, + cost_cache_read: row.cost.cacheRead, + cost_cache_write: row.cost.cacheWrite, + duration: row.duration, + timed: row.timed, + durations: JSON.stringify(row.durations), + }; +}; + +const keyOf = (row: Row) => + [ + row.machine_id, + row.store_id, + row.hour, + row.thread_id, + row.provider ?? row.kind, + row.model ?? row.name, + row.thinking, + ] + .map(String) + .join("|"); +const byKey = (rows: ReadonlyArray) => rows.toSorted((a, b) => keyOf(a).localeCompare(keyOf(b))); + +const mirrored = (home: string) => ({ + usage: query(home, "SELECT * FROM usage ORDER BY rev, provider, model"), + tally: query(home, "SELECT * FROM tally ORDER BY rev, kind, name"), +}); +/** What app.db should hold for `rows` of one store: usage rows, then tallies. */ +const expected = (machineId: string, storeId: string, rows: ReadonlyArray) => ({ + usage: byKey(rows.filter((row) => row._tag === "Usage").map((row) => stored(machineId, storeId, row))), + tally: byKey(rows.filter((row) => row._tag === "Tally").map((row) => stored(machineId, storeId, row))), +}); +const sorted = (rows: { usage: ReadonlyArray; tally: ReadonlyArray }) => ({ + usage: byKey(rows.usage), + tally: byKey(rows.tally), +}); + +const requestsOf = (rows: ReadonlyArray) => rows.reduce((sum, row) => sum + (row.requests as number), 0); +const inputOf = (rows: ReadonlyArray) => rows.reduce((sum, row) => sum + (row.input as number), 0); +const itemsOf = (payloads: ReadonlyArray) => payloads.reduce((sum, payload) => sum + (payload.items as number), 0); + +// Requests are made at the scripted model's own times, in the first hour of 1970; prompts in the real one. +const hours = (value: unknown, key?: string): unknown => { + if (key === "hour") return value === 0 ? "" : ""; + if (Array.isArray(value)) return value.map((item) => hours(item)); + if (typeof value === "object" && value !== null) + return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, hours(v, k)])); + return value; +}; + +it("the widget mirror's duration edges are CodeWork's", () => { + expect(Wire.edges).toEqual(edges); +}); + +it("mirrors usage and tallies with replaced running totals, resumes after a restart and keeps a deleted session's rows", async () => { + const cwHome = tempDir("usage-cw"); + const appHome = tempDir("usage-app"); + const server = await codework(cwHome, await freePort()); + let desk = await app(appHome); + const machineId = await addMachine(appHome, server, desk.run); + + // (1) A turn: two requests and a tool call. + const first = await session(server, cwHome, "echo one"); + const sessionId = first.session.id; + const { storeId } = await rolled(appHome, machineId, server); + const one = await catchUp(server); + const afterOne = mirrored(appHome); + expect(sorted(afterOne)).toEqual(expected(machineId, storeId, one.rows)); + expect(cursorOf(appHome, machineId)).toEqual({ rollup: one.synced.rev, rollup_store: storeId }); + expect(one.synced.storeId).toBe(storeId); + expect(afterOne.usage).toHaveLength(1); + expect(afterOne.usage[0]).toMatchObject({ requests: 2, input: 2 }); + expect(afterOne.tally.map((row) => [row.kind, row.name, row.count])).toEqual([ + ["prompt", "", 1], + ["tool", "bash", 1], + ]); + + // (2) A second turn in the same hour: the row's totals are replaced by the larger running ones. + await turn(server, sessionId, "echo two"); + await rolled(appHome, machineId, server); + const two = await catchUp(server); + const afterTwo = mirrored(appHome); + const thread = await server.run((rpc) => rpc["thread.info"]({ sessionId })); + expect(sorted(afterTwo)).toEqual(expected(machineId, storeId, two.rows)); + expect(afterTwo.usage).toHaveLength(1); + expect(afterTwo.usage[0]).toMatchObject({ + requests: 4, + input: thread.usage.input, + output: thread.usage.output, + cost: thread.usage.cost, + }); + expect(afterTwo.tally.map((row) => [row.kind, row.name, row.count])).toEqual([ + ["prompt", "", 2], + ["tool", "bash", 2], + ]); + + // (3) A turn of another session while the app is down: a restart resumes from the cursor. + const cursor = cursorOf(appHome, machineId).rollup; + await stop(desk.proc); + await session(server, cwHome, "echo three"); + const resumed = await catchUp(server, cursor); + const restartedAt = lastSeq(appHome); + desk = await app(appHome); + await rolled(appHome, machineId, server); + const three = await catchUp(server); + const afterThree = mirrored(appHome); + const changed = notices(appHome, Contract.syncTopics.usage, restartedAt); + expect(sorted(afterThree)).toEqual(expected(machineId, storeId, three.rows)); + expect(itemsOf(changed)).toBe(resumed.rows.length); + expect(resumed.rows.length).toBeLessThan(three.rows.length); + expect(afterThree.usage.find((row) => row.thread_id === first.id)).toEqual(afterTwo.usage[0]); + + // (4) A deleted session's rows stay. + await server.run((rpc) => rpc["session.delete"]({ sessionId })); + await synced(appHome, machineId, server); + await rolled(appHome, machineId, server); + expect(query(appHome, "SELECT deleted_at FROM session WHERE id = ?", sessionId)[0]?.deleted_at).not.toBeNull(); + expect(mirrored(appHome)).toEqual(afterThree); + + await snapshot( + "mirror", + normalizer( + [cwHome, appHome], + [[storeId, ""]], + )( + hours({ + first: afterOne, + second: afterTwo, + resumed: { items: resumed.rows.length, changed }, + ...afterThree, + }), + ), + ); +}, 120_000); + +it("a new store's rows accumulate beside the old store's", async () => { + const cwHome = tempDir("usage-store-cw"); + const appHome = tempDir("usage-store-app"); + const port = await freePort(); + let server = await codework(cwHome, port); + const desk = await app(appHome); + const machineId = await addMachine(appHome, server, desk.run); + await session(server, cwHome, "echo old"); + const old = await rolled(appHome, machineId, server); + const before = mirrored(appHome); + + // The app stays up: it reconnects to the new store on its own. + await stop(server.proc, "SIGTERM"); + for (const suffix of ["", "-wal", "-shm"]) rmSync(`${server.database}${suffix}`, { force: true }); + server = await codework(cwHome, port); + await session(server, cwHome, "echo new"); + const fresh = await rolled(appHome, machineId, server); + expect(fresh.storeId).not.toBe(old.storeId); + const now = await catchUp(server); + + const rows = mirrored(appHome); + const ofStore = (list: ReadonlyArray, id: string) => list.filter((row) => row.store_id === id); + expect(ofStore(rows.usage, old.storeId)).toEqual(before.usage); + expect(ofStore(rows.tally, old.storeId)).toEqual(before.tally); + expect(sorted({ usage: ofStore(rows.usage, fresh.storeId), tally: ofStore(rows.tally, fresh.storeId) })).toEqual( + expected(machineId, fresh.storeId, now.rows), + ); + expect(requestsOf(rows.usage)).toBe(requestsOf(before.usage) + requestsOf(ofStore(rows.usage, fresh.storeId))); + expect(requestsOf(rows.usage)).toBe(4); + expect(inputOf(rows.usage)).toBe(4); + expect(cursorOf(appHome, machineId)).toEqual({ rollup: now.synced.rev, rollup_store: fresh.storeId }); + + await snapshot( + "store", + normalizer( + [cwHome, appHome], + [ + [old.storeId, ""], + [fresh.storeId, ""], + ], + )(hours(rows)), + ); +}, 120_000); + +it("a removed server's usage stays, and the server added again replaces its rows, stale ones included", async () => { + const cwHome = tempDir("usage-removed-cw"); + const appHome = tempDir("usage-removed-app"); + const server = await codework(cwHome, await freePort()); + const desk = await app(appHome); + const removedId = await addMachine(appHome, server, desk.run); + await session(server, cwHome, "echo kept"); + const { storeId } = await rolled(appHome, removedId, server); + const before = mirrored(appHome); + + await desk.run((rpc) => rpc["servers.remove"]({ id: removedId })); + expect(query(appHome, "SELECT id FROM machine WHERE id = ?", removedId)).toEqual([]); + expect(mirrored(appHome)).toEqual(before); + + // A row the store no longer has, as a backup restored while the server was away would leave. + query( + appHome, + `INSERT INTO tally (machine_id, store_id, hour, thread_id, kind, name, session_id, project_id, count, rev) + VALUES (?, ?, 0, 'thr_gone', 'tool', 'bash', 'ses_gone', 'prj_gone', 1, 1) RETURNING hour`, + removedId, + storeId, + ); + + const addedId = await addMachine(appHome, server, desk.run); + await rolled(appHome, addedId, server); + const now = await catchUp(server); + const rows = mirrored(appHome); + expect(sorted(rows)).toEqual(expected(addedId, storeId, now.rows)); + expect(requestsOf(rows.usage)).toBe(requestsOf(before.usage)); + + await snapshot( + "removed", + normalizer( + [cwHome, appHome], + [ + [storeId, ""], + [removedId, ""], + [addedId, ""], + ], + )(hours({ before, after: rows })), + ); +}, 120_000); + +it("a restored store replaces its own rows", async () => { + const cwHome = tempDir("usage-restored-cw"); + const appHome = tempDir("usage-restored-app"); + const backup = tempDir("usage-restored-backup"); + const port = await freePort(); + let server = await codework(cwHome, port); + await session(server, cwHome, "echo kept"); + // A clean stop leaves the database whole in its main file. + await stop(server.proc, "SIGTERM"); + copyFileSync(server.database, join(backup, "codework.db")); + server = await codework(cwHome, port); + + let desk = await app(appHome); + const machineId = await addMachine(appHome, server, desk.run); + const later = await session(server, cwHome, "echo lost"); + const { rev: head } = await rolled(appHome, machineId, server); + expect(query(appHome, "SELECT * FROM usage WHERE thread_id = ?", later.id)).toHaveLength(1); + + await stop(desk.proc); + await stop(server.proc, "SIGTERM"); + for (const suffix of ["-wal", "-shm"]) rmSync(`${server.database}${suffix}`, { force: true }); + copyFileSync(join(backup, "codework.db"), server.database); + server = await codework(cwHome, port); + const restored = await server.run((rpc) => rpc["server.info"]({})); + expect(restored.rev).toBeLessThan(head); + + const restartedAt = lastSeq(appHome); + desk = await app(appHome); + await waitFor( + () => (cursorOf(appHome, machineId).rollup === restored.rev ? true : undefined), + "the re-read of the restored store", + ); + const now = await catchUp(server); + const rows = mirrored(appHome); + expect(sorted(rows)).toEqual(expected(machineId, restored.storeId, now.rows)); + expect(rows.usage.some((row) => row.thread_id === later.id)).toBe(false); + const changed = notices(appHome, Contract.syncTopics.usage, restartedAt); + expect(itemsOf(changed)).toBe(now.rows.length); + + await snapshot( + "restored", + normalizer([cwHome, appHome], [[restored.storeId, ""]])(hours({ changed, ...rows })), + ); +}, 120_000); + +it("a restored store is replaced even once its head passed the cursor; a plain restart only resumes", async () => { + const cwHome = tempDir("usage-passed-cw"); + const appHome = tempDir("usage-passed-app"); + const backup = tempDir("usage-passed-backup"); + const port = await freePort(); + let server = await codework(cwHome, port); + await session(server, cwHome, "echo kept"); + await stop(server.proc, "SIGTERM"); + copyFileSync(server.database, join(backup, "codework.db")); + server = await codework(cwHome, port); + + let desk = await app(appHome); + const machineId = await addMachine(appHome, server, desk.run); + const lost = await session(server, cwHome, "echo lost"); + const { rev: head } = await rolled(appHome, machineId, server); + + // (1) A restart of the same store moves on from the cursor: only the new rows come. + await stop(desk.proc); + await stop(server.proc, "SIGTERM"); + server = await codework(cwHome, port); + await session(server, cwHome, "echo moved"); + const moved = await catchUp(server, head); + let restartedAt = lastSeq(appHome); + desk = await app(appHome); + await rolled(appHome, machineId, server); + expect(itemsOf(notices(appHome, Contract.syncTopics.usage, restartedAt))).toBe(moved.rows.length); + expect(sorted(mirrored(appHome))).toEqual(expected(machineId, moved.synced.storeId, (await catchUp(server)).rows)); + const { rev: passed } = await rolled(appHome, machineId, server); + + // (2) The backup comes back and works past the cursor before the app sees it again. + await stop(desk.proc); + await stop(server.proc, "SIGTERM"); + for (const suffix of ["-wal", "-shm"]) rmSync(`${server.database}${suffix}`, { force: true }); + copyFileSync(join(backup, "codework.db"), server.database); + server = await codework(cwHome, port); + for (const command of ["echo one", "echo two", "echo three"]) await session(server, cwHome, command); + const restored = await server.run((rpc) => rpc["server.info"]({})); + expect(restored.rev).toBeGreaterThanOrEqual(passed); + + restartedAt = lastSeq(appHome); + desk = await app(appHome); + await rolled(appHome, machineId, server); + const now = await catchUp(server); + const rows = mirrored(appHome); + expect(sorted(rows)).toEqual(expected(machineId, restored.storeId, now.rows)); + expect(rows.usage.some((row) => row.thread_id === lost.id)).toBe(false); + expect(itemsOf(notices(appHome, Contract.syncTopics.usage, restartedAt))).toBe(now.rows.length); + + await snapshot( + "passed", + normalizer([cwHome, appHome], [[restored.storeId, ""]])(hours({ moved: moved.rows.length, ...rows })), + ); +}, 180_000); + +it("a server without usage.sync keeps its index synced and its usage empty, asked again only after a while", async () => { + const cwHome = tempDir("usage-older-cw"); + const appHome = tempDir("usage-older-app"); + const server = await codework(cwHome, await freePort(), { older: true }); + const desk = await app(appHome); + const machineId = await addMachine(appHome, server, desk.run); + const created = await session(server, cwHome, "echo older"); + await synced(appHome, machineId, server); + // Longer than the retry after any other failure. + await new Promise((resolve) => setTimeout(resolve, 3_000)); + await turn(server, created.session.id, "echo again"); + await synced(appHome, machineId, server); + + const told = desk.lines.filter((line) => line.includes("does not serve usage.sync")); + expect(query(appHome, "SELECT id FROM thread")).toHaveLength(1); + expect(query(appHome, "SELECT tokens_input FROM thread")[0]).toEqual({ tokens_input: 4 }); + expect(mirrored(appHome)).toEqual({ usage: [], tally: [] }); + expect(cursorOf(appHome, machineId)).toEqual({ rollup: 0, rollup_store: null }); + expect(told).toHaveLength(1); + expect(desk.lines.filter((line) => line.includes("Usage sync of machine"))).toEqual([]); + + await snapshot("older", { told: told.length, ...mirrored(appHome), cursor: cursorOf(appHome, machineId) }); +}, 90_000); diff --git a/apps/web/src/landing/sidebar.tsx b/apps/web/src/landing/sidebar.tsx index e296e738..05e5c014 100644 --- a/apps/web/src/landing/sidebar.tsx +++ b/apps/web/src/landing/sidebar.tsx @@ -1,6 +1,7 @@ import { ArrowsDownUpIcon, CaretRightIcon, + ChartBarIcon, ChatsIcon, FolderIcon, FolderOpenIcon, @@ -255,11 +256,19 @@ export function SessionsSidebar() { )} - - + + Settings + + + diff --git a/apps/web/src/metrics/activity.tsx b/apps/web/src/metrics/activity.tsx new file mode 100644 index 00000000..ef2a7721 --- /dev/null +++ b/apps/web/src/metrics/activity.tsx @@ -0,0 +1,563 @@ +import { useAtomValue } from "@effect/atom-react"; +import { Option } from "effect"; +import { AsyncResult } from "effect/reactivity"; +import { useLayoutEffect, useMemo, useRef, useState } from "react"; + +import { ProviderIcon } from "@/components/provider"; +import { Card, Columns, Heat, heatCuts, HeatScale, Meter, OTHER, Segmented, Share, SINGLE, TipRows } from "./charts"; +import { + type Day, + formatCost, + formatCount, + formatDay, + formatDayLong, + formatDuration, + formatQuantile, + formatTokens, + hoursAtom, + hoursKey, + listed, + type Metrics, + modelNamer, + modelsOn, + RANGES, + type Scope, + sessionsIn, + unpriced, +} from "./data"; +import { SessionLine } from "./session"; + +const GAP = 2; +const MIN_CELL = 11; +const MAX_CELL = 16; +const LABEL_GAP = 8; +const BAR_HEIGHT = 20; +const WEEKS_FOLDED = 8; +/** Monday first: the working week reads as one run, the weekend at the end. */ +const WEEKDAYS = [ + { day: 1, label: "Mon" }, + { day: 2, label: "Tue" }, + { day: 3, label: "Wed" }, + { day: 4, label: "Thu" }, + { day: 5, label: "Fri" }, + { day: 6, label: "Sat" }, + { day: 0, label: "Sun" }, +] as const; + +const weekdayOf = (day: Day) => new Date(day.date).getDay(); +const month = new Intl.DateTimeFormat(undefined, { month: "short" }); +const monthDay = new Intl.DateTimeFormat(undefined, { month: "short", day: "numeric" }); + +/** What the calendar counts, like a contribution graph's kinds. */ +type Measure = "prompts" | "sessions" | "tokens" | "cost"; +const MEASURE: Record< + Measure, + { readonly label: string; readonly one: string; readonly many: string; readonly format: (value: number) => string } +> = { + prompts: { label: "Prompts", one: "Prompt", many: "Prompts", format: formatCount }, + sessions: { label: "Sessions", one: "Session", many: "Sessions", format: formatCount }, + tokens: { label: "Tokens", one: "Token", many: "Tokens", format: formatTokens }, + cost: { label: "Cost", one: "Spent", many: "Spent", format: formatCost }, +}; +const KEYS = ["prompts", "sessions", "tokens", "cost"] as const; +/** The day view leads with spend. */ +const HEADLINE: ReadonlyArray = ["tokens", "cost", "sessions", "prompts"]; +const MEASURES = KEYS.map((key) => ({ key, label: MEASURE[key].label })); +/** "1,204 Prompts", "$12.40 Spent". */ +const amount = (measure: Measure, value: number) => { + const { one, many, format } = MEASURE[measure]; + return `${format(value)} ${value === 1 ? one : many}`; +}; + +/** The year calendar with when-you-work beside it; a picked day opens its sessions below. */ +export function Activity({ + year, + scope, + colors, +}: { + readonly year: Metrics; + readonly scope: Scope; + /** The page's model colors by model key; a model missing here folds into Other. */ + readonly colors: ReadonlyMap; +}) { + const [picked, setPicked] = useState(undefined); + const [measure, setMeasure] = useState("prompts"); + const pickedDay = picked === undefined ? undefined : year.days[year.days.length - 1 - picked]; + return ( + + } + > +
+
+ setPicked((current) => (current === ago ? undefined : ago))} + /> + +
+ +
+ {pickedDay !== undefined && ( + setPicked(undefined)} + /> + )} +
+ ); +} + +function Calendar({ + metrics, + measure, + picked, + onPick, +}: { + readonly metrics: Metrics; + readonly measure: Measure; + readonly picked: number | undefined; + readonly onPick: (ago: number) => void; +}) { + const { days } = metrics; + const lead = days[0] === undefined ? 0 : weekdayOf(days[0]); + const columns = Math.ceil((lead + days.length) / 7); + const scroller = useRef(null); + const labels = useRef(null); + const [cell, setCell] = useState(MIN_CELL); + // The cell grows with the card so the weeks fill the column; past the minimum it scrolls instead. + useLayoutEffect(() => { + const box = scroller.current; + if (box === null) return; + const fit = () => { + const room = box.clientWidth - (labels.current?.offsetWidth ?? 0) - LABEL_GAP; + setCell(Math.min(MAX_CELL, Math.max(MIN_CELL, Math.floor((room - (columns - 1) * GAP) / columns)))); + }; + // Fitted before the first paint, so the calendar never paints at the minimum and then grows. + fit(); + const observer = new ResizeObserver(fit); + observer.observe(box); + return () => observer.disconnect(); + }, [columns]); + // Open on the latest weeks when the calendar is wider than the card; a refresh of the data keeps the scroll. + useLayoutEffect(() => { + const box = scroller.current; + if (box !== null) box.scrollLeft = box.scrollWidth; + }, [cell]); + + const width = columns * cell + (columns - 1) * GAP; + const months = useMemo(() => { + const starts = days.flatMap((entry, index) => { + const date = new Date(entry.date); + return index === 0 || date.getDate() === 1 + ? [{ column: Math.floor((lead + index) / 7), label: month.format(date) }] + : []; + }); + // The period's opening month only gets a label when there is room before the next one. + return starts.filter((start, index) => index !== 0 || (starts[1]?.column ?? columns) - start.column >= 3); + }, [days, lead, columns]); + const cuts = useMemo(() => heatCuts(days.map((entry) => entry[measure])), [days, measure]); + const dayAt = (ago: number) => days[days.length - 1 - ago]; + const order = [measure, ...KEYS.filter((key) => key !== measure)]; + const { many, format } = MEASURE[measure]; + + return ( +
+

+ {format(metrics.totals[measure])} {many} in the Last 12 + Months +

+
+
+
+ {["", "Mon", "", "Wed", "", "Fri", ""].map((label, index) => ( + {label} + ))} +
+
+
+ {months.map((start) => ( + + {start.label} + + ))} +
+ ({ + ago: entry.ago, + weekday: weekdayOf(entry), + value: entry[measure], + }))} + title={({ ago }) => { + const entry = dayAt(ago); + return ( + ({ + label: MEASURE[key].label, + value: MEASURE[key].format(entry[key]), + })) + } + /> + ); + }} + /> +
+
+
+
+ amount(measure, value)} /> +
+
+ ); +} + +/** The measure's total per weekday over the year; the rows spread to the calendar column's height. */ +function Weekdays({ year, measure }: { readonly year: Metrics; readonly measure: Measure }) { + const weekdays = useMemo(() => { + const sums = new Map(); + for (const day of year.days) sums.set(weekdayOf(day), (sums.get(weekdayOf(day)) ?? 0) + day[measure]); + return WEEKDAYS.map(({ day, label }) => ({ label, total: sums.get(day) ?? 0 })); + }, [year.days, measure]); + const max = Math.max(...weekdays.map((weekday) => weekday.total), 0); + const { label, format } = MEASURE[measure]; + return ( +
+

By Weekday

+
    + {weekdays.map((weekday) => ( +
  • + 0 && weekday.total === max ? "text-ink" : "text-ink-muted"}`}> + {weekday.label} + + + + {format(weekday.total)} + +
  • + ))} +
+
+ ); +} + +function Streaks({ year, measure }: { readonly year: Metrics; readonly measure: Measure }) { + const busiest = year.days.reduce( + (best, day) => (day[measure] > (best?.[measure] ?? 0) ? day : best), + undefined, + ); + return ( +
+ + + + +
+ ); +} + +const days = (count: number) => `${formatCount(count)} ${count === 1 ? "Day" : "Days"}`; + +function Stat({ label, value }: { readonly label: string; readonly value: string }) { + return ( +
+
{label}
+
+ {value} +
+
+ ); +} + +// --------------------------------------------------------------------------------------------- + +const HOURS = Array.from({ length: 24 }, (_, hour) => String(hour).padStart(2, "0")); +const HOUR_TICKS = [0, 6, 12, 18, 23].map((at) => ({ at, label: HOURS[at] ?? "" })); +const MODEL_ROW = + "grid grid-cols-[minmax(0,1fr)_48px_64px] items-center gap-x-3 @xs:grid-cols-[minmax(0,1fr)_48px_48px_64px]"; + +/** A picked day at a glance: headline numbers, tokens through the day by model, then its sessions. */ +function DayDetail({ + day, + edges, + scope, + colors, + onClear, +}: { + readonly day: Day; + readonly edges: ReadonlyArray; + readonly scope: Scope; + readonly colors: ReadonlyMap; + readonly onClear: () => void; +}) { + // Until the day's hours arrive its chart stays empty; its totals are already here. + const hours = Option.getOrElse(AsyncResult.value(useAtomValue(hoursAtom(hoursKey(scope, day.day)))), () => []); + const spent = modelsOn(day, edges, hours); + const name = modelNamer(spent); + const models = spent.map((row) => ({ ...row, label: name(row), color: colors.get(row.key) ?? OTHER })); + const sessions = listed(day.bySession, "tokens"); + const named = models.filter((row) => colors.has(row.key)); + const rest = models.filter((row) => !colors.has(row.key)); + const stacks = [ + ...named.map((row) => ({ label: row.label, color: row.color, values: row.hours })), + ...(rest.length === 0 + ? [] + : [ + { + label: "Other", + color: OTHER, + values: HOURS.map((_, hour) => rest.reduce((sum, row) => sum + (row.hours[hour] ?? 0), 0)), + }, + ]), + ]; + const top = sessions[0]?.tokens ?? 0; + return ( +
+
+

{formatDayLong(day.date)}

+ +
+ {day.bySession.length === 0 ? ( +

No Work on This Day

+ ) : ( + <> +
+ {[ + ...HEADLINE.map((key) => ({ label: MEASURE[key].label, value: MEASURE[key].format(day[key]) })), + { label: "Model Time", value: formatDuration(day.modelTime) }, + ].map((stat) => ( +
+
{stat.label}
+
+ {stat.value} +
+
+ ))} +
+
+
+

Through the Day

+ `${hour}:00`)} + ticks={HOUR_TICKS} + format={formatTokens} + label="Tokens by model per hour" + height={120} + /> +
+
+

Models Used

+ ({ label: row.label, value: row.tokens, color: row.color }))} + format={formatTokens} + label="Tokens by model" + /> +
    + {models.map((row) => ( +
  • + + + + + {row.label} + + + + {row.median === undefined ? "–" : formatQuantile(row.median)} + + + {formatTokens(row.tokens)} + + + {unpriced(row) ? "Unpriced" : formatCost(row.cost)} + +
  • + ))} +
+
+
+
+

+ Sessions {formatCount(day.bySession.length)} +

+
    + {sessions.map((line) => ( + + ))} +
+
+ + )} +
+ ); +} + +// --------------------------------------------------------------------------------------------- + +interface Week { + readonly slots: ReadonlyArray; + readonly label: string; + readonly sessions: number; + readonly prompts: number; + readonly tokens: number; + readonly cost: number; +} + +/** Calendar weeks, Sunday first like the heatmap, newest first. */ +function weeksOf(days: ReadonlyArray): ReadonlyArray { + const groups: Array> = []; + let current: Array | undefined; + for (const day of days) { + const weekday = weekdayOf(day); + if (current === undefined || weekday === 0) { + current = Array.from({ length: 7 }, () => undefined); + groups.push(current); + } + current[weekday] = day; + } + return groups.reverse().map((slots) => { + const present = slots.filter((slot): slot is Day => slot !== undefined); + const first = present[0]; + const last = present.at(-1); + const sum = (pick: (day: Day) => number) => present.reduce((total, day) => total + pick(day), 0); + return { + slots, + label: + first === undefined || last === undefined + ? "" + : last.ago === 0 + ? "This Week" + : rangeLabel(first.date, last.date), + sessions: sessionsIn(present).length, + prompts: sum((day) => day.prompts), + tokens: sum((day) => day.tokens), + cost: sum((day) => day.cost), + }; + }); +} + +/** "Nov 8 – 14", or "Oct 30 – Nov 5" across a month. */ +function rangeLabel(first: number, last: number) { + if (first === last) return monthDay.format(first); + const start = new Date(first); + const end = new Date(last); + return start.getMonth() === end.getMonth() + ? `${monthDay.format(start)} – ${end.getDate()}` + : `${monthDay.format(start)} – ${monthDay.format(end)}`; +} + +const LEDGER = "grid grid-cols-[minmax(0,1fr)_84px_repeat(4,56px)] items-center gap-x-3"; + +/** The period week by week, folded to the latest weeks. */ +export function Weeks({ metrics }: { readonly metrics: Metrics }) { + const [all, setAll] = useState(false); + const weeks = useMemo(() => weeksOf(metrics.days), [metrics.days]); + const shown = all ? weeks : weeks.slice(0, WEEKS_FOLDED); + // One scale for every row, so a tall bar means the same in any week. + const max = Math.max(...metrics.days.map((day) => day.steps), 1); + const label = RANGES.find(({ key }) => key === metrics.range)?.label ?? ""; + return ( + +
+
+ Week + Requests + Sessions + Prompts + Tokens + Cost +
+
    + {shown.map((week) => ( +
  • + {week.label} + + {week.slots.map((slot, index) => ( + + {slot === undefined ? null : slot.steps === 0 ? ( + + ) : ( + + )} + + ))} + + {formatCount(week.sessions)} + {formatCount(week.prompts)} + {formatTokens(week.tokens)} + {formatCost(week.cost)} +
  • + ))} +
+ {shown.length < weeks.length && ( + + )} +
+
+ ); +} diff --git a/apps/web/src/metrics/charts.tsx b/apps/web/src/metrics/charts.tsx new file mode 100644 index 00000000..6e84f0aa --- /dev/null +++ b/apps/web/src/metrics/charts.tsx @@ -0,0 +1,782 @@ +import { + type MouseEvent, + type ReactNode, + type Ref, + useCallback, + useLayoutEffect, + useRef, + useState, + useSyncExternalStore, +} from "react"; + +/* Chart pieces and the card they sit in. Marks carry color; text stays in ink tokens. */ + +/** Categorical slots, assigned in this order and never cycled; what is past the last folds into Other. */ +const SERIES = [ + "light-dark(#2a78d6, #3987e5)", + "light-dark(#eb6834, #d95926)", + "light-dark(#1baf7a, #199e70)", + "light-dark(#eda100, #c98500)", + "light-dark(#e87ba4, #d55181)", +] as const; +export const OTHER = "light-dark(#b4b3ad, #5c5b56)"; +/** One hue for a lone series: magnitude, not identity. */ +export const SINGLE = SERIES[0]; +/** Neutral ink at `percent` over the surface: for parts of one thing, never confused with a series. */ +export const ink = (percent: number) => `color-mix(in oklab, var(--cw-ink) ${percent}%, var(--cw-frame))`; +/** An ordered scale in one hue, least to most; the first step is "none" and stays neutral. */ +export const RAMP = [ + "light-dark(#d9d8d3, #3a3936)", + "light-dark(#cfe1f9, #16406f)", + "light-dark(#a3c6f2, #184f95)", + "light-dark(#6da7ec, #1c5cab)", + "light-dark(#2a78d6, #3987e5)", + "light-dark(#1a5bb0, #6ea8ee)", + "light-dark(#104281, #9ec5f4)", +] as const; +/** Quiet to busy, one hue; the first step is an empty day. */ +const HEAT = [ + "color-mix(in oklab, var(--cw-ink) 7%, transparent)", + "light-dark(#b7d3f6, #184f95)", + "light-dark(#6da7ec, #1c5cab)", + "light-dark(#2a78d6, #3987e5)", + "light-dark(#104281, #9ec5f4)", +] as const; + +/** A color for each of the first `SERIES.length` keys, in order; the rest get none and fold into Other. */ +export const palette = (keys: ReadonlyArray): ReadonlyMap => + new Map(SERIES.flatMap((color, index) => (keys[index] === undefined ? [] : [[keys[index], color] as const]))); + +/** Rows with a color keep it, in order; the rest are summed into one trailing Other row and also returned as `rest`. */ +export function fold( + rows: ReadonlyArray, + colors: ReadonlyMap, + sum: (rest: ReadonlyArray) => Row, +): { readonly parts: ReadonlyArray; readonly rest: ReadonlyArray } { + const head = rows.flatMap((row) => { + const color = colors.get(row.key); + return color === undefined ? [] : [{ ...row, color }]; + }); + const rest = rows.filter((row) => !colors.has(row.key)); + return { parts: rest.length === 0 ? head : [...head, { ...sum(rest), color: OTHER }], rest }; +} + +/** Consecutive runs of `size`, the last run short when the items do not divide evenly. */ +export function chunks
(items: ReadonlyArray, size: number): ReadonlyArray> { + const runs: Array> = []; + for (let start = 0; start < items.length; start += size) runs.push(items.slice(start, start + size)); + return runs; +} + +/** + * A round number at or above `max` whose half is also round, so both ticks read clean: whole from 1 + * up, and decimal below it, where only a fraction like a cost can land. + */ +function niceMax(max: number) { + if (max === 0 || (max >= 1 && max <= 2)) return 2; + const half = max / 2; + const magnitude = 10 ** Math.floor(Math.log10(half)); + const step = + [1, 1.5, 2, 2.5, 3, 4, 5, 6, 8, 10].find( + (candidate) => candidate * magnitude >= half && (max < 1 || Number.isInteger(candidate * magnitude)), + ) ?? 10; + return 2 * step * magnitude; +} + +export function Card({ + title, + caption, + info, + action, + className = "", + children, +}: { + readonly title: string; + readonly caption?: string; + /** A note on how to read the card, after the caption. */ + readonly info?: ReactNode; + /** A control at the header's end. */ + readonly action?: ReactNode; + readonly className?: string; + readonly children: ReactNode; +}) { + return ( +
+
+

{title}

+ {caption !== undefined && ( + + {caption} + + )} + {info !== undefined && {info}} + {action} +
+ {children} +
+ ); +} + +/** A few mutually exclusive options in a row. */ +export function Segmented({ + options, + value, + onChange, + label, + className = "", +}: { + readonly options: ReadonlyArray<{ readonly key: Key; readonly label: string }>; + readonly value: Key; + readonly onChange: (key: Key) => void; + readonly label: string; + readonly className?: string; +}) { + return ( +
+ {options.map((option) => ( + + ))} +
+ ); +} + +// --------------------------------------------------------------------------------------------- +// Hover tooltip: one floating card that follows the pointer. + +interface Tip { + readonly x: number; + readonly y: number; + readonly content: ReactNode; +} + +interface TipStore { + readonly get: () => Tip | null; + readonly set: (tip: Tip | null) => void; + readonly subscribe: (listener: () => void) => () => void; +} + +const tipStore = (): TipStore => { + let tip: Tip | null = null; + const listeners = new Set<() => void>(); + return { + get: () => tip, + set: (next) => { + if (next === tip) return; + tip = next; + for (const listener of listeners) listener(); + }, + subscribe: (listener) => { + listeners.add(listener); + return () => listeners.delete(listener); + }, + }; +}; + +/** The tooltip lives in its own store, so following the pointer re-renders only the tooltip, never the chart. */ +function useTip() { + const [store] = useState(tipStore); + const show = useCallback( + (event: MouseEvent, content: ReactNode) => store.set({ x: event.clientX, y: event.clientY, content }), + [store], + ); + const hide = useCallback(() => store.set(null), [store]); + return { show, hide, element: }; +} + +function TipLayer({ store }: { readonly store: TipStore }) { + const tip = useSyncExternalStore(store.subscribe, store.get); + if (tip === null) return null; + const flip = tip.x > window.innerWidth - 260; + return ( +
+ {tip.content} +
+ ); +} + +/** A tooltip's body: a title, then one row per value with its swatch. */ +export function TipRows({ + title, + rows, +}: { + readonly title: string; + readonly rows: ReadonlyArray<{ readonly label: string; readonly value: string; readonly color?: string }>; +}) { + return ( +
+ {title} + {rows.map((row) => ( + + {row.color !== undefined && ( + + )} + {row.label} + {row.value} + + ))} +
+ ); +} + +// --------------------------------------------------------------------------------------------- + +interface Stack { + readonly label: string; + readonly color: string; + /** One value per column, oldest first. */ + readonly values: ReadonlyArray; + /** This stack's tooltip value, when the chart's `format` does not fit it. */ + readonly format?: (value: number) => string; +} + +/** + * Stacked columns over time on one axis. Columns cap at 24px and leave the rest of their slot as + * air; segments are parted by a 2px gap of the surface. + */ +export function Columns({ + stacks, + titles, + ticks, + format, + label, + height = 160, +}: { + readonly stacks: ReadonlyArray; + /** A column's name in its tooltip. */ + readonly titles: ReadonlyArray; + /** The few x labels worth showing, by column index. */ + readonly ticks: ReadonlyArray<{ readonly at: number; readonly label: string }>; + readonly format: (value: number) => string; + /** What the chart shows, for a screen reader. */ + readonly label: string; + readonly height?: number; +}) { + const tip = useTip(); + const count = titles.length; + const sums = titles.map((_, index) => stacks.reduce((sum, stack) => sum + (stack.values[index] ?? 0), 0)); + const top = niceMax(Math.max(...sums, 0)); + return ( + +
+ {sums.map((sum, index) => ( +
+ tip.show( + event, + ({ + label: stack.label, + value: (stack.format ?? format)(stack.values[index] ?? 0), + color: stack.color, + })), + { label: "Total", value: format(sum) }, + ] + } + />, + ) + } + > +
120 ? 0 : "12%" }} + > + {stacks.map((stack) => + (stack.values[index] ?? 0) === 0 ? null : ( + + ), + )} +
+
+ ))} +
+
+ ); +} + +/** The frame a time chart sits in: y labels at zero, half and top, a hairline grid, x ticks below. */ +function Plot({ + label, + top, + format, + height, + ticks, + count, + area, + onMouseMove, + onMouseLeave, + tooltip, + children, +}: { + readonly label: string; + readonly top: number; + readonly format: (value: number) => string; + readonly height: number; + readonly ticks: ReadonlyArray<{ readonly at: number; readonly label: string }>; + /** Columns the x axis is split into; a tick sits at its column's centre. */ + readonly count: number; + readonly area?: Ref; + readonly onMouseMove?: (event: MouseEvent) => void; + readonly onMouseLeave: () => void; + readonly tooltip: ReactNode; + readonly children: ReactNode; +}) { + return ( +
+
+
+ {format(top)} + {format(top / 2)} + {format(0)} +
+
+ {[0, 50, 100].map((position) => ( + + ))} + {children} +
+
+
+ {ticks.map((tick) => ( + = count - 1 + ? { right: 0 } + : { left: `${((tick.at + 0.5) / count) * 100}%`, transform: "translateX(-50%)" } + } + > + {tick.label} + + ))} +
+ {tooltip} +
+ ); +} + +interface Series { + readonly label: string; + readonly color: string; + /** One value per column, oldest first; undefined is a gap, not a zero. */ + readonly values: ReadonlyArray; +} + +/** The defined points, as [column, value] pairs. */ +const pointsOf = (values: ReadonlyArray) => + values.flatMap((value, index) => (value === undefined ? [] : [[index, value] as const])); + +/** + * Lines over time on one axis from zero. A line runs straight across the days it has no value, so + * a series used now and then still reads as a trend; the tooltip shows those days as empty. Drawn + * in measured pixels rather than a stretched viewBox, so markers stay round. + */ +export function Lines({ + series, + titles, + ticks, + format, + label, + height = 160, +}: { + readonly series: ReadonlyArray; + /** A column's name in its tooltip. */ + readonly titles: ReadonlyArray; + readonly ticks: ReadonlyArray<{ readonly at: number; readonly label: string }>; + readonly format: (value: number) => string; + readonly label: string; + readonly height?: number; +}) { + const tip = useTip(); + const area = useRef(null); + const [width, setWidth] = useState(0); + const [hover, setHover] = useState(undefined); + useLayoutEffect(() => { + const box = area.current; + if (box === null) return; + const fit = () => setWidth(box.clientWidth); + fit(); + const observer = new ResizeObserver(fit); + observer.observe(box); + return () => observer.disconnect(); + }, []); + const count = titles.length; + const defined = series.flatMap((line) => line.values.filter((value) => value !== undefined)); + const top = niceMax(Math.max(...defined, 0)); + const x = (index: number) => ((index + 0.5) / count) * width; + const y = (value: number) => height - (value / top) * height; + const dot = (key: string, index: number, value: number, color: string, radius: number) => ( + + ); + return ( + { + const box = event.currentTarget.getBoundingClientRect(); + const index = Math.min( + count - 1, + Math.max(0, Math.floor(((event.clientX - box.left) / box.width) * count)), + ); + setHover(index); + tip.show( + event, + { + const value = line.values[index]; + return { label: line.label, value: value === undefined ? "–" : format(value), color: line.color }; + })} + />, + ); + }} + onMouseLeave={() => { + setHover(undefined); + tip.hide(); + }} + tooltip={tip.element} + > + {width > 0 && ( + + {hover !== undefined && ( + + )} + {series.map((line) => { + const points = pointsOf(line.values); + const last = points.at(-1); + return ( + + `${at === 0 ? "M" : "L"}${x(index)} ${y(value)}`) + .join("")} + fill="none" + strokeWidth={2} + strokeLinejoin="round" + strokeLinecap="round" + style={{ stroke: line.color }} + /> + {last !== undefined && dot("end", last[0], last[1], line.color, 4)} + {hover !== undefined && + hover !== last?.[0] && + line.values[hover] !== undefined && + dot("hover", hover, line.values[hover], line.color, 3.5)} + + ); + })} + + )} + + ); +} + +/** Swatch and name for each series; present whenever a chart has more than one. */ +export function Legend({ + items, +}: { + readonly items: ReadonlyArray<{ readonly label: string; readonly color: string }>; +}) { + return ( +
    + {items.map((item) => ( +
  • + + {item.label} +
  • + ))} +
+ ); +} + +/** A trend at a glance: a 1.5px line over a faint wash, no axes. Decorative; its tile carries the number. */ +export function Spark({ + values, + color = SINGLE, + className = "h-8 w-full", +}: { + readonly values: ReadonlyArray; + readonly color?: string; + readonly className?: string; +}) { + const max = Math.max(...values, 0); + if (values.length < 2 || max === 0) + return ( + + + + ); + const points = values.map((value, index) => [(index / (values.length - 1)) * 100, 30 - (value / max) * 28] as const); + const line = points.map(([x, y], index) => `${index === 0 ? "M" : "L"}${x.toFixed(2)} ${y.toFixed(2)}`).join(""); + return ( + + + + + ); +} + +/** Parts of a whole on one line, parted by 2px gaps. Rows too small to see still get a sliver. */ +export function Share({ + parts, + format, + label, +}: { + readonly parts: ReadonlyArray<{ readonly label: string; readonly value: number; readonly color: string }>; + readonly format: (value: number) => string; + readonly label: string; +}) { + const tip = useTip(); + const sum = parts.reduce((all, part) => all + part.value, 0); + return ( +
+ {sum === 0 ? ( + + ) : ( + parts.map((part) => + part.value === 0 ? null : ( + + tip.show( + event, + , + ) + } + /> + ), + ) + )} + {tip.element} +
+ ); +} + +/** A value against the largest in its column: the bar a table row carries. */ +export function Meter({ + value, + max, + color = SINGLE, +}: { + readonly value: number; + readonly max: number; + readonly color?: string; +}) { + return ( + + 0 ? 2 : 0)}%`, + backgroundColor: color, + }} + /> + + ); +} + +/** + * The upper bounds of the lower three heat levels: quartiles of the active values, so one outlier day + * does not wash out the rest. + */ +export function heatCuts(values: ReadonlyArray): readonly [number, number, number] { + const active = values.filter((value) => value > 0).toSorted((a, b) => a - b); + const cut = (share: number) => active[Math.min(active.length - 1, Math.floor(active.length * share))] ?? 0; + return [cut(0.25), cut(0.5), cut(0.75)]; +} + +/** A calendar of days, a column per week with Sunday on top, shaded by `heatCuts` levels. */ +export function Heat({ + days, + cuts, + title, + onPick, + picked, + cell = 11, +}: { + /** Oldest first; `weekday` 0 is Sunday. */ + readonly days: ReadonlyArray<{ readonly ago: number; readonly weekday: number; readonly value: number }>; + readonly cuts: readonly [number, number, number]; + readonly title: (day: { readonly ago: number; readonly value: number }) => ReactNode; + readonly onPick?: (ago: number) => void; + readonly picked?: number; + readonly cell?: number; +}) { + const tip = useTip(); + const level = (value: number) => (value === 0 ? 0 : 1 + cuts.filter((threshold) => value > threshold).length); + const lead = days[0]?.weekday ?? 0; + return ( +
+ {Array.from({ length: lead }, (_, index) => ( + + ))} + {days.map((day) => { + const style = { backgroundColor: HEAT[level(day.value)] }; + const className = `rounded-[2px] ${picked === day.ago ? "outline-2 outline-offset-1 outline-ink" : ""}`; + return onPick === undefined ? ( + tip.show(event, title(day))} + /> + ) : ( +
+ ); +} + +/** The scale under a heatmap; each step names its range on hover. */ +export function HeatScale({ + cuts, + format, +}: { + readonly cuts: readonly [number, number, number]; + readonly format: (value: number) => string; +}) { + const [low, mid, high] = cuts; + const ranges = [ + "None", + `Up to ${format(low)}`, + `${format(low)} – ${format(mid)}`, + `${format(mid)} – ${format(high)}`, + `Over ${format(high)}`, + ]; + return ( + + Less + {HEAT.map((color, index) => ( + + ))} + More + + ); +} diff --git a/apps/web/src/metrics/data.ts b/apps/web/src/metrics/data.ts new file mode 100644 index 00000000..ed7ebb21 --- /dev/null +++ b/apps/web/src/metrics/data.ts @@ -0,0 +1,746 @@ +import { type MetricsHour, type MetricsRow, type MetricsUsage, readTopics } from "@codeworksh/contract"; +import { Effect, Stream } from "effect"; +import { Atom } from "effect/reactivity"; + +import { Server } from "../kernel/rpc"; + +/* + * The Metrics page reads a year of usage once, at the server's grain (local day × session × + * provider × model × thinking level), and derives every period, card and colour from it here. + */ + +export type RangeKey = "7d" | "30d" | "90d" | "1y"; +export const RANGES: ReadonlyArray<{ readonly key: RangeKey; readonly label: string; readonly days: number }> = [ + { key: "7d", label: "7 Days", days: 7 }, + { key: "30d", label: "30 Days", days: 30 }, + { key: "90d", label: "90 Days", days: 90 }, + { key: "1y", label: "Year", days: 365 }, +]; +const YEAR = 365; +const daysOf = (range: RangeKey) => RANGES.find(({ key }) => key === range)?.days ?? 30; + +// --------------------------------------------------------------------------------------------- +// Reads. + +/** What the page is filtered to; null is every one. */ +export interface Scope { + readonly machineId: string | null; + readonly projectId: string | null; +} + +const zone = Intl.DateTimeFormat().resolvedOptions().timeZone; + +const pad = (value: number) => String(value).padStart(2, "0"); +/** A local day as the server takes it, `YYYY-MM-DD`. */ +export const localDay = (date: Date) => `${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())}`; +/** Local noon `ago` days before `day`: safe to format across a clock change. */ +const noonOf = (day: string, ago: number) => { + const [year = 0, month = 1, date = 1] = day.split("-").map(Number); + return new Date(year, month - 1, date - ago, 12).getTime(); +}; + +const scoped = (scope: Scope) => ({ + ...(scope.machineId === null ? {} : { machineId: scope.machineId }), + ...(scope.projectId === null ? {} : { projectId: scope.projectId }), +}); + +/** The year ending `today` for `scope`, as a `usageAtom` key. */ +export const usageKey = (scope: Scope, today: string) => + JSON.stringify({ zone, since: localDay(new Date(noonOf(today, YEAR - 1))), until: today, ...scoped(scope) }); + +/** One day of `scope` by hour, as an `hoursAtom` key. */ +export const hoursKey = (scope: Scope, day: string) => JSON.stringify({ zone, day, ...scoped(scope) }); + +interface Request { + readonly zone: string; + readonly machineId?: string; + readonly projectId?: string; +} + +/** + * `read` when subscribed, and again after each change to usage; a failure ends it until refreshed. + * It outlives the page for a minute, so leaving and coming back shows the last read at once. + */ +const follow = (read: Effect.Effect) => + Server.runtime + .atom( + Stream.unwrap( + Server.use((client) => Effect.succeed(client("event.subscribe", { topics: readTopics.metrics }))), + ).pipe( + // `Synced` first, once subscribed: the first read can miss no change. + Stream.debounce("50 millis"), + Stream.mapEffect(() => read), + ), + ) + .pipe(Atom.setIdleTTL("1 minute")); + +export const usageAtom = Atom.family((key: string) => { + const request = JSON.parse(key) as Request & { readonly since: string; readonly until: string }; + return follow(Server.use((client) => client("metrics.usage", request))); +}); + +export const hoursAtom = Atom.family((key: string) => { + const request = JSON.parse(key) as Request & { readonly day: string }; + return follow(Server.use((client) => client("metrics.hours", request))); +}); + +// --------------------------------------------------------------------------------------------- +// The year, indexed. + +/** Every sum a row carries, each one additive. */ +const SUMMED = [ + "prompts", + "steps", + "input", + "output", + "reasoning", + "cacheRead", + "cacheWrite", + "cost", + "costInput", + "costOutput", + "costCacheRead", + "costCacheWrite", + "cacheSavings", + /** Ms, every timed request's duration summed. */ + "modelTime", + /** Requests with a duration: interrupted ones have none. */ + "timed", + /** Output of rows with any timed request, for speed. */ + "timedOutput", +] as const; +type Sums = Record<(typeof SUMMED)[number], number>; + +const zero = (): Sums => Object.fromEntries(SUMMED.map((field) => [field, 0])) as Sums; + +/** Bucket counts summed, bucket by bucket. */ +export const plus = (a: ReadonlyArray, b: ReadonlyArray) => + Array.from({ length: Math.max(a.length, b.length) }, (_, at) => (a[at] ?? 0) + (b[at] ?? 0)); + +export interface Project { + readonly id: string; + readonly name: string; +} +export interface Session { + readonly id: string; + readonly title: string; + readonly deleted: boolean; + readonly project: Project; +} + +/** A response row on its day. */ +interface Item extends Sums { + readonly ago: number; + readonly machineId: string; + readonly sessionId: string; + readonly provider: string; + readonly model: string; + readonly thinking: string; + readonly histogram: ReadonlyArray; +} + +/** Prompts, or one tool's calls, of a session on a day. */ +interface Count { + readonly ago: number; + readonly machineId: string; + readonly sessionId: string; + readonly name: string; + readonly count: number; +} + +export interface Totals extends Sums { + /** Sessions with any work in the period. */ + readonly sessions: number; + /** Every token billed: input, output, cache read and cache write. */ + readonly tokens: number; + readonly activeDays: number; + /** Timed requests per duration bucket. */ + readonly histogram: ReadonlyArray; +} + +interface Spend { + readonly tokens: number; + readonly cost: number; +} + +interface ModelSpend extends Spend { + readonly provider: string; + readonly model: string; + readonly timed: number; + readonly histogram: ReadonlyArray; +} + +interface SessionDay extends Spend { + readonly session: Session; + /** Threads worked that day. */ + readonly threads: number; +} + +export interface Day extends Totals { + readonly ago: number; + /** `YYYY-MM-DD`. */ + readonly day: string; + /** Local noon, for formatting. */ + readonly date: number; + /** The sessions worked that day, so a longer span can count each session once. */ + readonly worked: ReadonlyArray; + /** Its sessions' spend, most tokens first. */ + readonly bySession: ReadonlyArray; + /** By `modelKey`. */ + readonly models: ReadonlyMap; + readonly providers: ReadonlyMap; +} + +export interface Year { + /** Upper bounds in ms of the duration buckets. */ + readonly edges: ReadonlyArray; + /** Oldest first, every day of the year, quiet ones included. */ + readonly days: ReadonlyArray; + readonly items: ReadonlyArray; + readonly prompts: ReadonlyArray; + readonly tools: ReadonlyArray; + readonly sessions: ReadonlyMap; + readonly projects: ReadonlyArray; + readonly machines: ReadonlyArray<{ readonly id: string; readonly name: string }>; +} + +const UNKNOWN_PROJECT = "Unknown Project"; +const tokensOf = (sums: Pick) => + sums.input + sums.output + sums.cacheRead + sums.cacheWrite; +const modelKey = (provider: string, model: string) => `${provider}/${model}`; + +/** What cache reads saved over paying the input price for them, by the row's own prices. */ +const savingsOf = ({ tokens, cost }: MetricsRow) => + tokens.input > 0 && tokens.cacheRead > 0 + ? tokens.cacheRead * (cost.input / tokens.input - cost.cacheRead / tokens.cacheRead) + : 0; + +const itemOf = (row: MetricsRow, ago: number): Item => ({ + ago, + machineId: row.machineId, + sessionId: row.sessionId, + provider: row.provider, + model: row.model, + thinking: row.thinking, + histogram: row.durations, + prompts: 0, + steps: row.requests, + input: row.tokens.input, + output: row.tokens.output, + reasoning: row.tokens.reasoning, + cacheRead: row.tokens.cacheRead, + cacheWrite: row.tokens.cacheWrite, + cost: row.cost.total, + costInput: row.cost.input, + costOutput: row.cost.output, + costCacheRead: row.cost.cacheRead, + costCacheWrite: row.cost.cacheWrite, + cacheSavings: savingsOf(row), + modelTime: row.duration, + timed: row.timed, + timedOutput: row.timed > 0 ? row.tokens.output : 0, +}); + +function groupBy(items: ReadonlyArray
, key: (item: A) => K): Map> { + const groups = new Map>(); + for (const item of items) { + const group = groups.get(key(item)); + if (group === undefined) groups.set(key(item), [item]); + else group.push(item); + } + return groups; +} + +function total(items: ReadonlyArray, prompts: ReadonlyArray): Totals { + const sum = zero(); + let histogram: ReadonlyArray = []; + for (const item of items) { + for (const field of SUMMED) sum[field] += item[field]; + histogram = plus(histogram, item.histogram); + } + for (const prompt of prompts) sum.prompts += prompt.count; + return { + ...sum, + tokens: tokensOf(sum), + sessions: new Set([...items, ...prompts].map((item) => item.sessionId)).size, + activeDays: new Set(items.filter((item) => item.steps > 0).map((item) => item.ago)).size, + histogram, + }; +} + +const spendOf = (items: ReadonlyArray) => ({ + tokens: items.reduce((sum, item) => sum + tokensOf(item), 0), + cost: items.reduce((sum, item) => sum + item.cost, 0), +}); + +/** The response for the year ending `today`, indexed by day; a row outside the year is left out. */ +export function yearOf(usage: MetricsUsage, today: string): Year { + const dates = Array.from({ length: YEAR }, (_, ago) => localDay(new Date(noonOf(today, ago)))); + const agoOf = new Map(dates.map((day, ago) => [day, ago])); + const items = usage.rows.flatMap((row) => { + const ago = agoOf.get(row.day); + return ago === undefined ? [] : [itemOf(row, ago)]; + }); + const counts = (kind: "prompt" | "tool") => + usage.tallies.flatMap((tally) => { + const ago = agoOf.get(tally.day); + return tally.kind !== kind || ago === undefined + ? [] + : [{ ago, machineId: tally.machineId, sessionId: tally.sessionId, name: tally.name, count: tally.count }]; + }); + const prompts = counts("prompt"); + + const projects = usage.projects.map((project) => ({ id: project.id, name: project.name ?? UNKNOWN_PROJECT })); + const projectOf = new Map(projects.map((project) => [project.id, project])); + const sessions = new Map( + usage.sessions.map((session) => [ + session.id, + { + id: session.id, + title: session.title ?? "", + deleted: session.deleted, + project: projectOf.get(session.projectId) ?? { id: session.projectId, name: UNKNOWN_PROJECT }, + }, + ]), + ); + const threads = new Map(); + for (const row of usage.threads) { + const key = `${row.day}\u0000${row.sessionId}`; + threads.set(key, (threads.get(key) ?? 0) + row.threads); + } + + const itemsByDay = groupBy(items, (item) => item.ago); + const promptsByDay = groupBy(prompts, (prompt) => prompt.ago); + const days = dates + .map((day, ago): Day => { + const dayItems = itemsByDay.get(ago) ?? []; + const dayPrompts = promptsByDay.get(ago) ?? []; + const totals = total(dayItems, dayPrompts); + const models = new Map(); + for (const [key, group] of groupBy(dayItems, (item) => modelKey(item.provider, item.model))) { + const { provider, model } = group[0] ?? { provider: "", model: "" }; + const { timed, histogram } = total(group, []); + models.set(key, { ...spendOf(group), provider, model, timed, histogram }); + } + const providers = new Map( + [...groupBy(dayItems, (item) => item.provider)].map(([key, group]) => [key, spendOf(group)] as const), + ); + const worked = [...new Set([...dayItems, ...dayPrompts].map((item) => item.sessionId))]; + const byItems = groupBy(dayItems, (item) => item.sessionId); + return { + ...totals, + ago, + day, + date: noonOf(today, ago), + worked, + bySession: worked + .map((id) => ({ + ...spendOf(byItems.get(id) ?? []), + session: sessionOf(sessions, id), + threads: threads.get(`${day}\u0000${id}`) ?? 0, + })) + .sort((a, b) => b.tokens - a.tokens), + models, + providers, + }; + }) + .reverse(); + + return { + edges: usage.edges, + days, + items, + prompts, + tools: counts("tool"), + sessions, + projects, + machines: usage.machines, + }; +} + +/** A session the dictionary misses reads as deleted, so it folds away with them. */ +const sessionOf = (sessions: ReadonlyMap, id: string): Session => + sessions.get(id) ?? { id, title: "", deleted: true, project: { id: "", name: UNKNOWN_PROJECT } }; + +// --------------------------------------------------------------------------------------------- +// A period. + +type Dimension = "project" | "model" | "provider" | "thinking"; + +export interface SliceRow extends Totals { + readonly key: string; + readonly label: string; + /** A second line: a model's provider. */ + readonly hint: string | undefined; +} + +interface SessionRow extends Totals { + readonly session: Session; +} + +export interface Metrics { + readonly range: RangeKey; + /** Oldest first, every day of the period, quiet ones included. */ + readonly days: ReadonlyArray; + readonly totals: Totals; + /** The period before, for a change; the year has none. */ + readonly previous: Totals | undefined; + /** Consecutive active days ending today or yesterday. */ + readonly streak: number; + readonly bestStreak: number; + readonly sessions: ReadonlyArray; + /** Most calls first. */ + readonly tools: ReadonlyArray<{ readonly name: string; readonly calls: number }>; + readonly slice: (dimension: Dimension) => ReadonlyArray; + readonly edges: ReadonlyArray; +} + +/** Ordinal, least to most effort. */ +const THINKING_LEVELS = ["off", "minimal", "low", "medium", "high", "xhigh", "max"] as const; +const THINKING_LABEL: Partial> = { + off: "Off", + minimal: "Minimal", + low: "Low", + medium: "Medium", + high: "High", + xhigh: "Extra High", + max: "Max", +}; +/** A thinking level's place on the off → max scale. */ +export const levelIndex = (key: string) => THINKING_LEVELS.findIndex((level) => level === key); + +/** The metrics of a period of `year`. */ +export function query(year: Year, range: RangeKey): Metrics { + const span = daysOf(range); + const within = (list: ReadonlyArray, from: number, to: number) => + list.filter((item) => item.ago >= from && item.ago < to); + const items = within(year.items, 0, span); + const prompts = within(year.prompts, 0, span); + const days = year.days.slice(YEAR - span); + + let streak = 0; + for (let ago = (year.days.at(-1)?.steps ?? 0) > 0 ? 0 : 1; (year.days[YEAR - 1 - ago]?.steps ?? 0) > 0; ago++) + streak++; + let bestStreak = 0; + let run = 0; + for (const day of days) { + run = day.steps > 0 ? run + 1 : 0; + bestStreak = Math.max(bestStreak, run); + } + + const promptsBySession = groupBy(prompts, (prompt) => prompt.sessionId); + const itemsBySession = groupBy(items, (item) => item.sessionId); + const sessions = [...new Set([...itemsBySession.keys(), ...promptsBySession.keys()])].map((id) => ({ + ...total(itemsBySession.get(id) ?? [], promptsBySession.get(id) ?? []), + session: sessionOf(year.sessions, id), + })); + + const tools = new Map(); + for (const tool of within(year.tools, 0, span)) tools.set(tool.name, (tools.get(tool.name) ?? 0) + tool.count); + + return { + range, + days, + totals: total(items, prompts), + previous: + range === "1y" ? undefined : total(within(year.items, span, span * 2), within(year.prompts, span, span * 2)), + streak, + bestStreak, + sessions, + tools: [...tools].map(([name, calls]) => ({ name, calls })).sort((a, b) => b.calls - a.calls), + slice: (dimension) => { + const keyOf = (sessionId: string, item?: Item) => { + if (dimension === "project") { + const { project } = sessionOf(year.sessions, sessionId); + return { key: project.id, label: project.name, hint: undefined }; + } + if (item === undefined) return undefined; + if (dimension === "model") + return { key: modelKey(item.provider, item.model), label: item.model, hint: item.provider }; + if (dimension === "provider") return { key: item.provider, label: item.provider, hint: undefined }; + return { key: item.thinking, label: THINKING_LABEL[item.thinking] ?? item.thinking, hint: undefined }; + }; + const groups = new Map< + string, + { label: string; hint: string | undefined; items: Array; prompts: Array } + >(); + const into = (named: { key: string; label: string; hint: string | undefined }) => { + const group = groups.get(named.key) ?? { label: named.label, hint: named.hint, items: [], prompts: [] }; + groups.set(named.key, group); + return group; + }; + for (const item of items) { + const named = keyOf(item.sessionId, item); + if (named !== undefined) into(named).items.push(item); + } + // Prompts belong to a session, not a request, so only the project slice counts them. + if (dimension === "project") + for (const prompt of prompts) { + const named = keyOf(prompt.sessionId); + if (named !== undefined) into(named).prompts.push(prompt); + } + const rows = [...groups] + .map(([key, group]) => ({ + ...total(group.items, group.prompts), + key, + label: group.label, + hint: group.hint, + })) + // A model or provider whose requests all failed produced nothing; its requests still count by thinking level. + .filter((row) => row.tokens > 0 || dimension === "project" || dimension === "thinking"); + return dimension === "thinking" + ? rows.sort((a, b) => levelIndex(a.key) - levelIndex(b.key)) + : rows.sort((a, b) => b.prompts - a.prompts); + }, + edges: year.edges, + }; +} + +/** Each session once, across the given days. */ +export const sessionsIn = (days: ReadonlyArray): ReadonlyArray => [ + ...new Set(days.flatMap((day) => day.worked)), +]; + +/** A session list's line: a session, or every deleted one folded into one. */ +export type Listed = + | { + readonly kind: "session"; + readonly key: string; + readonly session: Session; + readonly threads: number | undefined; + readonly tokens: number; + readonly cost: number; + } + | { + readonly kind: "deleted"; + readonly key: string; + readonly count: number; + readonly tokens: number; + readonly cost: number; + }; + +/** Sessions most `by` first, the deleted ones summed into one line that sorts by its sum. */ +export function listed( + rows: ReadonlyArray<{ + readonly session: Session; + readonly tokens: number; + readonly cost: number; + readonly threads?: number; + }>, + by: "tokens" | "cost", +): ReadonlyArray { + const deleted = rows.filter((row) => row.session.deleted); + const lines: Array = rows + .filter((row) => !row.session.deleted) + .map((row) => ({ + kind: "session", + key: row.session.id, + session: row.session, + threads: row.threads, + tokens: row.tokens, + cost: row.cost, + })); + if (deleted.length > 0) + lines.push({ + kind: "deleted", + key: "deleted", + count: deleted.length, + tokens: deleted.reduce((sum, row) => sum + row.tokens, 0), + cost: deleted.reduce((sum, row) => sum + row.cost, 0), + }); + return lines.sort((a, b) => b[by] - a[by]); +} + +interface ModelDay extends Spend { + readonly key: string; + readonly provider: string; + readonly model: string; + /** Its median request duration that day; none when no request was timed. */ + readonly median: Quantile | undefined; + /** Tokens in each local hour, 0 to 23. */ + readonly hours: ReadonlyArray; +} + +/** A day's models, most tokens first, with their tokens by hour from that day's hourly read. */ +export function modelsOn(day: Day, edges: ReadonlyArray, hours: ReadonlyArray) { + const byModel = groupBy(hours, (row) => modelKey(row.provider, row.model)); + return [...day.models] + .filter(([, spend]) => spend.tokens > 0) + .map(([key, spend]): ModelDay => { + const perHour = Array.from({ length: 24 }, () => 0); + for (const row of byModel.get(key) ?? []) perHour[row.hour] = (perHour[row.hour] ?? 0) + row.tokens; + return { + key, + provider: spend.provider, + model: spend.model, + tokens: spend.tokens, + cost: spend.cost, + median: spend.timed === 0 ? undefined : quantile(spend.histogram, edges, 0.5), + hours: perHour, + }; + }) + .sort((a, b) => b.tokens - a.tokens); +} + +export interface ProjectOption { + readonly project: Project; + readonly prompts: number; + readonly sessions: number; +} + +/** Every project's prompts and active sessions in a period, on one machine or all. */ +export function projectOptions(year: Year, range: RangeKey, machineId: string | null): ReadonlyArray { + const span = daysOf(range); + const rows = new Map(year.projects.map((project) => [project.id, { prompts: 0, sessions: new Set() }])); + for (const item of [...year.items, ...year.prompts]) { + if (item.ago >= span || (machineId !== null && item.machineId !== machineId)) continue; + const row = rows.get(sessionOf(year.sessions, item.sessionId).project.id); + if (row === undefined) continue; + row.sessions.add(item.sessionId); + if ("count" in item) row.prompts += item.count; + } + return year.projects.map((project) => { + const row = rows.get(project.id); + return { project, prompts: row?.prompts ?? 0, sessions: row?.sessions.size ?? 0 }; + }); +} + +// --------------------------------------------------------------------------------------------- +// Response time, from duration buckets. + +/** A duration read from buckets; `open` when it fell in the last, unbounded one: at least `ms`. */ +interface Quantile { + readonly ms: number; + readonly open: boolean; +} + +/** + * The `q` quantile of bucket counts, interpolated linearly inside its bucket. Bucket i spans + * `edges[i - 1]` to `edges[i]`, the first from 0; the last is open-ended, so its lower edge stands in. + */ +export function quantile(histogram: ReadonlyArray, edges: ReadonlyArray, q: number) { + const sum = histogram.reduce((all, count) => all + count, 0); + if (sum === 0) return undefined; + const target = q * sum; + let seen = 0; + for (const [at, count] of histogram.entries()) { + if (count > 0 && seen + count >= target) { + const low = at === 0 ? 0 : (edges[at - 1] ?? 0); + const high = edges[at]; + if (high === undefined) return { ms: low, open: true } satisfies Quantile; + return { ms: low + ((high - low) * (target - seen)) / count, open: false } satisfies Quantile; + } + seen += count; + } + return undefined; +} + +export interface Response { + readonly median: Quantile; + readonly p95: Quantile; + /** Output tokens per second of request time; none when no time was recorded. */ + readonly speed: number | undefined; +} + +/** Response time over timed requests; durations are right-skewed, so median and p95, never a mean. */ +export function responseOf( + totals: Pick, + edges: ReadonlyArray, +): Response | undefined { + if (totals.timed === 0) return undefined; + const median = quantile(totals.histogram, edges, 0.5); + const p95 = quantile(totals.histogram, edges, 0.95); + if (median === undefined || p95 === undefined) return undefined; + return { + median, + p95, + speed: totals.modelTime > 0 ? totals.timedOutput / (totals.modelTime / 1000) : undefined, + }; +} + +/** + * Names a model by its id, led by its provider when the id occurs under more than one provider in + * `rows`: the provider is what tells those apart, so a truncated label must keep it. + */ +export function modelNamer(rows: ReadonlyArray<{ readonly provider: string; readonly model: string }>) { + const providers = new Map>(); + for (const row of rows) providers.set(row.model, (providers.get(row.model) ?? new Set()).add(row.provider)); + return (row: { readonly provider: string; readonly model: string }) => + (providers.get(row.model)?.size ?? 0) > 1 ? `${row.provider} · ${row.model}` : row.model; +} + +/** A free or preview model reports tokens but no cost; "$0" would read as a bargain. */ +export const unpriced = (spend: Spend) => spend.cost === 0 && spend.tokens > 0; + +// --------------------------------------------------------------------------------------------- +// Formatting. + +const whole = new Intl.NumberFormat("en-US", { maximumFractionDigits: 0 }); +const compact = new Intl.NumberFormat("en-US", { notation: "compact", maximumFractionDigits: 1 }); +const dollars = new Intl.NumberFormat("en-US", { style: "currency", currency: "USD" }); +const dollarsWhole = new Intl.NumberFormat("en-US", { style: "currency", currency: "USD", maximumFractionDigits: 0 }); +const dollarsSmall = new Intl.NumberFormat("en-US", { + style: "currency", + currency: "USD", + minimumFractionDigits: 0, + maximumFractionDigits: 4, +}); +const shortDay = new Intl.DateTimeFormat(undefined, { month: "short", day: "numeric" }); +const longDay = new Intl.DateTimeFormat(undefined, { + weekday: "short", + month: "short", + day: "numeric", + year: "numeric", +}); + +export const formatCount = (value: number) => whole.format(value); +/** 48.3K, 1.2M, 2.4B. */ +export const formatTokens = (value: number) => compact.format(value); +/** Whole dollars from a thousand, cents from a cent, up to four decimals below; "<$0.0001" rather than a rounded zero. */ +export const formatCost = (value: number) => + value === 0 + ? "$0" + : value >= 1000 + ? dollarsWhole.format(value) + : value >= 0.01 + ? dollars.format(value) + : value >= 0.0001 + ? dollarsSmall.format(value) + : "<$0.0001"; +/** A day by its local noon. */ +export const formatDay = (date: number) => shortDay.format(date); +export const formatDayLong = (date: number) => longDay.format(date); +/** 850 ms, 4.6 s, 1m 12s, 30m, 2h 05m. */ +export function formatDuration(ms: number) { + if (ms === 0) return "0 s"; + if (Math.round(ms) < 1000) return `${Math.round(ms)} ms`; + const tenths = Math.round(ms / 100); + if (tenths < 600) return `${(tenths / 10).toFixed(1)} s`; + const seconds = Math.round(ms / 1000); + if (seconds < 3600) return `${Math.floor(seconds / 60)}m${seconds % 60 === 0 ? "" : ` ${pad(seconds % 60)}s`}`; + const minutes = Math.round(ms / 60_000); + return `${Math.floor(minutes / 60)}h ${pad(minutes % 60)}m`; +} +/** "4.6 s", or "≥ 30m" past the last bucket. */ +export const formatQuantile = (value: Quantile) => `${value.open ? "≥ " : ""}${formatDuration(value.ms)}`; +/** 52 tok/s. */ +export const formatSpeed = (perSecond: number) => `${Math.round(perSecond)} tok/s`; +export const formatPercent = (ratio: number) => `${Math.round(ratio * 100)}%`; +/** The change from `before` to `now` as a signed percent, or undefined when there is nothing to compare. */ +export function change(now: number, before: number | undefined) { + if (before === undefined || before === 0) return undefined; + return (now - before) / before; +} +/** Unsigned: the badge's arrow carries the direction. */ +export const formatChange = (ratio: number) => + Math.abs(ratio) >= 10 ? ">999%" : `${Math.round(Math.abs(ratio) * 100)}%`; +/** A change between two rates, unsigned: "10 pts". */ +export const formatPoints = (difference: number) => `${Math.round(Math.abs(difference) * 100)} pts`; +/** + * Share of input served from cache, or null with no input. A cache write counts as a miss: that + * input was processed in full. + */ +export function cacheHit(totals: Pick) { + const all = totals.input + totals.cacheRead + totals.cacheWrite; + return all === 0 ? null : totals.cacheRead / all; +} diff --git a/apps/web/src/metrics/filters.tsx b/apps/web/src/metrics/filters.tsx new file mode 100644 index 00000000..a4ac152d --- /dev/null +++ b/apps/web/src/metrics/filters.tsx @@ -0,0 +1,243 @@ +import { CaretUpDownIcon, CheckIcon, DesktopIcon, SquaresFourIcon } from "@phosphor-icons/react"; +import { type ReactNode, useEffect, useState } from "react"; + +import { Command, CommandEmpty, CommandGroup, CommandInput, CommandItem, CommandList } from "@/components/ui/command"; +import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover"; +import { Monogram } from "@/components/monogram"; +import { formatCount, type Project, type ProjectOption } from "./data"; + +const ALL = "all"; +const ROW = "min-h-7 gap-2 py-1 text-[13px]"; +const GROUP = "[&_[cmdk-group-heading]]:text-2xs [&_[cmdk-group-heading]]:pb-1"; + +/** cmdk filters on the value, so it carries the name; the id keeps two same-named projects apart. */ +const valueOf = (item: { readonly id: string; readonly name: string }) => `${item.name}\u0000${item.id}`; + +/** Whether a key press belongs to a field rather than to the page's shortcuts. */ +const typing = (target: EventTarget | null) => + target instanceof HTMLElement && + (target.isContentEditable || target.closest("input, textarea, select, [role=textbox]") !== null); + +/** The current pick's check, or the same room empty, so counts line up on every row. */ +function Check({ on }: { readonly on: boolean }) { + return on ? ( + + ) : ( + + ); +} + +/** A header filter's button: what is picked, and the list it opens. */ +function Filter({ + label, + icon, + name, + open, + onOpenChange, + width, + children, +}: { + /** What it filters, for a screen reader: "Project". */ + readonly label: string; + readonly icon: ReactNode; + readonly name: string; + readonly open: boolean; + readonly onOpenChange: (open: boolean) => void; + readonly width: string; + readonly children: ReactNode; +}) { + return ( + + + + + + {children} + + + ); +} + +/** The page's project scope: every project, or one picked from a filterable list ordered by activity. */ +export function ProjectSelect({ + value, + onChange, + options, +}: { + readonly value: string | null; + readonly onChange: (projectId: string | null) => void; + readonly options: ReadonlyArray; +}) { + const [open, setOpen] = useState(false); + const [search, setSearch] = useState(""); + const [highlight, setHighlight] = useState(ALL); + + const active = options.filter((row) => row.prompts > 0).toSorted((a, b) => b.prompts - a.prompts); + const idle = options + .filter((row) => row.prompts === 0) + .toSorted((a, b) => a.project.name.localeCompare(b.project.name)); + const sessions = options.reduce((sum, row) => sum + row.sessions, 0); + const selected = options.find((row) => row.project.id === value)?.project; + + const show = (next: boolean) => { + if (next) { + setSearch(""); + setHighlight(selected === undefined ? ALL : valueOf(selected)); + } + setOpen(next); + }; + const pick = (projectId: string | null) => { + onChange(projectId); + setOpen(false); + }; + + useEffect(() => { + const onKeyDown = (event: KeyboardEvent) => { + if (event.key.toLowerCase() !== "p" || event.metaKey || event.ctrlKey || event.altKey || event.shiftKey) + return; + if (event.repeat || typing(event.target)) return; + event.preventDefault(); + show(true); + }; + window.addEventListener("keydown", onKeyDown); + return () => window.removeEventListener("keydown", onKeyDown); + }); + + const row = (project: Project, prompts: number | undefined) => ( + pick(project.id)} className={ROW}> + + + {project.name} + + + {prompts !== undefined && formatCount(prompts)} + + + + ); + + return ( + + ) : ( + + ) + } + name={selected?.name ?? "All Projects"} + open={open} + onOpenChange={show} + width="w-96" + > + +
+ + + P + +
+ + + No Projects Match “{search.trim()}” + + + pick(null)} className={ROW}> + + All Projects + + {formatCount(sessions)} + + + + + {active.length > 0 && ( + + {active.map(({ project, prompts }) => row(project, prompts))} + + )} + {idle.length > 0 && ( + + {idle.map(({ project }) => row(project, undefined))} + + )} + +
+
+ ); +} + +/** The page's machine scope: every machine, or one by name. */ +export function MachineSelect({ + value, + onChange, + machines, +}: { + readonly value: string | null; + readonly onChange: (machineId: string | null) => void; + readonly machines: ReadonlyArray<{ readonly id: string; readonly name: string }>; +}) { + const [open, setOpen] = useState(false); + const sorted = machines.toSorted((a, b) => a.name.localeCompare(b.name)); + const selected = machines.find((machine) => machine.id === value); + const pick = (machineId: string | null) => { + onChange(machineId); + setOpen(false); + }; + return ( + } + name={selected?.name ?? "All Machines"} + open={open} + onOpenChange={setOpen} + width="w-64" + > + + + + pick(null)} className={ROW}> + All Machines + + + + {sorted.map((machine) => ( + pick(machine.id)} + className={ROW} + > + + {machine.name} + + + + + ))} + + + + + ); +} diff --git a/apps/web/src/metrics/metrics.tsx b/apps/web/src/metrics/metrics.tsx new file mode 100644 index 00000000..a8f2d7dc --- /dev/null +++ b/apps/web/src/metrics/metrics.tsx @@ -0,0 +1,921 @@ +import { ChartBarIcon, InfoIcon } from "@phosphor-icons/react"; +import { type ReactNode, useMemo, useState } from "react"; + +import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover"; +import { ProviderIcon } from "@/components/provider"; +import { Activity, Weeks } from "./activity"; +import { + Card, + chunks, + Columns, + fold, + ink, + Legend, + Lines, + Meter, + OTHER, + palette, + RAMP, + Segmented, + Share, + Spark, +} from "./charts"; +import { + cacheHit, + change, + type Day, + formatChange, + formatCost, + formatCount, + formatDay, + formatDuration, + formatPercent, + formatPoints, + formatQuantile, + formatSpeed, + formatTokens, + levelIndex, + listed, + type Metrics, + modelNamer, + plus, + quantile, + RANGES, + responseOf, + type Scope, + sessionsIn, + type SliceRow, + type Totals, + unpriced, +} from "./data"; +import { SessionLine } from "./session"; + +/** The page below the header: the period's metrics, already scoped to the chosen machine and project. */ +export function MetricsBody({ + year, + metrics, + scope, + projectName, + onProject, +}: { + /** The whole year, for the calendar and the colours. */ + readonly year: Metrics; + readonly metrics: Metrics; + readonly scope: Scope; + /** The project the page is scoped to, for an empty state to name. */ + readonly projectName: string | undefined; + /** Scopes the page to a project, or back to all with null. */ + readonly onProject: (projectId: string | null) => void; +}) { + // Colors follow the year's ranking, so a model keeps its color in every period and card. + const modelColors = useMemo(() => palette(byTokens(year.slice("model")).map((row) => row.key)), [year]); + const providerColors = useMemo(() => palette(byTokens(year.slice("provider")).map((row) => row.key)), [year]); + const models = useMemo( + () => fold(byTokens(metrics.slice("model")).map(partOf), modelColors, rest), + [metrics, modelColors], + ); + const providers = useMemo( + () => fold(byTokens(metrics.slice("provider")).map(partOf), providerColors, rest), + [metrics, providerColors], + ); + // Where a label shows only the model id, an id under two providers also names its provider. + const named = useMemo(() => { + const name = modelNamer(models.parts.flatMap(modelOf).concat(models.rest.flatMap(modelOf))); + return models.parts.map((part) => { + const [model] = modelOf(part); + return model === undefined ? part : { ...part, label: name(model) }; + }); + }, [models]); + const period = RANGES.find(({ key }) => key === metrics.range)?.label.toLowerCase() ?? ""; + if (year.totals.steps === 0) return ; + return ( +
+ + + +
+ + +
+ + + + + + +
+ ); +} + +function Empty({ projectName, period }: { readonly projectName: string | undefined; readonly period: string }) { + return ( +
+ + + +
+

No Activity Yet

+

+ Nothing ran in {projectName ?? "any project"} over the last{" "} + {period}. +

+
+
+ ); +} + +// --------------------------------------------------------------------------------------------- + +type Stat = "cost" | "tokens" | "sessions"; + +/** A year of daily points is too dense for a small chart; weeks read better. */ +const buckets = (metrics: Metrics) => chunks(metrics.days, metrics.range === "1y" ? 7 : 1); + +const sumOf = (days: ReadonlyArray, pick: (day: Day) => number) => days.reduce((sum, day) => sum + pick(day), 0); + +/** Each session counted once per bucket. */ +const trend = (metrics: Metrics, key: Stat) => + buckets(metrics).map((days) => (key === "sessions" ? sessionsIn(days).length : sumOf(days, (day) => day[key]))); + +const hitLabel = (totals: Pick) => { + const hit = cacheHit(totals); + return hit === null ? "–" : formatPercent(hit); +}; + +/** A change badge in neutral ink, or nothing when there is nothing to compare. */ +function Delta({ + ratio, + period, + format = formatChange, +}: { + readonly ratio: number | undefined; + readonly period: string; + readonly format?: (ratio: number) => string; +}) { + if (ratio === undefined) return null; + return ( + + {ratio >= 0 ? "↑" : "↓"} {format(ratio)} + + ); +} + +function Stats({ metrics, period }: { readonly metrics: Metrics; readonly period: string }) { + const { totals, previous } = metrics; + const tiles: ReadonlyArray<{ key: Stat; label: string; value: string; hint: string | undefined }> = [ + { + key: "cost", + label: "Cost", + value: formatCost(totals.cost), + hint: totals.sessions === 0 ? undefined : `${formatCost(totals.cost / totals.sessions)} per Session`, + }, + { + key: "tokens", + label: "Tokens", + value: formatTokens(totals.tokens), + hint: `${hitLabel(totals)} Cached`, + }, + { + key: "sessions", + label: "Sessions", + value: formatCount(totals.sessions), + hint: `${formatCount(totals.prompts)} Prompts`, + }, + ]; + return ( +
+ {tiles.map((tile) => ( +
+

{tile.label}

+
+ + {tile.value} + + +
+ {tile.hint} + +
+ ))} +
+ ); +} + +// --------------------------------------------------------------------------------------------- + +interface Part { + readonly key: string; + readonly label: string; + readonly hint: string | undefined; + readonly sessions: number; + readonly prompts: number; + readonly tokens: number; + readonly cost: number; + readonly input: number; + readonly cacheRead: number; + readonly cacheWrite: number; + readonly output: number; + readonly modelTime: number; + readonly timed: number; + readonly timedOutput: number; + readonly histogram: ReadonlyArray; +} +type Colored = Part & { readonly color: string }; + +const partOf = (row: SliceRow): Part => ({ + key: row.key, + label: row.label, + hint: row.hint, + sessions: row.sessions, + prompts: row.prompts, + tokens: row.tokens, + cost: row.cost, + input: row.input, + cacheRead: row.cacheRead, + cacheWrite: row.cacheWrite, + output: row.output, + modelTime: row.modelTime, + timed: row.timed, + timedOutput: row.timedOutput, + histogram: row.histogram, +}); + +const OTHER_KEY = "other"; + +/** A model slice row's provider and id: its hint and label; Other has none. */ +const modelOf = (part: Part) => + part.key === OTHER_KEY || part.hint === undefined ? [] : [{ provider: part.hint, model: part.label }]; + +const byTokens = (rows: ReadonlyArray) => rows.toSorted((a, b) => b.tokens - a.tokens); + +function rest(rows: ReadonlyArray): Part { + const sum = (pick: (row: Part) => number) => rows.reduce((total, row) => total + pick(row), 0); + return { + key: OTHER_KEY, + label: "Other", + hint: undefined, + sessions: sum((row) => row.sessions), + prompts: sum((row) => row.prompts), + tokens: sum((row) => row.tokens), + cost: sum((row) => row.cost), + input: sum((row) => row.input), + cacheRead: sum((row) => row.cacheRead), + cacheWrite: sum((row) => row.cacheWrite), + output: sum((row) => row.output), + modelTime: sum((row) => row.modelTime), + timed: sum((row) => row.timed), + timedOutput: sum((row) => row.timedOutput), + histogram: rows.reduce>((all, row) => plus(all, row.histogram), []), + }; +} + +type Metric = "tokens" | "cost"; +const METRICS: ReadonlyArray<{ readonly key: Metric; readonly label: string }> = [ + { key: "tokens", label: "Tokens" }, + { key: "cost", label: "Cost" }, +]; + +/** Up to four evenly spread column indices: both ends and two between. */ +const spread = (count: number) => + [...new Set([0, Math.round((count - 1) / 3), Math.round(((count - 1) * 2) / 3), count - 1])].filter((at) => at >= 0); + +/** The period's columns, a day each or a week for the year, with their tooltip names and x ticks. */ +function timeline(metrics: Metrics) { + const weekly = metrics.range === "1y"; + const columns = buckets(metrics); + const startOf = (at: number) => columns[at]?.[0]?.date ?? 0; + return { + weekly, + columns, + titles: columns.map((_, at) => (weekly ? `Week of ${formatDay(startOf(at))}` : formatDay(startOf(at)))), + ticks: spread(columns.length).map((at) => ({ at, label: formatDay(startOf(at)) })), + }; +} + +type View = "model" | "provider"; +const VIEWS: ReadonlyArray<{ readonly key: View; readonly label: string }> = [ + { key: "model", label: "Model" }, + { key: "provider", label: "Provider" }, +]; + +function Work({ + metrics, + models, + providers, +}: { + readonly metrics: Metrics; + readonly models: ReadonlyArray; + readonly providers: ReadonlyArray; +}) { + const [metric, setMetric] = useState("tokens"); + const [view, setView] = useState("model"); + const { weekly, columns, titles, ticks } = timeline(metrics); + const parts = view === "model" ? models : providers; + const named = parts.filter((part) => part.key !== OTHER_KEY); + const valueOf = (days: ReadonlyArray, key: string) => + sumOf(days, (day) => (view === "model" ? day.models : day.providers).get(key)?.[metric] ?? 0); + const stacks = parts.map((model) => ({ + label: model.label, + color: model.color, + ...(metric === "cost" && unpriced(model) ? { format: () => "Unpriced" } : {}), + values: columns.map((days) => + model.key === OTHER_KEY + ? Math.max( + 0, + sumOf(days, (day) => day[metric]) - named.reduce((sum, head) => sum + valueOf(days, head.key), 0), + ) + : valueOf(days, model.key), + ), + })); + const noun = metric === "tokens" ? "Tokens" : "Cost"; + return ( + + + +
+ } + > + + + + ); +} + +// --------------------------------------------------------------------------------------------- + +interface Column { + readonly label: string; + readonly cell: (part: Colored) => ReactNode; + /** Shown only when the card is wide enough. */ + readonly extra?: true; +} + +function Breakdown({ + title, + caption, + head, + noun, + parts, + rest, + by, + grid, + columns, + describe, + provider = false, + onPick, +}: { + readonly title: string; + readonly caption: string; + /** The first column's heading. */ + readonly head: string; + /** What a row is, plural: "Show All 12 Models". */ + readonly noun: string; + readonly parts: ReadonlyArray; + /** The rows folded into Other, listed under it on demand. */ + readonly rest: ReadonlyArray; + readonly by: "prompts" | "tokens"; + /** The row's grid template: the name, then one track per column. */ + readonly grid: string; + readonly columns: ReadonlyArray; + /** A row's tooltip, for what hidden columns would have said. */ + readonly describe?: (part: Colored) => string; + /** The hint is a provider, so it wears the provider's mark. */ + readonly provider?: boolean; + /** Makes each row, Other aside, a button. */ + readonly onPick?: (key: string) => void; +}) { + const [open, setOpen] = useState(false); + const row = `grid ${grid} items-center gap-x-3`; + const pressable = `${row} -mx-2 w-[calc(100%+16px)] rounded-md px-2 py-2 text-left outline-none hover:bg-ink/4 focus-visible:ring-2 focus-visible:ring-ring`; + const hide = (column: Column) => (column.extra === true ? "hidden @sm:block" : ""); + const body = (part: Colored, hint: string | undefined) => ( + <> + + + + {part.label} + {hint !== undefined && ( + + {provider && part.key !== OTHER_KEY && } + {hint} + + )} + + + {columns.map((column) => ( + + {column.cell(part)} + + ))} + + ); + const item = (part: Colored) => ( +
  • + {onPick === undefined ? ( +
    {body(part, part.hint)}
    + ) : ( + + )} +
  • + ); + return ( + + ({ label: part.label, value: part[by], color: part.color }))} + format={by === "tokens" ? formatTokens : formatCount} + label={`${title} by ${by}`} + /> +
    +
    + {head} + {columns.map((column) => ( + + {column.label} + + ))} +
    +
      + {parts.map((part) => + part.key !== OTHER_KEY ? ( + item(part) + ) : ( +
    • + + {open && ( +
        + {rest + .toSorted((a, b) => b[by] - a[by]) + .map((folded) => item({ ...folded, color: OTHER }))} +
      + )} +
    • + ), + )} +
    +
    +
    + ); +} + +const muted = (text: string) => {text}; +const secondary = (text: string) => {text}; +const strong = (text: string) => {text}; + +/** The project breakdown, each row scoping the page to its project. */ +function Projects({ + rows, + projectId, + projectName, + onProject, +}: { + readonly rows: ReadonlyArray; + readonly projectId: string | null; + readonly projectName: string | undefined; + readonly onProject: (projectId: string | null) => void; +}) { + if (projectId === null) { + const { parts, rest: folded } = fold(rows.map(partOf), palette(rows.map((row) => row.key)), rest); + return ( + secondary(formatCount(part.sessions)) }, + { label: "Tokens", cell: (part) => muted(formatTokens(part.tokens)), extra: true }, + { label: "Cost", cell: (part) => strong(formatCost(part.cost)) }, + ]} + onPick={onProject} + /> + ); + } + return ( + +
    + + Showing {projectName ?? rows[0]?.label ?? "Project"} + + +
    +
    + ); +} + +const perMillion = (part: Part) => + part.tokens === 0 || unpriced(part) ? "–" : formatCost((part.cost / part.tokens) * 1_000_000); + +function Models({ + parts, + rest, + edges, +}: { + readonly parts: ReadonlyArray; + readonly rest: ReadonlyArray; + readonly edges: ReadonlyArray; +}) { + return ( + secondary(formatTokens(part.tokens)) }, + { label: "Cached", cell: (part) => muted(hitLabel(part)), extra: true }, + { label: "$/M", cell: (part) => muted(perMillion(part)), extra: true }, + { + label: "Cost", + cell: (part) => (unpriced(part) ? muted("Unpriced") : strong(formatCost(part.cost))), + }, + ]} + describe={(part) => { + const response = responseOf(part, edges); + const speed = response?.speed === undefined ? "" : ` · ${formatSpeed(response.speed)}`; + const pace = response === undefined ? "" : ` · Median ${formatQuantile(response.median)}${speed}`; + return `${hitLabel(part)} Cached · ${perMillion(part)}/M${pace}`; + }} + provider + /> + ); +} + +// --------------------------------------------------------------------------------------------- + +const RESPONSE_ROW = "grid grid-cols-[minmax(0,1fr)_52px_52px_60px] items-center gap-x-3"; + +/** + * Median request duration over time for the page's coloured models, beside a table that doubles as + * the legend. Other gets no line: a median across unrelated models means nothing. + */ +function ResponseTime({ + metrics, + models, + more, +}: { + readonly metrics: Metrics; + readonly models: ReadonlyArray; + /** Models folded into Other. */ + readonly more: number; +}) { + const { columns, titles, ticks } = timeline(metrics); + const named = models.filter((model) => model.key !== OTHER_KEY); + const overall = responseOf(metrics.totals, metrics.edges); + return ( + } + action={ + overall !== undefined && ( + + Median {formatQuantile(overall.median)} · P95{" "} + {formatQuantile(overall.p95)} + + ) + } + > + {overall === undefined ? ( +

    No Requests

    + ) : ( +
    + ({ + label: model.label, + color: model.color, + values: columns.map((days) => { + const spent = days.flatMap((day) => day.models.get(model.key) ?? []); + // No timed request is a gap, never a zero. + return spent.some((spend) => spend.timed > 0) + ? quantile( + spent.reduce>((all, spend) => plus(all, spend.histogram), []), + metrics.edges, + 0.5, + )?.ms + : undefined; + }), + }))} + titles={titles} + ticks={ticks} + format={formatDuration} + label="Median response time by model" + height={168} + /> +
    +
    + Model + Median + P95 + Speed +
    +
      + {named.map((model) => { + const response = responseOf(model, metrics.edges); + return ( +
    • + + + {response === undefined ? "–" : formatQuantile(response.median)} + + + {response === undefined ? "–" : formatQuantile(response.p95)} + + + {response?.speed === undefined ? "–" : formatSpeed(response.speed)} + +
    • + ); + })} + {more > 0 && ( +
    • + {formatCount(more)} More {more === 1 ? "Model" : "Models"} +
    • + )} +
    +
    +
    + )} +
    + ); +} + +/** What the response figures are estimated from, on click or Enter. */ +function Estimated() { + return ( + + + + + + Estimated From Duration Buckets +

    + Median and P95 are accurate to one duration bucket. Interrupted requests are not timed. +

    +
    +
    + ); +} + +// --------------------------------------------------------------------------------------------- + +/** A swatch and a name: the first cell of a legend row. */ +function Swatch({ color, label }: { readonly color: string; readonly label: string }) { + return ( + + + {label} + + ); +} + +const TOOLS_SHOWN = 12; +const TOOL_ROW = "grid grid-cols-[minmax(0,1fr)_64px_48px_36px] items-center gap-x-3"; + +/** Calls per tool, ranked down each column so the list fills the row. */ +function Tools({ metrics }: { readonly metrics: Metrics }) { + const { tools } = metrics; + const callsOf = (list: ReadonlyArray<{ readonly calls: number }>) => list.reduce((all, tool) => all + tool.calls, 0); + const sum = callsOf(tools); + const shown = tools.slice(0, TOOLS_SHOWN); + const hidden = tools.slice(TOOLS_SHOWN); + const top = shown[0]?.calls ?? 0; + const share = (calls: number) => (sum === 0 ? "–" : formatPercent(calls / sum)); + const item = `${TOOL_ROW} break-inside-avoid border-b border-edge/50 py-1.5`; + return ( + + {tools.length === 0 ? ( +

    No Tool Calls

    + ) : ( +
      + {shown.map((tool) => ( +
    1. + + {tool.name} + + + {formatCount(tool.calls)} + {share(tool.calls)} +
    2. + ))} + {hidden.length > 0 && ( +
    3. + + Other + {formatCount(hidden.length)} more + + + {formatCount(callsOf(hidden))} + {share(callsOf(hidden))} +
    4. + )} +
    + )} +
    + ); +} + +// --------------------------------------------------------------------------------------------- + +const SESSION_ORDER: ReadonlyArray<{ readonly key: "cost" | "tokens"; readonly label: string }> = [ + { key: "cost", label: "Cost" }, + { key: "tokens", label: "Tokens" }, +]; + +function TopSessions({ metrics }: { readonly metrics: Metrics }) { + const [by, setBy] = useState<"cost" | "tokens">("cost"); + const top = listed(metrics.sessions, by).slice(0, 8); + return ( + } + > +
      + {top.map((line) => ( + + ))} +
    +
    + ); +} + +/** Requests by the reasoning effort asked for, on its ordered off → max scale. */ +function Thinking({ metrics }: { readonly metrics: Metrics }) { + const rows = metrics + .slice("thinking") + .filter((row) => row.steps > 0) + .map((row) => ({ ...row, color: RAMP[levelIndex(row.key)] ?? OTHER })); + const { steps, reasoning, output } = metrics.totals; + return ( + + Reasoning {formatTokens(reasoning)} + {output > 0 && ` · ${formatPercent(reasoning / output)} of Output`} +
    + } + > + ({ label: row.label, value: row.steps, color: row.color }))} + format={formatCount} + label="Requests by thinking level" + /> +
      + {rows.map((row) => ( +
    • + + + + {formatCount(row.steps)} + + {steps === 0 ? "–" : formatPercent(row.steps / steps)} · {formatTokens(row.reasoning)} + +
    • + ))} +
    + + ); +} + +/** Token types as ink tints, so they never read as one of the page's models. */ +const TYPES = [ + { key: "input", label: "Input", color: ink(60), tokens: "input", cost: "costInput" }, + { key: "cacheRead", label: "Cache Read", color: ink(30), tokens: "cacheRead", cost: "costCacheRead" }, + { key: "cacheWrite", label: "Cache Write", color: ink(72), tokens: "cacheWrite", cost: "costCacheWrite" }, + { key: "output", label: "Output", color: ink(100), tokens: "output", cost: "costOutput" }, +] as const; + +const CACHE_ROW = "grid grid-cols-[minmax(0,1fr)_56px_64px] items-center gap-x-3"; + +/** Hit rate per bucket; a bucket with no input repeats the last known rate so the line has no holes. */ +function hitTrend(metrics: Metrics) { + const rates = buckets(metrics).map((days) => + cacheHit({ + input: sumOf(days, (day) => day.input), + cacheRead: sumOf(days, (day) => day.cacheRead), + cacheWrite: sumOf(days, (day) => day.cacheWrite), + }), + ); + let last = rates.find((rate) => rate !== null) ?? 0; + return rates.map((rate) => { + if (rate !== null) last = rate; + return last; + }); +} + +function Cache({ metrics }: { readonly metrics: Metrics }) { + const { totals, previous } = metrics; + const hit = cacheHit(totals); + const before = previous === undefined ? undefined : (cacheHit(previous) ?? undefined); + const period = RANGES.find(({ key }) => key === metrics.range)?.label.toLowerCase() ?? ""; + return ( + +
    +
    + Cache Hit Rate +
    + + {hit === null ? "–" : formatPercent(hit)} + + +
    + +
    +
    Saved by Cache
    +
    {formatCost(totals.cacheSavings)}
    +
    +
    +
    + Tokens by Type + ({ label: type.label, value: totals[type.tokens], color: type.color }))} + format={formatTokens} + label="Tokens by type" + /> + Cost by Type + ({ label: type.label, value: totals[type.cost], color: type.color }))} + format={formatCost} + label="Cost by type" + /> +
    +
    +
    + Type + Tokens + Cost +
    +
      + {TYPES.map((type) => ( +
    • + + + {formatTokens(totals[type.tokens])} + + {formatCost(totals[type.cost])} +
    • + ))} +
    +
    +
    +
    + ); +} diff --git a/apps/web/src/metrics/page.tsx b/apps/web/src/metrics/page.tsx new file mode 100644 index 00000000..17923885 --- /dev/null +++ b/apps/web/src/metrics/page.tsx @@ -0,0 +1,181 @@ +import { MetricsError, type MetricsUsage } from "@codeworksh/contract"; +import { useAtom, useAtomRefresh, useAtomValue } from "@effect/atom-react"; +import { ChartBarIcon, WarningCircleIcon } from "@phosphor-icons/react"; +import { Option } from "effect"; +import { Atom, AsyncResult } from "effect/reactivity"; +import { useMemo, useState, useSyncExternalStore } from "react"; + +import { Skeleton } from "@/components/ui/skeleton"; +import { minuteTick } from "../kernel/clock"; +import { Page } from "../shell/page"; +import { Segmented } from "./charts"; +import { + localDay, + projectOptions, + query, + RANGES, + type RangeKey, + type Scope, + usageAtom, + usageKey, + yearOf, +} from "./data"; +import { MachineSelect, ProjectSelect } from "./filters"; +import { MetricsBody } from "./metrics"; + +const EVERYTHING: Scope = { machineId: null, projectId: null }; + +/** The period and filters, kept across navigation so coming back shows the same view. */ +const viewAtom = Atom.make({ range: "30d", ...EVERYTHING }).pipe(Atom.keepAlive); + +/** + * The year of usage for `scope`, read again on every change to it. A new scope or day keeps the last + * year on screen until its own arrives; each year is indexed against the day it was read for. + */ +function useYear(scope: Scope, today: string) { + const atom = usageAtom(usageKey(scope, today)); + const result = useAtomValue(atom); + const retry = useAtomRefresh(atom); + const current = Option.getOrUndefined(AsyncResult.value(result)); + const [kept, setKept] = useState<{ readonly usage: MetricsUsage; readonly today: string }>(); + if (current !== undefined && current !== kept?.usage) setKept({ usage: current, today }); + const usage = current ?? kept?.usage; + const day = current === undefined ? kept?.today : today; + const year = useMemo( + () => (usage === undefined || day === undefined ? undefined : yearOf(usage, day)), + [usage, day], + ); + const error = Option.getOrUndefined(AsyncResult.error(result)); + const failure = !AsyncResult.isFailure(result) + ? undefined + : error instanceof MetricsError + ? error.message + : "The App Server Is Unreachable"; + return { year, failure, retry }; +} + +/** Usage across machines and projects; the header's filters and period scope everything below. */ +export function MetricsPage() { + const [{ range, machineId: picked, projectId }, setView] = useAtom(viewAtom); + const setRange = (next: RangeKey) => setView((view) => ({ ...view, range: next })); + const setMachineId = (next: string | null) => setView((view) => ({ ...view, machineId: next })); + const setProjectId = (next: string | null) => setView((view) => ({ ...view, projectId: next })); + const today = localDay(new Date(useSyncExternalStore(minuteTick.subscribe, minuteTick.getSnapshot))); + // Filtered reads name only what they cover, so the filters list from the unfiltered year: the same read when nothing is filtered. + const unfiltered = useYear(EVERYTHING, today).year; + // A machine removed since it was picked is no longer listed: every machine shows instead. + const machineId = unfiltered?.machines.some((machine) => machine.id === picked) === false ? null : picked; + const scope = useMemo(() => ({ machineId, projectId }), [machineId, projectId]); + const { year, failure, retry } = useYear(scope, today); + const everything = unfiltered ?? year; + const metrics = useMemo(() => (year === undefined ? undefined : query(year, range)), [year, range]); + const whole = useMemo(() => (year === undefined ? undefined : query(year, "1y")), [year]); + const projects = useMemo( + () => (everything === undefined ? [] : projectOptions(everything, range, machineId)), + [everything, range, machineId], + ); + const machines = everything?.machines ?? []; + + return ( + + {machines.length > 1 && } + + + + } + > +
    + {failure !== undefined && } + {metrics !== undefined && whole !== undefined ? ( + row.project.id === projectId)?.project.name} + onProject={setProjectId} + /> + ) : ( + failure === undefined && + )} +
    +
    + ); +} + +/** What failed and a way to try again: in place of the page with nothing to show, or above what is shown. */ +function Failed({ + detail, + onRetry, + alone, +}: { + readonly detail: string; + readonly onRetry: () => void; + /** Nothing is shown below it. */ + readonly alone: boolean; +}) { + return ( +
    + + + + Could Not Load Metrics + + + {detail} + + + +
    + ); +} + +/** The first cards' shapes at their sizes, until the first year arrives. */ +function Loading() { + const card = "flex flex-col gap-4 rounded-lg border border-edge p-4"; + return ( +
    +
    + {[0, 1, 2].map((tile) => ( +
    + + + +
    + ))} +
    + {[330, 290].map((height) => ( +
    + + +
    + ))} +
    + {[0, 1].map((half) => ( +
    + + + +
    + ))} +
    +
    + ); +} diff --git a/apps/web/src/metrics/session.tsx b/apps/web/src/metrics/session.tsx new file mode 100644 index 00000000..c115c9da --- /dev/null +++ b/apps/web/src/metrics/session.tsx @@ -0,0 +1,56 @@ +import { Monogram } from "@/components/monogram"; +import { Meter } from "./charts"; +import { formatCost, formatCount, formatTokens, type Listed } from "./data"; + +/** + * One session on one line (two when narrow): title, then project and threads, then a meter against + * the list's top line by `by`, which also takes the ink. Deleted sessions are one muted line. + */ +export function SessionLine({ + line, + by, + max, +}: { + readonly line: Listed; + readonly by: "tokens" | "cost"; + /** The list's top value of `by`, for the meter. */ + readonly max: number; +}) { + const tone = (key: "tokens" | "cost") => (key === by ? "text-ink" : "text-ink-secondary"); + const title = + line.kind === "deleted" + ? `${formatCount(line.count)} Deleted ${line.count === 1 ? "Session" : "Sessions"}` + : line.session.title; + return ( +
  • + + {title} + + {line.kind === "session" && ( + + + + {line.session.project.name} + + {line.threads !== undefined && ( + + {formatCount(line.threads)} {line.threads === 1 ? "Thread" : "Threads"} + + )} + + )} + + + + + + {formatTokens(line.tokens)} + + {formatCost(line.cost)} + +
  • + ); +} diff --git a/apps/web/src/routeTree.gen.ts b/apps/web/src/routeTree.gen.ts index bde17270..3635dfd8 100644 --- a/apps/web/src/routeTree.gen.ts +++ b/apps/web/src/routeTree.gen.ts @@ -11,6 +11,7 @@ import { Route as rootRouteImport } from './routes/__root' import { Route as IndexRouteImport } from './routes/index' import { Route as ShellRouteImport } from './routes/_shell' +import { Route as ShellMetricsRouteImport } from './routes/_shell.metrics' import { Route as ShellProjectsRouteImport } from './routes/_shell.projects' import { Route as ShellSettingsRouteImport } from './routes/_shell.settings' import { Route as ShellThreadsRouteImport } from './routes/_shell.threads' @@ -38,6 +39,11 @@ const ShellRoute = ShellRouteImport.update({ id: '/_shell', getParentRoute: () => rootRouteImport, } as any) +const ShellMetricsRoute = ShellMetricsRouteImport.update({ + id: '/metrics', + path: '/metrics', + getParentRoute: () => ShellRoute, +} as any) const ShellProjectsRoute = ShellProjectsRouteImport.update({ id: '/projects', path: '/projects', @@ -128,6 +134,7 @@ const ShellWorkspacesWorkspaceIdRoute = export interface FileRoutesByFullPath { '/': typeof IndexRoute + '/metrics': typeof ShellMetricsRoute '/projects': typeof ShellProjectsRoute '/settings': typeof ShellSettingsRouteWithChildren '/threads': typeof ShellThreadsRoute @@ -148,6 +155,7 @@ export interface FileRoutesByFullPath { } export interface FileRoutesByTo { '/': typeof IndexRoute + '/metrics': typeof ShellMetricsRoute '/projects': typeof ShellProjectsRoute '/threads': typeof ShellThreadsRoute '/settings/archive': typeof ShellSettingsArchiveRoute @@ -169,6 +177,7 @@ export interface FileRoutesById { __root__: typeof rootRouteImport '/': typeof IndexRoute '/_shell': typeof ShellRouteWithChildren + '/_shell/metrics': typeof ShellMetricsRoute '/_shell/projects': typeof ShellProjectsRoute '/_shell/settings': typeof ShellSettingsRouteWithChildren '/_shell/threads': typeof ShellThreadsRoute @@ -191,6 +200,7 @@ export interface FileRouteTypes { fileRoutesByFullPath: FileRoutesByFullPath fullPaths: | '/' + | '/metrics' | '/projects' | '/settings' | '/threads' @@ -211,6 +221,7 @@ export interface FileRouteTypes { fileRoutesByTo: FileRoutesByTo to: | '/' + | '/metrics' | '/projects' | '/threads' | '/settings/archive' @@ -231,6 +242,7 @@ export interface FileRouteTypes { | '__root__' | '/' | '/_shell' + | '/_shell/metrics' | '/_shell/projects' | '/_shell/settings' | '/_shell/threads' @@ -271,6 +283,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof ShellRouteImport parentRoute: typeof rootRouteImport } + '/_shell/metrics': { + id: '/_shell/metrics' + path: '/metrics' + fullPath: '/metrics' + preLoaderRoute: typeof ShellMetricsRouteImport + parentRoute: typeof ShellRoute + } '/_shell/projects': { id: '/_shell/projects' path: '/projects' @@ -430,6 +449,7 @@ const ShellSettingsRouteWithChildren = ShellSettingsRoute._addFileChildren( ) interface ShellRouteChildren { + ShellMetricsRoute: typeof ShellMetricsRoute ShellProjectsRoute: typeof ShellProjectsRoute ShellSettingsRoute: typeof ShellSettingsRouteWithChildren ShellThreadsRoute: typeof ShellThreadsRoute @@ -437,6 +457,7 @@ interface ShellRouteChildren { } const ShellRouteChildren: ShellRouteChildren = { + ShellMetricsRoute: ShellMetricsRoute, ShellProjectsRoute: ShellProjectsRoute, ShellSettingsRoute: ShellSettingsRouteWithChildren, ShellThreadsRoute: ShellThreadsRoute, diff --git a/apps/web/src/routes/_shell.metrics.tsx b/apps/web/src/routes/_shell.metrics.tsx new file mode 100644 index 00000000..59e2e9b4 --- /dev/null +++ b/apps/web/src/routes/_shell.metrics.tsx @@ -0,0 +1,5 @@ +import { createFileRoute } from "@tanstack/react-router"; + +import { MetricsPage } from "../metrics/page"; + +export const Route = createFileRoute("/_shell/metrics")({ component: MetricsPage }); diff --git a/apps/web/src/settings/blocks.tsx b/apps/web/src/settings/blocks.tsx index 31030ac2..e553d233 100644 --- a/apps/web/src/settings/blocks.tsx +++ b/apps/web/src/settings/blocks.tsx @@ -84,7 +84,7 @@ export function SavingInput({ }: { value: string; label: string; - placeholder?: string; + placeholder?: string | undefined; autoFocus?: boolean; className?: string; onCommit: (name: string) => void; diff --git a/apps/web/src/settings/servers/add.tsx b/apps/web/src/settings/servers/add.tsx index ef7ff9a0..b5c756b1 100644 --- a/apps/web/src/settings/servers/add.tsx +++ b/apps/web/src/settings/servers/add.tsx @@ -28,19 +28,20 @@ const STEPS = ["Kind", "Identity", "Config"] as const; */ export function AddServer({ onClose, - defaultName, + placeholder, detected, onAdded, }: { onClose: () => void; - defaultName: string; + /** The name the server is shown by when it is given none. */ + placeholder: string | undefined; /** The `codework` found without a choice, shown as the executable's default. */ detected: string | undefined; onAdded: (id: string) => void; }) { const add = useAtomSet(addAtom, { mode: "promise" }); const [step, setStep] = useState(0); - const [name, setName] = useState(defaultName); + const [name, setName] = useState(""); const [executable, setExecutable] = useState(""); const [home, setHome] = useState(""); // A free port, so the new server runs beside the others rather than on the default one. @@ -62,7 +63,7 @@ export function AddServer({ .finally(() => setAdding(false)); }; - const canContinue = step === 0 || (step === 1 && name.trim() !== "") || (step === 2 && home.trim() !== ""); + const canContinue = step !== 2 || home.trim() !== ""; return ( !next && onClose()}> @@ -110,6 +111,7 @@ export function AddServer({ setName(event.target.value)} diff --git a/apps/web/src/settings/servers/local.tsx b/apps/web/src/settings/servers/local.tsx index b34d27dc..890674df 100644 --- a/apps/web/src/settings/servers/local.tsx +++ b/apps/web/src/settings/servers/local.tsx @@ -46,6 +46,10 @@ export const Badge = ({ tone, children }: { tone: Tone; children: ReactNode }) = export const Mark = () => ; +/** What a server is called without the user's override: as shown while there is none. */ +export const reportedName = (status: LocalServerStatus) => + status.override === null ? status.name : (status.host ?? undefined); + const Version = ({ version }: { version: string }) => ( v{version} ); @@ -87,6 +91,9 @@ export function LocalItem({ status }: { status: LocalServerStatus }) {
    {status.name} + {status.host !== null && status.host !== status.name && ( + {status.host} + )} {cli !== null && "version" in cli && } {problem !== undefined && ( @@ -172,11 +179,10 @@ export function LocalDetail({ status }: { status: LocalServerStatus }) { { - // The server keeps its name when the field is cleared. - if (!next) return; setNameError(null); void configure({ name: next }).then((failure) => setNameError(failure ?? null)); }} diff --git a/apps/web/src/settings/servers/servers.tsx b/apps/web/src/settings/servers/servers.tsx index abc55460..d5d88dbb 100644 --- a/apps/web/src/settings/servers/servers.tsx +++ b/apps/web/src/settings/servers/servers.tsx @@ -1,6 +1,6 @@ import { ArrowClockwiseIcon, ArrowCounterClockwiseIcon, MinusIcon, PlusIcon } from "@phosphor-icons/react"; import { useAtom, useAtomSet, useAtomValue } from "@effect/atom-react"; -import { DEFAULT_SERVER_HEALTH, type LocalServerStatus } from "@codeworksh/contract"; +import { DEFAULT_SERVER_HEALTH, LOCAL_SERVER, type LocalServerStatus } from "@codeworksh/contract"; import { AsyncResult } from "effect/reactivity"; import { useEffect, useState } from "react"; @@ -12,7 +12,7 @@ import { Server } from "../../kernel/rpc"; import { serversAtom } from "../../kernel/servers"; import { Group, Page, quietField, Row, SavingInput, Section } from "../blocks"; import { AddServer } from "./add"; -import { LocalDetail, LocalItem } from "./local"; +import { LocalDetail, LocalItem, reportedName } from "./local"; const checkAtom = Server.mutation("servers.check"); const healthKey = ["health"]; @@ -35,6 +35,8 @@ export function ServersSettings() { const [adding, setAdding] = useState(false); // The default until one is picked, and again when the picked one is gone. const selected = servers?.find((server) => server.id === selectedId) ?? servers?.find((server) => server.isDefault); + // A new local server runs on the same machine, so it reports the built-in one's name. + const builtin = servers?.find((server) => server.id === LOCAL_SERVER); // Shown as the new server's executable default: the binary found without a choice. const detected = servers?.flatMap((server) => server.cli !== null && !server.cli.custom ? [server.cli.path] : [], @@ -85,7 +87,7 @@ export function ServersSettings() { {adding && ( setAdding(false)} - defaultName={`CodeWork ${(servers?.length ?? 0) + 1}`} + placeholder={builtin && reportedName(builtin)} detected={detected} onAdded={setSelectedId} /> diff --git a/apps/web/src/shell/page.tsx b/apps/web/src/shell/page.tsx index b71a8941..a54a1211 100644 --- a/apps/web/src/shell/page.tsx +++ b/apps/web/src/shell/page.tsx @@ -15,11 +15,14 @@ export function Page({ instanceId, title, icon: Icon, + actions, children, }: { readonly instanceId: string; readonly title: string; readonly icon: Icon; + /** Controls at the header's end. */ + readonly actions?: ReactNode; readonly children: ReactNode; }) { const registry = useContext(RegistryContext); @@ -42,9 +45,10 @@ export function Page({
    -
    +
    -

    {title}

    +

    {title}

    + {actions}
    {children}
    diff --git a/apps/web/src/shell/shell.tsx b/apps/web/src/shell/shell.tsx index 45232d28..44ac2a76 100644 --- a/apps/web/src/shell/shell.tsx +++ b/apps/web/src/shell/shell.tsx @@ -11,7 +11,7 @@ import { settingsShortcut, workspaceShortcut } from "./switcher"; /** * Lives in the pathless `_shell` layout so switching workspaces (a URL change) never remounts * the desktop: widget order, pins, and per-workspace layout survive navigation. - * Settings, All Threads and Projects open over the desktop, which stays mounted underneath. + * Settings, All Threads, Projects and Metrics open over the desktop, which stays mounted underneath. */ export function Shell() { const { workspaceId } = useParams({ strict: false }); @@ -24,7 +24,8 @@ export function Shell() { select: (location) => location.pathname.startsWith("/settings") || location.pathname === "/threads" || - location.pathname === "/projects", + location.pathname === "/projects" || + location.pathname === "/metrics", }); // An id that no longer exists (a deleted workspace) shows the first one. const shown = workspaces.find((workspace) => workspace.id === workspaceId) ?? workspaces[0]; diff --git a/apps/widget/src/codework.ts b/apps/widget/src/codework.ts index 224cb9e8..e4b4585c 100644 --- a/apps/widget/src/codework.ts +++ b/apps/widget/src/codework.ts @@ -230,6 +230,78 @@ export const IndexItem = Schema.Union([ ]); export type IndexItem = typeof IndexItem.Type; +/** + * Upper bounds in ms of the request duration histogram: bucket `i` of `durations` counts durations + * below `edges[i]` and at or above `edges[i - 1]`; one more bucket past the last edge is open-ended. + */ +export const edges = [ + 250, 374, 561, 839, 1257, 1882, 2818, 4220, 6318, 9461, 14167, 21213, 31764, 47563, 71221, 106645, 159688, 239115, + 358046, 536133, 802797, 1202096, 1800000, +] as const; + +/** Where a `usage.sync` row belongs: a thread's UTC hour. */ +const UsageOwner = { + /** Start of the UTC hour, epoch milliseconds. */ + hour: Schema.Int, + threadId: Schema.String, + /** The thread's session and project when the row was first written. */ + sessionId: Schema.String, + projectId: Schema.String, +}; + +/** A `usage.sync` item: a rollup row with the revision that last wrote it, or progress. */ +export const UsageItem = Schema.Union([ + /** Requests of one thread in one hour to one provider, model and thinking level, aborted ones included. */ + Schema.TaggedStruct("Usage", { + ...UsageOwner, + provider: Schema.String, + model: Schema.String, + thinking: Schema.String, + requests: Schema.Int, + /** `reasoning` is part of `output`. */ + tokens: Schema.Struct({ + input: Schema.Int, + output: Schema.Int, + cacheRead: Schema.Int, + cacheWrite: Schema.Int, + reasoning: Schema.Int, + }), + /** US dollars, from the catalog's prices. */ + cost: Schema.Struct({ + total: Schema.Finite, + input: Schema.Finite, + output: Schema.Finite, + cacheRead: Schema.Finite, + cacheWrite: Schema.Finite, + }), + /** Total ms of the `timed` requests: the committed ones; aborted requests are never timed. */ + duration: Schema.Finite, + timed: Schema.Int, + /** The `timed` requests per bucket of `edges`, one more than there are edges. */ + durations: Schema.Array(Schema.Int), + rev: Schema.Int, + }), + /** Committed prompts (`name` is empty), or tool calls that ran by tool name, of one thread in one hour. */ + Schema.TaggedStruct("Tally", { + ...UsageOwner, + kind: Schema.Literals(["prompt", "tool"]), + name: Schema.String, + count: Schema.Int, + rev: Schema.Int, + }), + /** + * Every row with a revision up to `rev` has been sent. `boot` is the store's latest open (absent from + * an older server); `reset` marks a re-read from zero after the store left the client's `boot`. + */ + Schema.TaggedStruct("Synced", { + storeId: Schema.String, + rev: Schema.Int, + boot: Schema.optionalKey(Schema.String), + reset: Schema.optionalKey(Schema.Literal(true)), + }), +]); +export type UsageItem = typeof UsageItem.Type; + /** What a client calls a thread its first prompt has not named yet. */ export const UNTITLED = "Session"; @@ -495,6 +567,8 @@ export const Api = RpcGroup.make( storeId: Schema.String, /** The store's head: the last revision taken. */ rev: Schema.Int, + /** The machine it runs on, by the name its user knows it; absent when the OS names it nothing. */ + host: Schema.optionalKey(Schema.Struct({ name: Schema.String })), }), }), /** @@ -506,6 +580,19 @@ export const Api = RpcGroup.make( success: IndexItem, stream: true, }), + /** + * The usage rollup of this machine: every `usage` and `tally` row whose revision is past `after`, + * in revision order, then `Synced` at the head read when the call began; then the same live after + * each commit. `after: 0` reads every row. A row written again moves to a later revision and is + * sent again in full; the latest wins. Rows carry running totals, not deltas: a client replaces a + * row by its key, never adds to it. A deleted session's rows stay. With the last `Synced`'s `boot`, + * a store that left that line (a restored backup) reads from zero instead. + */ + Rpc.make("usage.sync", { + payload: { after: Schema.Int, boot: Schema.optionalKey(Schema.String) }, + success: UsageItem, + stream: true, + }), /** * One thread's detail. Without `after`, the newest entries of its line within `budget` bytes and * its pending inbox, then `Synced` with `floor`; with `after`, its own rows past it. Then the same diff --git a/knip.jsonc b/knip.jsonc index bfd4cb69..764eaf82 100644 --- a/knip.jsonc +++ b/knip.jsonc @@ -32,8 +32,8 @@ }, }, "apps/server": { - // The machine simulation runs it on each OrbStack machine by path. - "entry": ["test/machines/drive.ts"], + // The machine suites run these on each OrbStack machine by path. + "entry": ["test/machines/drive.ts", "test/machines/seed.ts"], }, "packages/codework": { // bundle.mjs builds the SEA binary from it by path. diff --git a/packages/codework/src/server/contract.ts b/packages/codework/src/server/contract.ts index 6d71d2f4..17736e70 100644 --- a/packages/codework/src/server/contract.ts +++ b/packages/codework/src/server/contract.ts @@ -18,6 +18,7 @@ import { SettingsFile, ThreadSchema, ThreadStore, + UsageSchema, } from "@codeworksh/harness/effect"; import { Schema } from "effect"; import { Rpc, RpcGroup } from "effect/rpc"; @@ -177,6 +178,76 @@ export const IndexItem = Schema.Union([ ]); export type IndexItem = typeof IndexItem.Type; +/** + * Upper bounds in ms of the request duration histogram: bucket `i` of `durations` counts durations + * below `edges[i]` and at or above `edges[i - 1]`; one more bucket past the last edge is open-ended. + */ +export const edges = UsageSchema.edges; + +/** Where a `usage.sync` row belongs: a thread's UTC hour. */ +const UsageOwner = { + /** Start of the UTC hour, epoch milliseconds. */ + hour: Schema.Int, + threadId: ThreadSchema.ID, + /** The thread's session and project when the row was first written. */ + sessionId: Session.SessionSchema.ID, + projectId: Schema.String, +}; + +/** A `usage.sync` item: a rollup row with the revision that last wrote it, or progress. */ +export const UsageItem = Schema.Union([ + /** Requests of one thread in one hour to one provider, model and thinking level, aborted ones included. */ + Schema.TaggedStruct("Usage", { + ...UsageOwner, + provider: Schema.String, + model: Schema.String, + thinking: Schema.String, + requests: Schema.Int, + /** `reasoning` is part of `output`. */ + tokens: Schema.Struct({ + input: Schema.Int, + output: Schema.Int, + cacheRead: Schema.Int, + cacheWrite: Schema.Int, + reasoning: Schema.Int, + }), + /** US dollars, from the catalog's prices. */ + cost: Schema.Struct({ + total: Schema.Finite, + input: Schema.Finite, + output: Schema.Finite, + cacheRead: Schema.Finite, + cacheWrite: Schema.Finite, + }), + /** Total ms of the `timed` requests: the committed ones; aborted requests are never timed. */ + duration: Schema.Finite, + timed: Schema.Int, + /** The `timed` requests per bucket of `edges`, one more than there are edges. */ + durations: Schema.Array(Schema.Int), + rev: Schema.Int, + }), + /** Committed prompts (`name` is empty), or tool calls that ran by tool name, of one thread in one hour. */ + Schema.TaggedStruct("Tally", { + ...UsageOwner, + kind: Schema.Literals(UsageSchema.tallyKinds), + name: Schema.String, + count: Schema.Int, + rev: Schema.Int, + }), + /** + * Every row with a revision up to `rev` has been sent. `boot` is the store's latest open, for the + * next call; `reset` marks the first one after a `boot` off the store's line: every row was sent + * again from zero, and a client drops what it held of this store. + */ + Schema.TaggedStruct("Synced", { + storeId: Schema.String, + rev: Schema.Int, + boot: Schema.String, + reset: optional(Schema.Literal(true)), + }), +]); +export type UsageItem = typeof UsageItem.Type; + /** A thread of a session: `threadId`, or the session's `main` thread when absent. */ const ThreadRef = { sessionId: Session.SessionSchema.ID, @@ -444,6 +515,8 @@ export const Api = RpcGroup.make( storeId: Schema.String, /** The store's head: the last revision taken. */ rev: Schema.Int, + /** The machine it runs on, by the name its user knows it; absent when the OS names it nothing. */ + host: optional(Schema.Struct({ name: Schema.String })), }), }), /** @@ -457,6 +530,19 @@ export const Api = RpcGroup.make( success: IndexItem, stream: true, }), + /** + * The usage rollup of this machine: every `usage` and `tally` row whose revision is past `after`, + * in revision order, then `Synced` at the head read when the call began; then the same live after + * each commit. `after: 0` reads every row. A row written again moves to a later revision and is + * sent again in full; the latest wins. Rows carry running totals, not deltas: a client replaces a + * row by its key, never adds to it. A deleted session's rows stay. `boot` is the one the last + * `Synced` carried: when the store left that line (a restored backup), it reads from zero instead. + */ + Rpc.make("usage.sync", { + payload: { after: Schema.Int, boot: optional(Schema.String) }, + success: UsageItem, + stream: true, + }), Rpc.make("auth.save", { payload: { credentials: OAuthCredentials }, success: OAuthInfo, diff --git a/packages/codework/src/server/handlers.ts b/packages/codework/src/server/handlers.ts index 2db276f3..c17a5337 100644 --- a/packages/codework/src/server/handlers.ts +++ b/packages/codework/src/server/handlers.ts @@ -36,6 +36,7 @@ import { type ThreadInfo, type ThreadItem, type ThreadSummary, + type UsageItem, } from "./contract.ts"; import * as Envelope from "./envelope.ts"; import { type EncodingError, type EventEnvelope } from "./envelope.ts"; @@ -112,6 +113,43 @@ const toIndexItem = (change: Sync.IndexChange): IndexItem => { } }; +const toUsageItem = (change: Sync.UsageChange): UsageItem => { + if (change._tag === "Tally") { + const { hour, threadId, sessionId, projectId, kind, name, count, rev } = change.tally; + return { _tag: "Tally", hour, threadId, sessionId, projectId, kind, name, count, rev }; + } + const row = change.usage; + return { + _tag: "Usage", + hour: row.hour, + threadId: row.threadId, + sessionId: row.sessionId, + projectId: row.projectId, + provider: row.provider, + model: row.model, + thinking: row.thinking, + requests: row.requests, + tokens: { + input: row.input, + output: row.output, + cacheRead: row.cacheRead, + cacheWrite: row.cacheWrite, + reasoning: row.reasoning, + }, + cost: { + total: row.cost, + input: row.costInput, + output: row.costOutput, + cacheRead: row.costCacheRead, + cacheWrite: row.costCacheWrite, + }, + duration: row.duration, + timed: row.timed, + durations: row.durations, + rev: row.rev, + }; +}; + // Wire configs name only the keys they set, so `undefined` never reaches the harness under // exactOptionalPropertyTypes. Creating takes choices only; `null` clears nothing there. const runtimeInput = (runtime: RuntimeConfig | undefined): Session.RuntimeInput => ({ @@ -252,7 +290,7 @@ export const layer = Contract.Api.toLayer( return Contract.Api.of({ "server.info": Effect.fnUntraced(function* () { const head = yield* store.head; - return { pid: identity.pid, version: identity.version, storeId: head.id, rev: head.rev }; + return { ...identity, storeId: head.id, rev: head.rev }; }), "index.sync": ({ after }) => Rows.follow( @@ -266,6 +304,33 @@ export const layer = Contract.Api.toLayer( ), (head): IndexItem => ({ _tag: "Synced", storeId: head.id, rev: head.rev }), ), + "usage.sync": ({ after, boot }) => + Stream.unwrap( + Effect.gen(function* () { + let reset = after > 0 && boot !== undefined && !(yield* store.onLine(boot, after)); + return Rows.follow( + reset ? 0 : after, + (from, upto) => + Sync.usage(from, upto, Rows.batch).pipe( + Effect.map(({ items, next }) => ({ + items: items.map(toUsageItem), + ...(next === undefined ? {} : { next }), + })), + ), + (head): UsageItem => { + const synced = { + _tag: "Synced" as const, + storeId: head.id, + rev: head.rev, + boot: head.boot, + ...(reset ? { reset: true as const } : {}), + }; + reset = false; + return synced; + }, + ); + }), + ), "auth.save": ({ credentials }) => oauth.save(credentials), "auth.status": ({ provider }) => oauth.status(provider), "auth.refresh": ({ provider }) => oauth.refresh(provider), diff --git a/packages/codework/src/server/identity.ts b/packages/codework/src/server/identity.ts index 833abcec..b547a5c5 100644 --- a/packages/codework/src/server/identity.ts +++ b/packages/codework/src/server/identity.ts @@ -1,4 +1,8 @@ -import { Context, Layer } from "effect"; +/* @effect-diagnostics nodeBuiltinImport:off -- the host's name is the OS's hostname. */ +import os from "node:os"; +import { NodeServices } from "@effect/platform-node"; +import { Context, Effect, Layer } from "effect"; +import { ChildProcess, ChildProcessSpawner } from "effect/process"; import pkg from "../../package.json" with { type: "json" }; /** @@ -11,6 +15,8 @@ import pkg from "../../package.json" with { type: "json" }; export interface Interface { readonly pid: number; readonly version: string; + /** The machine it runs on, by the name its user knows it; none when the OS names it nothing. */ + readonly host?: { readonly name: string }; } export class Service extends Context.Service()("@codeworksh/cli/server/identity/Service") {} @@ -18,4 +24,24 @@ export class Service extends Context.Service()("@codeworksh/ /** This process: a server answers with it, a client compares a server's version against it. */ export const current: Interface = { pid: process.pid, version: pkg.version }; -export const layer = Layer.succeed(Service, Service.of(current)); +// macOS's computer name is the one Finder and sharing show; its hostname is a mangled copy of it. +const hostName = Effect.gen(function* () { + if (process.platform !== "darwin") return os.hostname().trim(); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const computer = yield* spawner + .string(ChildProcess.make("scutil", ["--get", "ComputerName"])) + .pipe(Effect.orElseSucceed(() => "")); + return ( + computer.trim() || + os + .hostname() + .replace(/\.local$/, "") + .trim() + ); +}); + +/** The host's name is read once, when the server starts. */ +export const layer = Layer.effect( + Service, + hostName.pipe(Effect.map((name) => Service.of(name === "" ? current : { ...current, host: { name } }))), +).pipe(Layer.provide(NodeServices.layer)); diff --git a/packages/codework/src/server/server.ts b/packages/codework/src/server/server.ts index ed07a9e4..4f69c207 100644 --- a/packages/codework/src/server/server.ts +++ b/packages/codework/src/server/server.ts @@ -89,7 +89,13 @@ const announce = Layer.unwrap( }), ); -export const layer = (options: { host: string; port: number; harness: Harness.Options }) => { +export const layer = (options: { + host: string; + port: number; + harness: Harness.Options; + /** Who the server says it is; tests name its host. */ + identity?: Layer.Layer; +}) => { const rpc = RpcServer.layerHttp({ group: Contract.Api, path: Contract.path, protocol: "websocket" }).pipe( Layer.provide(Handlers.layer.pipe(Layer.provide(EventFeed.layer))), ); @@ -104,7 +110,7 @@ export const layer = (options: { host: string; port: number; harness: Harness.Op Layer.provideMerge(announce), Layer.provideMerge(NodeHttpServer.layer(createServer, { host: options.host, port: options.port })), Layer.provide(RpcSerialization.layerNdjson), - Layer.provide(Identity.layer), + Layer.provide(options.identity ?? Identity.layer), // Built after the harness and before everything above it, the listener included. Layer.provide(sweep), Layer.provide(Harness.layer(options.harness)), diff --git a/packages/codework/test/__artifacts__/sync.usage.json b/packages/codework/test/__artifacts__/sync.usage.json new file mode 100644 index 00000000..8e021ee3 --- /dev/null +++ b/packages/codework/test/__artifacts__/sync.usage.json @@ -0,0 +1,245 @@ +{ + "edges": 23, + "first": [ + { + "_tag": "Tally", + "hour": "", + "threadId": "
    ", + "sessionId": "", + "projectId": "", + "kind": "prompt", + "name": "", + "count": 1, + "rev": "" + }, + { + "_tag": "Tally", + "hour": "", + "threadId": "
    ", + "sessionId": "", + "projectId": "", + "kind": "tool", + "name": "probe", + "count": 1, + "rev": "" + }, + { + "_tag": "Usage", + "hour": "", + "threadId": "
    ", + "sessionId": "", + "projectId": "", + "provider": "openai", + "model": "gpt-5.6-luna", + "thinking": "medium", + "requests": 2, + "tokens": { + "input": 300, + "output": 30, + "cacheRead": 3, + "cacheWrite": 0, + "reasoning": 0 + }, + "cost": { + "total": 0.75, + "input": 0.375, + "output": 0.375, + "cacheRead": 0, + "cacheWrite": 0 + }, + "duration": 900, + "timed": 2, + "durations": [ + 0, + 1, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "rev": "" + }, + { + "_tag": "Synced", + "storeId": "", + "rev": "", + "boot": "" + } + ], + "second": [ + { + "_tag": "Tally", + "hour": "", + "threadId": "
    ", + "sessionId": "", + "projectId": "", + "kind": "prompt", + "name": "", + "count": 2, + "rev": "" + }, + { + "_tag": "Usage", + "hour": "", + "threadId": "
    ", + "sessionId": "", + "projectId": "", + "provider": "openai", + "model": "gpt-5.6-luna", + "thinking": "medium", + "requests": 3, + "tokens": { + "input": 600, + "output": 60, + "cacheRead": 6, + "cacheWrite": 0, + "reasoning": 0 + }, + "cost": { + "total": 1.5, + "input": 0.75, + "output": 0.75, + "cacheRead": 0, + "cacheWrite": 0 + }, + "duration": 1800, + "timed": 3, + "durations": [ + 0, + 1, + 0, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "rev": "" + }, + { + "_tag": "Synced", + "storeId": "", + "rev": "", + "boot": "" + } + ], + "resumed": [ + { + "_tag": "Tally", + "hour": "", + "threadId": "
    ", + "sessionId": "", + "projectId": "", + "kind": "prompt", + "name": "", + "count": 2, + "rev": "" + }, + { + "_tag": "Usage", + "hour": "", + "threadId": "
    ", + "sessionId": "", + "projectId": "", + "provider": "openai", + "model": "gpt-5.6-luna", + "thinking": "medium", + "requests": 3, + "tokens": { + "input": 600, + "output": 60, + "cacheRead": 6, + "cacheWrite": 0, + "reasoning": 0 + }, + "cost": { + "total": 1.5, + "input": 0.75, + "output": 0.75, + "cacheRead": 0, + "cacheWrite": 0 + }, + "duration": 1800, + "timed": 3, + "durations": [ + 0, + 1, + 0, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "rev": "" + }, + { + "_tag": "Synced", + "storeId": "", + "rev": "", + "boot": "" + } + ], + "idle": [ + { + "_tag": "Synced", + "storeId": "", + "rev": "", + "boot": "" + } + ], + "paged": { + "tallies": 600, + "rows": 603 + } +} diff --git a/packages/codework/test/server.test.ts b/packages/codework/test/server.test.ts index 0369eeb6..536963cc 100644 --- a/packages/codework/test/server.test.ts +++ b/packages/codework/test/server.test.ts @@ -9,7 +9,7 @@ import { HttpServer } from "effect/http"; import { RpcTest } from "effect/rpc"; import { execFile } from "node:child_process"; import { mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs"; -import { tmpdir } from "node:os"; +import { hostname, tmpdir } from "node:os"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; @@ -76,6 +76,17 @@ afterAll(() => { }); describe("server", () => { + it("server.info names the host as its user sees it", async () => { + const name = + process.platform === "darwin" + ? (await exec("scutil", ["--get", "ComputerName"])).stdout.trim() + : hostname().trim(); + await Effect.gen(function* () { + const rpc = yield* RpcTest.makeClient(Contract.Api); + expect((yield* rpc["server.info"]({})).host).toEqual({ name }); + }).pipe(Effect.scoped, Effect.provide(layer()), Effect.runPromise); + }); + it("sandbox.drivers returns the core drivers", () => Effect.gen(function* () { const rpc = yield* RpcTest.makeClient(Contract.Api); diff --git a/packages/codework/test/usage.test.ts b/packages/codework/test/usage.test.ts new file mode 100644 index 00000000..c85be3bf --- /dev/null +++ b/packages/codework/test/usage.test.ts @@ -0,0 +1,227 @@ +/* @effect-diagnostics nodeBuiltinImport:off -- starts the real server in a temporary home. */ +import { createAssistantMessageEventStream, type Message } from "@codeworksh/aikit"; +import { Effect, Layer, Queue, Stream } from "effect"; +import { HttpServer } from "effect/http"; +import { mkdtempSync, realpathSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, it } from "vite-plus/test"; +import type * as LLM from "../../harness/src/runner/llm.ts"; +import { assistant } from "../../harness/test/fixtures/llm.ts"; +import * as Client from "../src/server/client.ts"; +import { edges, type UsageItem } from "../src/server/contract.ts"; +import * as Server from "../src/server/server.ts"; +import { testModel } from "./utils/model.ts"; + +type Synced = Extract; +type Row = Exclude; + +// 2020-01-01T00:00:00Z: every request is made in this hour; prompts are tallied in the real one. +const H = Date.UTC(2020, 0, 1); +// More tool names in one reply than one page holds (`Rows.batch`), so its revision spans two pages. +const wide = 600; + +/** Request `n` of the server: 1 calls a tool, 4 calls `wide` tools at once, every other one stops. */ +const scripted: LLM.Open = (() => { + let count = 0; + return (input) => + Effect.sync(() => { + count += 1; + const index = count; + const calls = index === 1 ? ["probe"] : index === 4 ? Array.from({ length: wide }, (_, i) => `tool_${i}`) : []; + const message = assistant(input, index, { + usage: { + input: 100 * index, + output: 10 * index, + cacheRead: index, + cacheWrite: 0, + totalTokens: 111 * index, + cost: { input: 0.125 * index, output: 0.125 * index, cacheRead: 0, cacheWrite: 0, total: 0.25 * index }, + }, + time: { created: H + index * 1000, completed: H + index * 1000 + 300 * index }, + ...(calls.length === 0 + ? {} + : { + stopReason: "toolUse", + parts: calls.map((name, i): Message.ToolCallPendingPart => ({ + type: "toolCall", + callID: `call_${index}_${i}`, + name, + arguments: {}, + status: "pending", + time: { start: H, end: H }, + })), + }), + }); + const events = createAssistantMessageEventStream(); + events.push({ type: "start", partial: message }); + events.push({ type: "done", reason: calls.length === 0 ? "stop" : "toolUse", message }); + return events; + }); +})(); + +const keyOf = (row: Row) => + row._tag === "Usage" + ? `usage|${row.hour}|${row.threadId}|${row.provider}|${row.model}|${row.thinking}` + : `tally|${row.hour}|${row.threadId}|${row.kind}|${row.name}`; +const rowsOf = (items: ReadonlyArray) => + items.flatMap((item): Array => (item._tag === "Synced" ? [] : [item])); +const isUsage = (item: UsageItem): item is Extract => item._tag === "Usage"; + +it("usage.sync sends the rollup in revision order, resends rows with running totals, resumes and pages", async () => { + const home = realpathSync(mkdtempSync(join(tmpdir(), "codework-sync-usage-"))); + writeFileSync(join(home, "settings.jsonc"), JSON.stringify({ model: testModel })); + try { + const artifact = await Effect.gen(function* () { + const listener = yield* HttpServer.HttpServer; + if (listener.address._tag === "UnixPathAddress") return yield* Effect.die("Expected TCP listener"); + const context = yield* Layer.build(Client.layer(`ws://127.0.0.1:${listener.address.port}/rpc/v1`)); + const rpc = yield* Client.make.pipe(Effect.provideContext(context)); + + // Up to and including the first `Synced`: one catch-up. + const catchUp = (after: number) => + rpc["usage.sync"]({ after }).pipe( + Stream.takeUntil((item) => item._tag === "Synced"), + Stream.runCollect, + ); + + const live = yield* Queue.unbounded(); + yield* rpc["usage.sync"]({ after: 0 }).pipe( + Stream.runForEach((item) => Queue.offer(live, item)), + Effect.forkScoped, + ); + /** Live items up to and including `Synced` at the current head. */ + const settle = Effect.gen(function* () { + const head = (yield* rpc["server.info"]({})).rev; + const items: Array = []; + while (true) { + const item = yield* Queue.take(live); + items.push(item); + if (item._tag === "Synced" && item.rev >= head) return items; + } + }); + + // (1) A fresh server: `Synced` at revision 0 and nothing before it. + const empty = yield* Queue.take(live); + + const created = yield* rpc["session.create"]({ directory: home, project: { name: "Alpha" } }); + const sessionId = created.session.id; + const projectId = created.session.project?.id; + if (projectId === undefined) return yield* Effect.die("session has no project"); + const turn = (text: string) => + rpc["thread.prompt"]({ sessionId, text }).pipe(Effect.andThen(rpc["thread.wait"]({ sessionId }))); + + // Requests 1 and 2: a tool call, then the answer. + yield* turn("First prompt"); + const first = yield* settle; + const firstSynced = first.at(-1) as Synced; + const info = yield* rpc["thread.info"]({ sessionId }); + + // (2) Request 3, in the same hour: the same `Usage` key again, with larger running totals. + yield* turn("Second prompt"); + const second = yield* settle; + + // (3) Resume from either `Synced`. + const resumed = yield* catchUp(firstSynced.rev); + const latest = second.at(-1) as Synced; + const idle = yield* catchUp(latest.rev); + + // (4) Request 4 calls `wide` tools: their tallies share one revision across two pages. + yield* turn("Wide prompt"); + const paged = yield* settle; + const whole = yield* catchUp(0); + + // (5) Deleting the session retracts nothing. + const removed = yield* rpc["session.delete"]({ sessionId }); + const afterDelete = yield* settle; + const reread = yield* catchUp(0); + + const usageOf = (items: ReadonlyArray) => items.filter(isUsage); + const [usage1] = usageOf(first); + const [usage2] = usageOf(second); + if (usage1 === undefined || usage2 === undefined) return yield* Effect.die("no usage row was sent"); + const wideTallies = rowsOf(paged).filter((row) => row._tag === "Tally" && row.name.startsWith("tool_")); + const wholeRows = rowsOf(whole); + const revs = (items: ReadonlyArray) => items.map((item) => item.rev); + + expect(empty).toEqual({ _tag: "Synced", storeId: firstSynced.storeId, rev: 0, boot: firstSynced.boot }); + expect(first.filter((item) => item._tag === "Usage")).toHaveLength(1); + expect(usage1.requests).toBe(2); + expect({ + cost: usage1.cost.total, + input: usage1.tokens.input, + output: usage1.tokens.output, + cacheRead: usage1.tokens.cacheRead, + cacheWrite: usage1.tokens.cacheWrite, + }).toEqual(info.usage); + expect(rowsOf(first).some((row) => row._tag === "Tally" && row.kind === "tool" && row.name === "probe")).toBe( + true, + ); + expect(keyOf(usage2)).toBe(keyOf(usage1)); + expect(usage2.rev).toBeGreaterThan(usage1.rev); + expect(usage2.requests).toBe(3); + expect(usage2.tokens.input).toBe(usage1.tokens.input + 300); + expect(usage2.cost.total).toBe(usage1.cost.total + 0.75); + expect(rowsOf(resumed).map(keyOf).sort()).toEqual(rowsOf(second).map(keyOf).sort()); + expect(rowsOf(resumed).every((row) => row.rev > firstSynced.rev)).toBe(true); + expect(idle).toEqual([latest]); + expect(wideTallies).toHaveLength(wide); + expect(new Set(wideTallies.map((row) => row.rev)).size).toBe(1); + expect(new Set(wideTallies.map(keyOf)).size).toBe(wide); + expect(revs(paged)).toEqual(revs(paged).toSorted((a, b) => a - b)); + expect(wholeRows.length).toBeGreaterThan(500); + expect(new Set(wholeRows.map(keyOf)).size).toBe(wholeRows.length); + expect(revs(whole)).toEqual(revs(whole).toSorted((a, b) => a - b)); + expect(removed.deleted).toBe(true); + expect(rowsOf(afterDelete)).toEqual([]); + expect(rowsOf(reread)).toEqual(wholeRows); + + const labels = new Map([ + [firstSynced.storeId, ""], + [firstSynced.boot, ""], + [sessionId, ""], + [projectId, ""], + [created.id, "
    "], + ]); + // Revisions by rank within the artifact: their order is the contract, their values are not. + // How commits batch into live rounds varies, so a live segment shows its rows and its last `Synced`. + const settled = (items: ReadonlyArray) => [...rowsOf(items), items.at(-1)!]; + const shown = [...settled(first), ...settled(second), ...resumed, ...idle]; + const ranks = [...new Set(revs(shown))].sort((a, b) => a - b); + const normalize = (value: unknown, key?: string): unknown => { + if (typeof value === "string") return labels.get(value) ?? value; + if (typeof value === "number") { + if (key === "rev") return ranks.includes(value) ? `` : ""; + if (key === "hour") return value === H ? "" : ""; + return value; + } + if (Array.isArray(value)) return value.map((item) => normalize(item)); + if (typeof value === "object" && value !== null) + return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, normalize(v, k)])); + return value; + }; + return normalize({ + edges: edges.length, + first: settled(first), + second: settled(second), + resumed, + idle, + paged: { tallies: wideTallies.length, rows: wholeRows.length }, + }); + }).pipe( + Effect.scoped, + Effect.provide( + Server.layer({ + host: "127.0.0.1", + port: 0, + harness: { home, hostCwd: home, plugins: [], llm: scripted }, + }), + ), + Effect.timeout("60 seconds"), + Effect.runPromise, + ); + await expect(`${JSON.stringify(artifact, null, "\t")}\n`).toMatchFileSnapshot("./__artifacts__/sync.usage.json"); + } finally { + rmSync(home, { recursive: true, force: true }); + } +}, 80_000); diff --git a/packages/harness/src/db/db.ts b/packages/harness/src/db/db.ts index ff663934..625f6cdf 100644 --- a/packages/harness/src/db/db.ts +++ b/packages/harness/src/db/db.ts @@ -1,6 +1,7 @@ import { SqliteClient, SqliteMigrator } from "@effect/sql-sqlite-node"; import { Config, Effect, Layer, String as Str } from "effect"; import { Migrator, SqlClient } from "effect/sql"; +import { uuidv7 } from "uuidv7"; import * as Global from "../global.ts"; import { fileSystem } from "../host.ts"; import { posix } from "../util/posix.ts"; @@ -25,6 +26,7 @@ const setup = Effect.gen(function* () { yield* sql`PRAGMA cache_size = -64000`; yield* sql`PRAGMA foreign_keys = ON`; yield* SqliteMigrator.run({ loader: Migrator.fromRecord(migrations) }); + yield* sql`INSERT INTO boot (id, rev) SELECT ${uuidv7()}, rev FROM store`; }); // Provides `SqlClient` (and `SqliteClient`) for the database at `location`, diff --git a/packages/harness/src/db/migrations.ts b/packages/harness/src/db/migrations.ts index c82f383a..77c79b36 100644 --- a/packages/harness/src/db/migrations.ts +++ b/packages/harness/src/db/migrations.ts @@ -1,6 +1,7 @@ import { Effect, Schema } from "effect"; import { SqlClient } from "effect/sql"; import { uuidv7 } from "uuidv7"; +import * as Usage from "../usage/usage.ts"; // Migrations ship as code (Migrator.fromRecord) rather than .sql files on // disk, so they survive bundling and need no runtime path resolution. @@ -315,6 +316,81 @@ export const migrations = { } } }), + + // The usage rollup (usage/usage.ts), backfilled from the entries already finalised. + "202611140001_usage": Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* sql` + CREATE TABLE usage ( + hour INTEGER NOT NULL, + thread_id TEXT NOT NULL, + provider TEXT NOT NULL, + model TEXT NOT NULL, + thinking TEXT NOT NULL, + session_id TEXT NOT NULL, + project_id TEXT NOT NULL, + requests INTEGER NOT NULL, + input INTEGER NOT NULL, + output INTEGER NOT NULL, + cache_read INTEGER NOT NULL, + cache_write INTEGER NOT NULL, + reasoning INTEGER NOT NULL, + cost REAL NOT NULL, + cost_input REAL NOT NULL, + cost_output REAL NOT NULL, + cost_cache_read REAL NOT NULL, + cost_cache_write REAL NOT NULL, + duration REAL NOT NULL, + timed INTEGER NOT NULL, + durations TEXT NOT NULL, + rev INTEGER NOT NULL, + PRIMARY KEY (hour, thread_id, provider, model, thinking) + ) + `; + yield* sql` + CREATE TABLE tally ( + hour INTEGER NOT NULL, + thread_id TEXT NOT NULL, + kind TEXT NOT NULL, + name TEXT NOT NULL, + session_id TEXT NOT NULL, + project_id TEXT NOT NULL, + count INTEGER NOT NULL, + rev INTEGER NOT NULL, + PRIMARY KEY (hour, thread_id, kind, name) + ) + `; + yield* sql`CREATE INDEX usage_rev_idx ON usage (rev)`; + yield* sql`CREATE INDEX tally_rev_idx ON tally (rev)`; + for (const table of ["usage", "tally"]) { + for (const event of ["insert", "update"]) { + yield* sql.unsafe(` + CREATE TRIGGER ${table}_rev_${event} BEFORE ${event.toUpperCase()} ON ${table} + WHEN NEW.rev <> (SELECT rev FROM store) + BEGIN SELECT RAISE(ABORT, 'rev is not the current revision'); END + `); + } + } + yield* Usage.backfill( + sql<{ readonly rev: number }>`UPDATE store SET rev = rev + 1 RETURNING rev`.pipe( + Effect.map(([row]) => row!.rev), + Effect.orDie, + ), + ); + }), + + // Each open of the database (db.ts), with the revision it opened at: a client that remembers the + // latest one beside its cursor can tell a restored backup from a store that only moved on. + "202611140002_boot": Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* sql` + CREATE TABLE boot ( + seq INTEGER PRIMARY KEY, + id TEXT NOT NULL UNIQUE, + rev INTEGER NOT NULL + ) + `; + }), }; /** diff --git a/packages/harness/src/db/schema.sql.ts b/packages/harness/src/db/schema.sql.ts index 008d960a..736eb8ec 100644 --- a/packages/harness/src/db/schema.sql.ts +++ b/packages/harness/src/db/schema.sql.ts @@ -9,6 +9,7 @@ import * as DocumentSchema from "../document/schema.ts"; import * as SessionSchema from "../session/schema.ts"; import * as ThreadSchema from "../thread/schema.ts"; import { SpaceSchema } from "../space/schema.ts"; +import * as UsageSchema from "../usage/schema.ts"; // Column names derive from field names via the client's camelToSnake @@ -222,6 +223,48 @@ export class EntryPartRow extends Model.Class("EntryPartRow")({ ...Timestamps, }) {} +// Usage of one thread's finalised assistant entries per UTC hour, provider, model and thinking level. +// No foreign keys: the rows outlive their session, so its session and project are copied in. +export class UsageRow extends Model.Class("UsageRow")({ + hour: Schema.Int, + threadId: ThreadSchema.IDFromDb, + provider: Schema.String, + model: Schema.String, + thinking: Schema.String, + sessionId: SessionSchema.IDFromDb, + projectId: Schema.String, + requests: NonNegativeInt, + input: NonNegativeInt, + output: NonNegativeInt, + cacheRead: NonNegativeInt, + cacheWrite: NonNegativeInt, + reasoning: NonNegativeInt, + cost: Schema.Finite, + costInput: Schema.Finite, + costOutput: Schema.Finite, + costCacheRead: Schema.Finite, + costCacheWrite: Schema.Finite, + // `timed` counts the committed requests with a usable duration; aborted ones are billed, never timed. + // `duration` sums theirs in ms; `durations` counts them per `UsageSchema.edges` bucket. + duration: Schema.Finite, + timed: NonNegativeInt, + durations: UsageSchema.Durations, + rev: Rev, +}) {} + +// Small counters of one thread per UTC hour: committed prompts, and tool calls that ran (completed or +// error) by tool name. +export class TallyRow extends Model.Class("TallyRow")({ + hour: Schema.Int, + threadId: ThreadSchema.IDFromDb, + kind: Schema.Literals(UsageSchema.tallyKinds), + name: Schema.String, + sessionId: SessionSchema.IDFromDb, + projectId: Schema.String, + count: NonNegativeInt, + rev: Rev, +}) {} + // JSON state owned by a session (no thread) or a thread. Every change is a revision. export class DocumentRow extends Model.Class("DocumentRow")({ id: DocumentSchema.ID, diff --git a/packages/harness/src/db/store.ts b/packages/harness/src/db/store.ts index 97bb5cda..8cbdb9fe 100644 --- a/packages/harness/src/db/store.ts +++ b/packages/harness/src/db/store.ts @@ -1,10 +1,11 @@ import { Clock, Context, Effect, Layer, PubSub, type Scope } from "effect"; import { SqlClient, type SqlError } from "effect/sql"; -/** The store's identity and its last revision taken. */ +/** The store's identity, its last revision taken, and its latest open (`boot`). */ export interface Head { readonly id: string; readonly rev: number; + readonly boot: string; } export interface Interface { @@ -18,6 +19,12 @@ export interface Interface { /** The enclosing write's revision, taken on first use; every row the write touches is stamped with it. */ readonly rev: Effect.Effect; readonly head: Effect.Effect; + /** + * Whether a client that read up to `rev` while `boot` was the latest open is still on this store's + * line: `boot` is known and every open since came at `rev` or later. A restored backup fails it once + * the client had read past the backup. + */ + readonly onLine: (boot: string, rev: number) => Effect.Effect; /** * A doorbell for the caller's scope, rung after every commit that took a revision. Register it * before reading `head`, so no commit falls between the read and the first ring. @@ -83,11 +90,22 @@ export const layer = Layer.effect( }).pipe(Effect.orDie); const head = Effect.gen(function* () { - const [row] = yield* sql`SELECT id, rev FROM store`; + const [row] = yield* sql` + SELECT id, rev, (SELECT id FROM boot ORDER BY seq DESC LIMIT 1) AS boot FROM store + `; if (row === undefined) return yield* Effect.die(new Error("store row is missing")); return row; }).pipe(Effect.orDie); + const onLine = Effect.fn("Store.onLine")(function* (boot: string, rev: number) { + const [own] = yield* sql<{ readonly seq: number }>`SELECT seq FROM boot WHERE id = ${boot}`; + if (own === undefined) return false; + const [later] = yield* sql<{ readonly since: number | null }>` + SELECT MIN(rev) AS since FROM boot WHERE seq > ${own.seq} + `; + return later === undefined || later.since === null || later.since >= rev; + }, Effect.orDie); + const wake = Effect.gen(function* () { const pubsub = yield* PubSub.sliding(1); const subscription = yield* PubSub.subscribe(pubsub); @@ -98,6 +116,6 @@ export const layer = Layer.effect( return subscription; }); - return Service.of({ write, rev, head, wake }); + return Service.of({ write, rev, head, onLine, wake }); }), ); diff --git a/packages/harness/src/effect.ts b/packages/harness/src/effect.ts index 94d6d3e6..4887a5cb 100644 --- a/packages/harness/src/effect.ts +++ b/packages/harness/src/effect.ts @@ -15,6 +15,8 @@ export * as PromptSchema from "./session/prompt/schema.ts"; export * as SessionStore from "./session/session.ts"; export * as ThreadStore from "./thread/thread.ts"; export * as ThreadSchema from "./thread/schema.ts"; +export * as Usage from "./usage/usage.ts"; +export * as UsageSchema from "./usage/schema.ts"; export * as DocumentSchema from "./document/schema.ts"; export * as Runner from "./runner/run.ts"; export * as Git from "./git/git.ts"; diff --git a/packages/harness/src/effect/sync.ts b/packages/harness/src/effect/sync.ts index a7457d1b..57d77824 100644 --- a/packages/harness/src/effect/sync.ts +++ b/packages/harness/src/effect/sync.ts @@ -1,6 +1,6 @@ import { DateTime, Effect, Option, Schema } from "effect"; import { SqlClient, SqlSchema, type Statement } from "effect/sql"; -import { InboxRow, ProjectRow, SpaceRow } from "../db/schema.sql.ts"; +import { InboxRow, ProjectRow, SpaceRow, TallyRow, UsageRow } from "../db/schema.sql.ts"; import * as Inbox from "../inbox/inbox.ts"; import type { Admitted } from "../inbox/schema.ts"; import type { ProjectSchema } from "../project/schema.ts"; @@ -171,6 +171,47 @@ export const index = Effect.fn("Sync.index")(function* (from: Cursor, upto: numb .pipe(Effect.orDie); }); +/** A row of the usage rollup at the revision that last wrote it: running totals, not a delta. */ +export type UsageChange = + | { readonly _tag: "Usage"; readonly usage: UsageRow } + | { readonly _tag: "Tally"; readonly tally: TallyRow }; + +/** One page of the `usage` and `tally` rows past `from`, up to revision `upto`, read as one snapshot. */ +export const usage = Effect.fn("Sync.usage")(function* (from: Cursor, upto: number, limit: number) { + const sql = yield* SqlClient.SqlClient; + const findUsage = SqlSchema.findOne({ + Request: Schema.String, + Result: UsageRow, + execute: (id) => sql`SELECT * FROM usage WHERE rowid = ${Number(id)}`, + }); + const findTally = SqlSchema.findOne({ + Request: Schema.String, + Result: TallyRow, + execute: (id) => sql`SELECT * FROM tally WHERE rowid = ${Number(id)}`, + }); + // The rows are keyed by several columns, so their rowid names them in the cursor: an upsert keeps + // it, nothing deletes them, and nothing vacuums the database. + const select = (range: Statement.Fragment) => sql` + SELECT 'tally' AS kind, CAST(rowid AS TEXT) AS id, rev FROM tally WHERE ${range} + UNION ALL SELECT 'usage', CAST(rowid AS TEXT), rev FROM usage WHERE ${range} + `; + const change = Effect.fnUntraced(function* ({ kind, id }: Row) { + if (kind === "usage") return { _tag: "Usage", usage: yield* findUsage(id) } satisfies UsageChange; + return { _tag: "Tally", tally: yield* findTally(id) } satisfies UsageChange; + }); + return yield* sql + .withTransaction( + Effect.gen(function* () { + const { items, next } = yield* page(select, from, upto, limit); + return { + items: yield* Effect.forEach(items, change), + ...(next === undefined ? {} : { next }), + } satisfies Page; + }), + ) + .pipe(Effect.orDie); +}); + export type InboxState = "pending" | "promoted" | "cancelled"; /** A row of one thread's detail at the revision that last wrote it. */ diff --git a/packages/harness/src/session/session.ts b/packages/harness/src/session/session.ts index 27b4b310..adee0eb9 100644 --- a/packages/harness/src/session/session.ts +++ b/packages/harness/src/session/session.ts @@ -27,6 +27,7 @@ import { SpaceSchema } from "../space/schema.ts"; import * as Space from "../space/space.ts"; import * as ThreadSchema from "../thread/schema.ts"; import * as Thread from "../thread/thread.ts"; +import * as Usage from "../usage/usage.ts"; import { posix } from "../util/posix.ts"; import * as Slug from "../util/slug.ts"; import * as SessionSchema from "./schema.ts"; @@ -486,4 +487,8 @@ export const layer = Layer.effect( ); /** The session store with the thread and document stores it is built on. */ -export const storeLayer = layer.pipe(Layer.provideMerge(Thread.layer), Layer.provideMerge(Document.layer)); +export const storeLayer = layer.pipe( + Layer.provideMerge(Thread.layer), + Layer.provideMerge(Usage.layer), + Layer.provideMerge(Document.layer), +); diff --git a/packages/harness/src/thread/thread.ts b/packages/harness/src/thread/thread.ts index b8935867..6d6f8beb 100644 --- a/packages/harness/src/thread/thread.ts +++ b/packages/harness/src/thread/thread.ts @@ -18,6 +18,7 @@ import * as DocumentSchema from "../document/schema.ts"; import * as Event from "../event/event.ts"; import * as EventList from "../event/list.ts"; import * as SessionSchema from "../session/schema.ts"; +import * as Usage from "../usage/usage.ts"; import * as ThreadSchema from "./schema.ts"; /** A host interruption landed on a run the user had asked to stop; see `projectRun`. */ @@ -295,6 +296,7 @@ export const layer = Layer.effect( const store = yield* Store.Service; const events = yield* Event.Service; const documents = yield* Document.Service; + const usage = yield* Usage.Service; const findThread = SqlSchema.findOneOption({ Request: Schema.String, @@ -628,13 +630,16 @@ export const layer = Layer.effect( // The envelope is authoritative; aggregate deltas are never accepted // independently of the data persisted by this transaction. - const usage = + const billed = input.type === "assistant" ? yield* decodeEnvelopeUsage(input.data).pipe( Effect.map((envelope) => envelope.usage), Effect.mapError((error) => invalid(error.message)), ) : undefined; + // What the rollup reads once the entry is finalised, rejected at every write rather than inside + // the commit or abort, where a bad draft would make its turn impossible to end. + yield* Usage.validate(input.type, input.data).pipe(Effect.mapError((error) => invalid(error.message))); const result: AppendTxResult = yield* store .write( @@ -675,16 +680,17 @@ export const layer = Layer.effect( yield* writeParts(input.id, input.threadId, parts); yield* addUsage( thread.value, - usage === undefined + billed === undefined ? undefined : { - cost: usage.cost.total, - input: usage.input, - output: usage.output, - cacheRead: usage.cacheRead, - cacheWrite: usage.cacheWrite, + cost: billed.cost.total, + input: billed.input, + output: billed.output, + cacheRead: billed.cacheRead, + cacheWrite: billed.cacheWrite, }, ); + yield* usage.record(input.id); const stored = yield* findEntry(input.id); if (Option.isNone(stored)) { @@ -733,6 +739,7 @@ export const layer = Layer.effect( }); const replaceAssistant = Effect.fn("Thread.replaceAssistant")(function* (input: ReplaceAssistant) { + yield* Usage.validate("assistant", input.data); yield* store.write( Effect.gen(function* () { const { entry, thread } = yield* requireDraft(input.id, input.threadId, "replaceAssistant"); @@ -823,12 +830,14 @@ export const layer = Layer.effect( RETURNING id `; if (rows.length !== 1) return yield* Effect.die(`commit requires a fully-settled draft ${input.id}`); + yield* usage.record(input.id); }, store.write, Effect.orDie, ); const abort = Effect.fn("Thread.abort")(function* (input: ReplaceAssistant & EndAssistant) { + yield* Usage.validate("assistant", input.data); yield* store.write( Effect.gen(function* () { const { entry, thread } = yield* requireDraft(input.id, input.threadId, "abort"); @@ -843,6 +852,7 @@ export const layer = Layer.effect( WHERE id = ${input.id} AND state = 'draft' `; yield* addUsage(thread, delta); + yield* usage.record(input.id); }), ); }, Effect.orDie); diff --git a/packages/harness/src/usage/schema.ts b/packages/harness/src/usage/schema.ts new file mode 100644 index 00000000..f7ebd3e8 --- /dev/null +++ b/packages/harness/src/usage/schema.ts @@ -0,0 +1,33 @@ +import { Schema } from "effect"; +import { NonNegativeInt, optional } from "../schema.ts"; +import * as SessionSchema from "../session/schema.ts"; + +/** What the rollup reads of a finalised assistant envelope. `reasoning` is a subset of `output`. */ +export const Assistant = Schema.fromJsonString( + Schema.Struct({ + provider: Schema.Struct({ id: Schema.String }), + model: Schema.String, + thinkingLevel: Schema.String, + usage: Schema.Struct({ ...SessionSchema.Usage.fields, reasoning: optional(NonNegativeInt) }), + time: Schema.Struct({ created: Schema.Finite, completed: optional(Schema.Finite) }), + }), +); + +/** What the rollup reads of a committed user envelope. */ +export const User = Schema.fromJsonString(Schema.Struct({ time: Schema.Struct({ created: Schema.Finite }) })); + +/** + * Upper bounds in ms of the request duration histogram, log-spaced from 250 ms to 30 min: bucket `i` + * counts durations below `edges[i]` and at or above `edges[i - 1]`; one more bucket past the last edge + * is open-ended. Medians and percentiles read from it are accurate to a bucket. Persisted counts are + * positional, so the edges are frozen. + */ +export const edges = [ + 250, 374, 561, 839, 1257, 1882, 2818, 4220, 6318, 9461, 14167, 21213, 31764, 47563, 71221, 106645, 159688, 239115, + 358046, 536133, 802797, 1202096, 1800000, +] as const; + +export const Durations = Schema.fromJsonString(Schema.Array(NonNegativeInt)); + +export const tallyKinds = ["prompt", "tool"] as const; +export type TallyKind = (typeof tallyKinds)[number]; diff --git a/packages/harness/src/usage/usage.ts b/packages/harness/src/usage/usage.ts new file mode 100644 index 00000000..9e1456d0 --- /dev/null +++ b/packages/harness/src/usage/usage.ts @@ -0,0 +1,244 @@ +import { Context, Effect, Layer, Schema } from "effect"; +import { SqlClient } from "effect/sql"; +import type { ToolStatus } from "../db/schema.sql.ts"; +import * as Store from "../db/store.ts"; +import * as UsageSchema from "./schema.ts"; + +const HOUR = 3_600_000; +const hourOf = (time: number) => Math.floor(time / HOUR) * HOUR; + +/** The histogram bucket of a duration in ms; see `UsageSchema.edges`. */ +const bucketOf = (duration: number) => { + const index = UsageSchema.edges.findIndex((edge) => duration < edge); + return index === -1 ? UsageSchema.edges.length : index; +}; + +/** What the rollup reads of an entry. */ +export interface Entry { + readonly type: string; + readonly state: string; + readonly data: string; +} + +export interface UsageDelta { + readonly hour: number; + readonly provider: string; + readonly model: string; + readonly thinking: string; + readonly input: number; + readonly output: number; + readonly cacheRead: number; + readonly cacheWrite: number; + readonly reasoning: number; + readonly cost: number; + readonly costInput: number; + readonly costOutput: number; + readonly costCacheRead: number; + readonly costCacheWrite: number; + /** Absent unless the request committed with a usable duration. */ + readonly duration?: number; +} + +export interface TallyDelta { + readonly hour: number; + readonly kind: UsageSchema.TallyKind; + readonly name: string; + readonly count: number; +} + +/** A toolCall part of an entry. */ +export interface Call { + readonly name: string; + readonly status: ToolStatus; +} + +/** A call that ran and gave the model a result; skipped and aborted ones may never have started. */ +const ran = (call: Call) => call.status === "completed" || call.status === "error"; + +/** The rows one finalised entry adds to; empty for any other entry. */ +export interface Delta { + readonly usage?: UsageDelta; + readonly tallies: ReadonlyArray; +} + +const decodeAssistant = Schema.decodeUnknownEffect(UsageSchema.Assistant); +const decodeUser = Schema.decodeUnknownEffect(UsageSchema.User); +const decodeDurations = Schema.decodeUnknownEffect(UsageSchema.Durations); +const encodeDurations = Schema.encodeEffect(UsageSchema.Durations); + +/** Rejects an envelope the rollup could not read once its entry is finalised. */ +export const validate = Effect.fn("Usage.validate")(function* (type: string, data: string) { + if (type === "assistant") yield* decodeAssistant(data); + if (type === "user") yield* decodeUser(data); +}); + +/** + * The deltas of an entry with its toolCall parts. Only terminal assistant entries (committed or + * aborted: an aborted request is billed too) and committed user entries count, so an entry counts + * once, at the transition that finalises it. Only a committed request is timed: an aborted one's + * `completed` is when it was interrupted, failed or healed after a crash, not when it answered. + */ +export const delta = Effect.fn("Usage.delta")(function* (entry: Entry, calls: ReadonlyArray) { + const none: Delta = { tallies: [] }; + if (entry.type === "user" && entry.state === "committed") { + const envelope = yield* decodeUser(entry.data); + const prompt: Delta = { tallies: [{ hour: hourOf(envelope.time.created), kind: "prompt", name: "", count: 1 }] }; + return prompt; + } + if (entry.type !== "assistant" || (entry.state !== "committed" && entry.state !== "aborted")) { + return none; + } + const { provider, model, thinkingLevel, usage, time } = yield* decodeAssistant(entry.data); + const hour = hourOf(time.created); + const elapsed = entry.state !== "committed" || time.completed === undefined ? -1 : time.completed - time.created; + const counts = new Map(); + for (const call of calls.filter(ran)) counts.set(call.name, (counts.get(call.name) ?? 0) + 1); + const rolled: Delta = { + usage: { + hour, + provider: provider.id, + model, + thinking: thinkingLevel, + input: usage.input, + output: usage.output, + cacheRead: usage.cacheRead, + cacheWrite: usage.cacheWrite, + reasoning: usage.reasoning ?? 0, + cost: usage.cost.total, + costInput: usage.cost.input, + costOutput: usage.cost.output, + costCacheRead: usage.cost.cacheRead, + costCacheWrite: usage.cost.cacheWrite, + ...(elapsed >= 0 ? { duration: elapsed } : {}), + }, + tallies: [...counts].map(([name, count]): TallyDelta => ({ hour, kind: "tool", name, count })), + }; + return rolled; +}); + +/** Adds `change` to the rows of `threadId`, stamped `rev`. */ +const apply = Effect.fnUntraced(function* (threadId: string, rev: number, change: Delta) { + const sql = yield* SqlClient.SqlClient; + if (change.usage === undefined && change.tallies.length === 0) return; + const [owner] = yield* sql<{ readonly sessionId: string; readonly projectId: string }>` + SELECT t.session_id, sp.project_id FROM thread t + JOIN session s ON s.id = t.session_id + JOIN space sp ON sp.id = s.space_id + WHERE t.id = ${threadId} + `; + if (owner === undefined) return yield* Effect.die(new Error(`usage: thread ${threadId} is gone`)); + const usage = change.usage; + if (usage !== undefined) { + const key = sql`hour = ${usage.hour} AND thread_id = ${threadId} AND provider = ${usage.provider} + AND model = ${usage.model} AND thinking = ${usage.thinking}`; + const [stored] = yield* sql<{ readonly durations: string }>`SELECT durations FROM usage WHERE ${key}`; + const durations = [ + ...(stored === undefined + ? Array.from({ length: UsageSchema.edges.length + 1 }, () => 0) + : yield* decodeDurations(stored.durations)), + ]; + const timed = usage.duration === undefined ? 0 : 1; + if (usage.duration !== undefined) durations[bucketOf(usage.duration)]! += 1; + // Session and project are the row's first write's: a later relink never moves earlier usage. + yield* sql` + INSERT INTO usage ( + hour, thread_id, provider, model, thinking, session_id, project_id, requests, input, output, + cache_read, cache_write, reasoning, cost, cost_input, cost_output, cost_cache_read, cost_cache_write, + duration, timed, durations, rev + ) VALUES ( + ${usage.hour}, ${threadId}, ${usage.provider}, ${usage.model}, ${usage.thinking}, ${owner.sessionId}, + ${owner.projectId}, 1, ${usage.input}, ${usage.output}, ${usage.cacheRead}, ${usage.cacheWrite}, + ${usage.reasoning}, ${usage.cost}, ${usage.costInput}, ${usage.costOutput}, ${usage.costCacheRead}, + ${usage.costCacheWrite}, ${usage.duration ?? 0}, ${timed}, ${yield* encodeDurations(durations)}, ${rev} + ) + ON CONFLICT (hour, thread_id, provider, model, thinking) DO UPDATE SET + requests = requests + excluded.requests, + input = input + excluded.input, + output = output + excluded.output, + cache_read = cache_read + excluded.cache_read, + cache_write = cache_write + excluded.cache_write, + reasoning = reasoning + excluded.reasoning, + cost = cost + excluded.cost, + cost_input = cost_input + excluded.cost_input, + cost_output = cost_output + excluded.cost_output, + cost_cache_read = cost_cache_read + excluded.cost_cache_read, + cost_cache_write = cost_cache_write + excluded.cost_cache_write, + duration = duration + excluded.duration, + timed = timed + excluded.timed, + durations = excluded.durations, + rev = excluded.rev + `; + } + for (const tally of change.tallies) { + yield* sql` + INSERT INTO tally (hour, thread_id, kind, name, session_id, project_id, count, rev) + VALUES (${tally.hour}, ${threadId}, ${tally.kind}, ${tally.name}, ${owner.sessionId}, ${owner.projectId}, ${tally.count}, ${rev}) + ON CONFLICT (hour, thread_id, kind, name) DO UPDATE SET + count = count + excluded.count, + rev = excluded.rev + `; + } +}); + +/** + * Rolls the stored entry `entryId` and its toolCall parts into the rows, stamped `rev`. Fails only when + * its envelope does not decode. + */ +const rollup = Effect.fnUntraced(function* (entryId: string, rev: number) { + const sql = yield* SqlClient.SqlClient; + const [entry] = yield* sql` + SELECT thread_id, type, state, data FROM entry WHERE id = ${entryId} + `.pipe(Effect.orDie); + if (entry === undefined) return yield* Effect.die(new Error(`usage: entry ${entryId} is gone`)); + const calls = yield* sql` + SELECT tool_name AS name, status FROM part WHERE entry_id = ${entryId} AND type = 'toolCall' ORDER BY part_index + `.pipe(Effect.orDie); + const change = yield* delta(entry, calls); + yield* apply(entry.threadId, rev, change).pipe(Effect.orDie); +}); + +/** + * Rolls every finalised entry into the rows, the way the write path does, at one revision taken from + * `rev` only when there is something to roll up. Expects the rows empty. An entry whose envelope does + * not decode is skipped, so one bad historical row cannot keep the database from opening. + */ +export const backfill = Effect.fnUntraced(function* (rev: Effect.Effect) { + const sql = yield* SqlClient.SqlClient; + const entries = yield* sql<{ readonly id: string }>` + SELECT id FROM entry + WHERE (type = 'assistant' AND state IN ('committed', 'aborted')) OR (type = 'user' AND state = 'committed') + ORDER BY seq, id + `.pipe(Effect.orDie); + if (entries.length === 0) return; + const taken = yield* rev; + let skipped = 0; + for (const entry of entries) { + yield* rollup(entry.id, taken).pipe(Effect.catchTag("SchemaError", () => Effect.sync(() => skipped++))); + } + if (skipped > 0) + yield* Effect.logWarning(`usage backfill skipped ${skipped} entries whose envelope does not decode`); +}); + +export interface Interface { + /** + * Rolls a stored entry into the rows inside the enclosing `Store.write`. The write validated its + * envelope (`validate`), so one that does not decode is a defect. + */ + readonly record: (entryId: string) => Effect.Effect; +} + +export class Service extends Context.Service()("@codeworksh/harness/usage/usage/Service") {} + +export const layer = Layer.effect( + Service, + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const store = yield* Store.Service; + + const record = Effect.fn("Usage.record")(function* (entryId: string) { + yield* rollup(entryId, yield* store.rev).pipe(Effect.orDie, Effect.provideService(SqlClient.SqlClient, sql)); + }); + + return Service.of({ record }); + }), +); diff --git a/packages/harness/test/__artifacts__/usage.rollup.json b/packages/harness/test/__artifacts__/usage.rollup.json new file mode 100644 index 00000000..6141443a --- /dev/null +++ b/packages/harness/test/__artifacts__/usage.rollup.json @@ -0,0 +1,519 @@ +{ + "usage": [ + { + "hour": "2026-11-14T09:00:00.000Z", + "threadId": "main", + "provider": "openai", + "model": "gpt-5.6-luna", + "thinking": "medium", + "sessionId": "S1", + "projectId": "P1", + "requests": 1, + "input": 100, + "output": 40, + "cacheRead": 20, + "cacheWrite": 5, + "reasoning": 10, + "cost": 0.5, + "costInput": 0.25, + "costOutput": 0.125, + "costCacheRead": 0.0625, + "costCacheWrite": 0.0625, + "duration": 2000, + "timed": 1, + "durations": [ + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + }, + { + "hour": "2026-11-14T10:00:00.000Z", + "threadId": "main", + "provider": "openai", + "model": "gpt-5.6-luna", + "thinking": "medium", + "sessionId": "S1", + "projectId": "P1", + "requests": 1, + "input": 7, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0, + "reasoning": 0, + "cost": 0, + "costInput": 0, + "costOutput": 0, + "costCacheRead": 0, + "costCacheWrite": 0, + "duration": 0, + "timed": 0, + "durations": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + }, + { + "hour": "2026-11-14T10:00:00.000Z", + "threadId": "main", + "provider": "openai", + "model": "other-model", + "thinking": "high", + "sessionId": "S1", + "projectId": "P1", + "requests": 1, + "input": 300, + "output": 60, + "cacheRead": 0, + "cacheWrite": 0, + "reasoning": 0, + "cost": 0.75, + "costInput": 0.5, + "costOutput": 0.25, + "costCacheRead": 0, + "costCacheWrite": 0, + "duration": 0, + "timed": 0, + "durations": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + }, + { + "hour": "2026-11-14T10:00:00.000Z", + "threadId": "subagent", + "provider": "openai", + "model": "gpt-5.6-luna", + "thinking": "low", + "sessionId": "S1", + "projectId": "P1", + "requests": 1, + "input": 1, + "output": 1, + "cacheRead": 0, + "cacheWrite": 0, + "reasoning": 0, + "cost": 0, + "costInput": 0, + "costOutput": 0, + "costCacheRead": 0, + "costCacheWrite": 0, + "duration": 250, + "timed": 1, + "durations": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + }, + { + "hour": "2026-11-14T11:00:00.000Z", + "threadId": "fork", + "provider": "openai", + "model": "gpt-5.6-luna", + "thinking": "medium", + "sessionId": "S1", + "projectId": "P1", + "requests": 1, + "input": 50, + "output": 5, + "cacheRead": 0, + "cacheWrite": 0, + "reasoning": 0, + "cost": 0.25, + "costInput": 0.125, + "costOutput": 0.125, + "costCacheRead": 0, + "costCacheWrite": 0, + "duration": 90000, + "timed": 1, + "durations": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + }, + { + "hour": "2026-11-14T12:00:00.000Z", + "threadId": "healed", + "provider": "openai", + "model": "gpt-5.6-luna", + "thinking": "medium", + "sessionId": "S1", + "projectId": "P1", + "requests": 1, + "input": 11, + "output": 3, + "cacheRead": 0, + "cacheWrite": 0, + "reasoning": 0, + "cost": 0, + "costInput": 0, + "costOutput": 0, + "costCacheRead": 0, + "costCacheWrite": 0, + "duration": 0, + "timed": 0, + "durations": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + }, + { + "hour": "2026-11-14T13:00:00.000Z", + "threadId": "recovered", + "provider": "openai", + "model": "gpt-5.6-luna", + "thinking": "medium", + "sessionId": "S1", + "projectId": "P1", + "requests": 1, + "input": 1, + "output": 1, + "cacheRead": 0, + "cacheWrite": 0, + "reasoning": 0, + "cost": 0, + "costInput": 0, + "costOutput": 0, + "costCacheRead": 0, + "costCacheWrite": 0, + "duration": 2000, + "timed": 1, + "durations": [ + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + }, + { + "hour": "2026-11-14T20:00:00.000Z", + "threadId": "healed", + "provider": "openai", + "model": "gpt-5.6-luna", + "thinking": "medium", + "sessionId": "S1", + "projectId": "P1", + "requests": 1, + "input": 1, + "output": 1, + "cacheRead": 0, + "cacheWrite": 0, + "reasoning": 0, + "cost": 0, + "costInput": 0, + "costOutput": 0, + "costCacheRead": 0, + "costCacheWrite": 0, + "duration": 2000, + "timed": 1, + "durations": [ + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + }, + { + "hour": "2026-11-14T21:00:00.000Z", + "threadId": "recovered", + "provider": "openai", + "model": "gpt-5.6-luna", + "thinking": "medium", + "sessionId": "S1", + "projectId": "P1", + "requests": 1, + "input": 1, + "output": 1, + "cacheRead": 0, + "cacheWrite": 0, + "reasoning": 0, + "cost": 0, + "costInput": 0, + "costOutput": 0, + "costCacheRead": 0, + "costCacheWrite": 0, + "duration": 500, + "timed": 1, + "durations": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + } + ], + "tally": [ + { + "hour": "2026-11-14T09:00:00.000Z", + "threadId": "main", + "kind": "prompt", + "name": "", + "sessionId": "S1", + "projectId": "P1", + "count": 1 + }, + { + "hour": "2026-11-14T09:00:00.000Z", + "threadId": "main", + "kind": "tool", + "name": "echo", + "sessionId": "S1", + "projectId": "P1", + "count": 2 + }, + { + "hour": "2026-11-14T09:00:00.000Z", + "threadId": "main", + "kind": "tool", + "name": "shout", + "sessionId": "S1", + "projectId": "P1", + "count": 1 + }, + { + "hour": "2026-11-14T10:00:00.000Z", + "threadId": "main", + "kind": "prompt", + "name": "", + "sessionId": "S1", + "projectId": "P1", + "count": 1 + }, + { + "hour": "2026-11-14T10:00:00.000Z", + "threadId": "subagent", + "kind": "prompt", + "name": "", + "sessionId": "S1", + "projectId": "P1", + "count": 1 + }, + { + "hour": "2026-11-14T11:00:00.000Z", + "threadId": "fork", + "kind": "prompt", + "name": "", + "sessionId": "S1", + "projectId": "P1", + "count": 1 + }, + { + "hour": "2026-11-14T13:00:00.000Z", + "threadId": "recovered", + "kind": "tool", + "name": "echo", + "sessionId": "S1", + "projectId": "P1", + "count": 1 + }, + { + "hour": "2026-11-14T20:00:00.000Z", + "threadId": "healed", + "kind": "prompt", + "name": "", + "sessionId": "S1", + "projectId": "P1", + "count": 1 + }, + { + "hour": "2026-11-14T21:00:00.000Z", + "threadId": "recovered", + "kind": "prompt", + "name": "", + "sessionId": "S1", + "projectId": "P1", + "count": 1 + } + ] +} diff --git a/packages/harness/test/db/db.test.ts b/packages/harness/test/db/db.test.ts index b7d52e2d..f361886d 100644 --- a/packages/harness/test/db/db.test.ts +++ b/packages/harness/test/db/db.test.ts @@ -85,19 +85,24 @@ const registerInstances = (...ids: ReadonlyArray) => describe("Database", () => { describe("migrations", () => { memoryIt( - "applies the one threads migration and creates its tables", + "applies the migrations and creates their tables", Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; const applied = yield* sql<{ migrationId: number; name: string }>` SELECT migration_id, name FROM effect_sql_migrations ORDER BY migration_id `; - expect(applied.map((row) => `${row.migrationId}_${row.name}`)).toEqual(["202610290001_init"]); + expect(applied.map((row) => `${row.migrationId}_${row.name}`)).toEqual([ + "202610290001_init", + "202611140001_usage", + "202611140002_boot", + ]); const tables = yield* sql<{ name: string }>` SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' AND name <> 'effect_sql_migrations' ORDER BY name `; expect(tables.map((row) => row.name)).toEqual([ + "boot", "document", "entry", "event", @@ -111,7 +116,9 @@ describe("Database", () => { "session", "space", "store", + "tally", "thread", + "usage", ]); }), ); diff --git a/packages/harness/test/session.test.ts b/packages/harness/test/session.test.ts index 771c690d..4a466558 100644 --- a/packages/harness/test/session.test.ts +++ b/packages/harness/test/session.test.ts @@ -75,8 +75,12 @@ const assistantEntry = ( data: JSON.stringify({ messageId: id, role: "assistant", + provider: { id: "test" }, + model: "test", + thinkingLevel: "off", stopReason: "stop", usage: options?.usage ?? usage(), + time: { created: 1, completed: 1 }, }), parts: [{ type: "text", data: JSON.stringify({ type: "text", text: "ok" }) }], }); diff --git a/packages/harness/test/usage.test.ts b/packages/harness/test/usage.test.ts new file mode 100644 index 00000000..ffa41096 --- /dev/null +++ b/packages/harness/test/usage.test.ts @@ -0,0 +1,526 @@ +import "./utils/env.ts"; +import { createAssistantMessageEventStream, type Message } from "@codeworksh/aikit"; +import { DateTime, Effect, Layer, Schema } from "effect"; +import { SqlClient } from "effect/sql"; +import * as TestClock from "effect/testing/TestClock"; +import { describe, expect } from "vite-plus/test"; +import * as Context from "../src/context/context.ts"; +import * as Control from "../src/control.ts"; +import * as Database from "../src/db/db.ts"; +import { migrations } from "../src/db/migrations.ts"; +import type { TallyRow, UsageRow } from "../src/db/schema.sql.ts"; +import * as Store from "../src/db/store.ts"; +import * as Sync from "../src/effect/sync.ts"; +import * as Event from "../src/event/event.ts"; +import * as EventList from "../src/event/list.ts"; +import * as EventRegistry from "../src/event/registry.ts"; +import * as Global from "../src/global.ts"; +import { builtins } from "../src/plugin/builtin.ts"; +import type { Plugin } from "../src/plugin/plugin.ts"; +import * as RunnerExecute from "../src/runner/execute.ts"; +import * as Hooks from "../src/runner/hooks.ts"; +import * as LLM from "../src/runner/llm.ts"; +import * as Loop from "../src/runner/loop.ts"; +import * as SandboxController from "../src/sandbox/control.ts"; +import * as SandboxDriverRegistry from "../src/sandbox/registry.ts"; +import * as SessionMessageSchema from "../src/session/message/schema.ts"; +import * as PromptSchema from "../src/session/prompt/schema.ts"; +import * as SessionProjector from "../src/session/projector.ts"; +import * as SessionRuntime from "../src/session/runtime.ts"; +import * as SessionSchema from "../src/session/schema.ts"; +import * as Session from "../src/session/session.ts"; +import * as Settings from "../src/settings/settings.ts"; +import * as State from "../src/state/state.ts"; +import * as ThreadSchema from "../src/thread/schema.ts"; +import * as Thread from "../src/thread/thread.ts"; +import * as Tool from "../src/tool/tool.ts"; +import * as Usage from "../src/usage/usage.ts"; +import * as Mcp from "../src/mcp/mcp.ts"; +import * as Process from "../src/sandbox/utils/process.ts"; +import { assistant } from "./fixtures/llm.ts"; +import { credentials, named } from "./fixtures/model.ts"; +import { pooled } from "./fixtures/pool.ts"; +import { seedSpace } from "./fixtures/space.ts"; +import { testEffect } from "./utils/effect.ts"; + +/** One model request: its times, usage, what it calls, and whether the provider fails it. */ +interface Step { + readonly created: number; + readonly completed: number; + readonly tools?: ReadonlyArray; + readonly model?: string; + readonly thinking?: Message.AssistantMessage["thinkingLevel"]; + readonly usage?: Partial; + readonly fail?: boolean; +} + +const HOUR = 3_600_000; +// 2026-11-14T10:00:00Z: the scenario straddles it. +const H = Date.UTC(2026, 10, 14, 10); + +const script = new Map>(); +let responses = 0; +/** The last request's input, which a crashed draft is built from. */ +let requested: LLM.Input | undefined; + +const open: LLM.Open = (input) => + Effect.sync(() => { + requested = input; + const step = script.get(input.threadId)?.shift(); + if (step === undefined) throw new Error(`no step scripted for ${input.threadId}`); + responses += 1; + const base = assistant(input, responses).usage; + const message = assistant(input, responses, { + ...(step.model === undefined ? {} : { model: step.model }), + ...(step.thinking === undefined ? {} : { thinkingLevel: step.thinking }), + usage: { ...base, ...step.usage }, + time: { created: step.created, completed: step.completed }, + ...(step.fail === true + ? { stopReason: "error", errorMessage: "provider failed" } + : step.tools === undefined + ? {} + : { + stopReason: "toolUse", + parts: step.tools.map((name, index) => ({ + type: "toolCall", + callID: `call_${responses}_${index}`, + name, + arguments: { value: name }, + status: "pending", + time: { start: step.created, end: step.created }, + })), + }), + }); + const stream = createAssistantMessageEventStream(); + stream.push({ type: "start", partial: message }); + if (step.fail === true) { + const failure = { _tag: "Authentication", reason: "missing", message: "provider failed", retryable: false }; + stream.push({ type: "error", reason: "error", error: { ...message, failure } as Message.AssistantMessage }); + } else stream.push({ type: "done", reason: step.tools === undefined ? "stop" : "toolUse", message }); + return stream; + }); + +const echoTools: Plugin = { + id: "test.tool.echo", + kind: "tool", + setup: (ctx) => { + for (const name of ["echo", "shout"]) { + ctx.plugin.tools.add( + Tool.register( + Tool.make({ + name, + description: `${name}es its value`, + parameters: Schema.Struct({ value: Schema.String }), + success: Schema.String, + encodeContent: (value: string) => [{ type: "text", text: value }], + handler: ({ value }) => Effect.succeed(value), + }), + ), + ); + } + }, +}; + +const layer = () => { + const database = Database.layer(":memory:"); + const sandbox = SandboxController.layer({ hostCwd: "/" }).pipe( + Layer.provideMerge(SandboxDriverRegistry.layer()), + Layer.provideMerge(database), + ); + const seeded = pooled([...builtins, echoTools]); + return Control.layer.pipe( + Layer.provideMerge( + RunnerExecute.layer.pipe( + Layer.provide(Loop.layer({ request: LLM.make(open) }).pipe(Layer.provide(Hooks.layer))), + ), + ), + Layer.provideMerge(State.layer({}, seeded.ref, seeded.references, seeded.follow, seeded.rebuild)), + Layer.provideMerge(credentials), + Layer.provideMerge(Mcp.layer({ hostCwd: "/", home: "/unused/home" }).pipe(Layer.provide(Process.host))), + Layer.provideMerge(SessionRuntime.layer), + Layer.provideMerge( + Layer.succeed(Settings.Service, { + load: () => Effect.succeed(named), + locate: () => Effect.die("unused"), + }), + ), + Layer.provideMerge(sandbox), + Layer.provideMerge(Context.layer), + Layer.provideMerge(SessionProjector.layer), + Layer.provideMerge(Session.storeLayer), + Layer.provideMerge(Event.layer), + Layer.provideMerge(EventRegistry.layer()), + Layer.provideMerge(database), + Layer.provideMerge(Layer.succeed(Global.Service, Global.make({ home: "/unused/home" }))), + ); +}; + +const { effect: it } = testEffect(layer()); + +const artifact = (name: string, value: unknown) => + Effect.promise(() => + expect(`${JSON.stringify(value, null, "\t")}\n`).toMatchFileSnapshot(`./__artifacts__/${name}`), + ); + +/** Prompts `threadId` at clock time `at` and waits for the run, which may fail. */ +const run = (sessionId: SessionSchema.ID, threadId: ThreadSchema.ID, at: number, text: string) => + Effect.gen(function* () { + const control = yield* Control.Service; + yield* TestClock.setTime(at); + yield* control + .run({ sessionId, threadId, prompt: PromptSchema.Prompt.fromText(text), delivery: "followUp" }) + .pipe(Effect.exit); + yield* control.wait(threadId); + }); + +/** + * What a host that died mid-turn leaves for the next run to heal: the draft it was writing, ended when + * its response finished, with the calls in `ran` settled as completed, and the run interrupted. + */ +const crashed = Effect.fnUntraced(function* ( + sessionId: SessionSchema.ID, + threadId: ThreadSchema.ID, + draft: Message.AssistantMessage, + ran: ReadonlyArray, +) { + const events = yield* Event.Service; + const ids = { sessionId, threadId, messageId: SessionMessageSchema.ID.from(draft.messageId) }; + const at = DateTime.makeUnsafe(draft.time.completed); + yield* events.publish(EventList.LLMStarted, { timestamp: at, ...ids, message: draft }); + if (draft.stopReason === "toolUse") { + yield* events.publish(EventList.LLMEnded, { timestamp: at, ...ids, reason: "toolUse", message: draft }); + } + for (const part of draft.parts) { + if (part.type !== "toolCall" || part.status !== "pending" || !ran.includes(part.callID)) continue; + yield* events.publish(EventList.ToolSettled, { + timestamp: at, + ...ids, + callID: part.callID, + part: { ...part, status: "completed", result: { content: [{ type: "text", text: "ran" }], isError: false } }, + }); + } + yield* events.publish(EventList.ExecutionInterrupted, { timestamp: at, sessionId, threadId, reason: "crash" }); +}); + +/** + * One session: main runs a tool turn across the 10:00 boundary (two tools, two requests on different + * models and thinking levels) and a request the provider fails; a fork and a subagent thread run one + * request each. Two more threads heal what a crash left: `healed` a request in flight at 12:00, aborted + * when the host is back at 20:00; `recovered` a response that ended at 13:00 with two calls, one of which + * ran, committed at 21:00 with the other aborted unrun. + */ +const scenario = Effect.gen(function* () { + script.clear(); + responses = 0; + const sessions = yield* Session.Service; + const threads = yield* Thread.Service; + const { spaceId, location } = yield* seedSpace({ location: process.cwd(), projectId: "p" }); + const created = yield* sessions.create({ spaceId, directory: location }); + const sessionId = SessionSchema.ID.make(created.session.id); + const main = ThreadSchema.ID.make(created.thread.id); + + const cost = (input: number, output: number, cacheRead: number, cacheWrite: number) => ({ + input, + output, + cacheRead, + cacheWrite, + total: input + output + cacheRead + cacheWrite, + }); + script.set(main, [ + { + created: H - 30_000, + completed: H - 28_000, + tools: ["echo", "shout", "echo"], + usage: { + input: 100, + output: 40, + cacheRead: 20, + cacheWrite: 5, + reasoning: 10, + totalTokens: 165, + cost: cost(0.25, 0.125, 0.0625, 0.0625), + }, + }, + // Completed before it was created: billed, but no duration. + { + created: H + 1_000, + completed: H + 400, + model: "other-model", + thinking: "high", + usage: { input: 300, output: 60, totalTokens: 360, cost: cost(0.5, 0.25, 0, 0) }, + }, + { created: H + 130_000, completed: H + 135_000, fail: true, usage: { input: 7, output: 0, totalTokens: 7 } }, + ]); + yield* run(sessionId, main, H - 60_000, "first"); + yield* run(sessionId, main, H + 120_000, "second"); + + const line = yield* threads.line(main); + const second = line.filter(({ entry }) => entry.type === "assistant" && entry.state === "committed")[1]!; + const fork = ThreadSchema.ID.make((yield* threads.fork({ threadId: main, entryId: second.entry.id })).thread.id); + script.set(fork, [ + { + created: H + HOUR + 100_000, + completed: H + HOUR + 190_000, + usage: { input: 50, output: 5, totalTokens: 55, cost: cost(0.125, 0.125, 0, 0) }, + }, + ]); + yield* run(sessionId, fork, H + HOUR + 90_000, "fork"); + + const subagent = ThreadSchema.ID.create(); + yield* threads.insert({ + id: subagent, + sessionId, + origin: { kind: "subagent", threadId: main, callId: "call_1_0" }, + seq: 1, + }); + // Exactly on the first edge: the second bucket. + script.set(subagent, [{ created: H + 5_000, completed: H + 5_250, thinking: "low" }]); + yield* run(sessionId, subagent, H + 4_000, "subagent"); + + const thread = Effect.fnUntraced(function* (callId: string) { + const id = ThreadSchema.ID.create(); + yield* threads.insert({ id, sessionId, origin: { kind: "subagent", threadId: main, callId }, seq: 1 }); + return id; + }); + const input = requested!; + const healed = yield* thread("call_1_1"); + yield* crashed( + sessionId, + healed, + assistant(input, 101, { + usage: { ...assistant(input, 101).usage, input: 11, output: 3, totalTokens: 14 }, + time: { created: H + 2 * HOUR, completed: H + 2 * HOUR }, + }), + [], + ); + script.set(healed, [{ created: H + 10 * HOUR + 1_000, completed: H + 10 * HOUR + 3_000 }]); + yield* run(sessionId, healed, H + 10 * HOUR, "healed"); + + const recovered = yield* thread("call_1_2"); + const call = (callID: string, name: string): Message.ToolCallPendingPart => ({ + type: "toolCall", + callID, + name, + arguments: { value: name }, + status: "pending", + time: { start: H + 3 * HOUR, end: H + 3 * HOUR }, + }); + yield* crashed( + sessionId, + recovered, + assistant(input, 102, { + stopReason: "toolUse", + parts: [call("call_ran", "echo"), call("call_unrun", "shout")], + time: { created: H + 3 * HOUR, completed: H + 3 * HOUR + 2_000 }, + }), + ["call_ran"], + ); + script.set(recovered, [{ created: H + 11 * HOUR + 1_000, completed: H + 11 * HOUR + 1_500 }]); + yield* run(sessionId, recovered, H + 11 * HOUR, "recovered"); + + const sql = yield* SqlClient.SqlClient; + const [owner] = yield* sql<{ projectId: string }>` + SELECT sp.project_id FROM session s JOIN space sp ON sp.id = s.space_id WHERE s.id = ${sessionId} + `; + return { sessionId, projectId: owner!.projectId, threads: { main, fork, subagent, healed, recovered } }; +}); + +type Plain = Omit; +const plain = (rows: ReadonlyArray): Array> => + rows.map(({ rev: _rev, ...rest }) => rest); +const byKey = (rows: ReadonlyArray) => + [...rows].sort((left, right) => JSON.stringify(left).localeCompare(JSON.stringify(right))); + +/** Every row, read the way the sync reads them: one page wide enough for all of them. */ +const rollup = Effect.gen(function* () { + const store = yield* Store.Service; + const { items, next } = yield* Sync.usage({ rev: 0 }, (yield* store.head).rev, 10_000); + expect(next).toBeUndefined(); + return { + usage: items.flatMap((item) => (item._tag === "Usage" ? [item.usage] : [])), + tally: items.flatMap((item) => (item._tag === "Tally" ? [item.tally] : [])), + }; +}); + +/** The invariants the rollup holds against the thread rows and entries, per thread. */ +const verify = Effect.fnUntraced(function* ( + sessionId: SessionSchema.ID, + rows: { readonly usage: ReadonlyArray; readonly tally: ReadonlyArray }, +) { + const sql = yield* SqlClient.SqlClient; + const threads = yield* sql<{ + id: string; + cost: number; + tokensInput: number; + tokensOutput: number; + tokensCacheRead: number; + tokensCacheWrite: number; + }>`SELECT * FROM thread WHERE session_id = ${sessionId}`; + for (const thread of threads) { + const own = rows.usage.filter((row) => row.threadId === thread.id); + const sum = (pick: (row: UsageRow) => number) => own.reduce((total, row) => total + pick(row), 0); + expect({ + cost: sum((row) => row.cost), + input: sum((row) => row.input), + output: sum((row) => row.output), + cacheRead: sum((row) => row.cacheRead), + cacheWrite: sum((row) => row.cacheWrite), + }).toEqual({ + cost: thread.cost, + input: thread.tokensInput, + output: thread.tokensOutput, + cacheRead: thread.tokensCacheRead, + cacheWrite: thread.tokensCacheWrite, + }); + const [counts] = yield* sql<{ prompts: number; requests: number; timed: number }>` + SELECT + count(*) FILTER (WHERE type = 'user' AND state = 'committed') AS prompts, + count(*) FILTER (WHERE type = 'assistant' AND state IN ('committed', 'aborted')) AS requests, + count(*) FILTER ( + WHERE type = 'assistant' AND state = 'committed' + AND data ->> '$.time.completed' >= data ->> '$.time.created' + ) AS timed + FROM entry WHERE thread_id = ${thread.id} + `; + const tallied = (kind: string) => rows.tally.filter((row) => row.threadId === thread.id && row.kind === kind); + expect(tallied("prompt").reduce((total, row) => total + row.count, 0)).toBe(counts!.prompts); + expect(sum((row) => row.requests)).toBe(counts!.requests); + // Only committed requests are timed. + expect(sum((row) => row.timed)).toBe(counts!.timed); + const calls = yield* sql<{ toolName: string; calls: number }>` + SELECT p.tool_name, count(*) AS calls FROM part p JOIN entry e ON e.id = p.entry_id + WHERE e.thread_id = ${thread.id} AND p.type = 'toolCall' AND e.state IN ('committed', 'aborted') + AND p.status IN ('completed', 'error') + GROUP BY p.tool_name + `; + const tools = new Map(); + for (const row of tallied("tool")) tools.set(row.name, (tools.get(row.name) ?? 0) + row.count); + expect(Object.fromEntries(tools)).toEqual(Object.fromEntries(calls.map((row) => [row.toolName, row.calls]))); + for (const row of own) { + expect(row.timed).toBeLessThanOrEqual(row.requests); + expect(row.durations.reduce((total, count) => total + count, 0)).toBe(row.timed); + } + } +}); + +describe("usage rollup", () => { + it( + "rolls finalised entries into hourly rows that match the thread totals", + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const { sessionId, projectId, threads } = yield* scenario; + const rows = yield* rollup; + yield* verify(sessionId, rows); + expect( + yield* sql`SELECT state FROM entry WHERE session_id = ${sessionId} AND type = 'assistant' ORDER BY seq`, + ).toEqual( + ["committed", "committed", "aborted", "committed", "committed"] + .concat(["aborted", "committed", "committed", "committed"]) + .map((state) => ({ state })), + ); + const at = (threadId: string, hour: number) => + rows.usage.filter((row) => row.threadId === threadId && row.hour === hour); + const untimed = { timed: 0, duration: 0, durations: Array.from({ length: 24 }, () => 0) }; + // The provider failed it, and the crash healed hours later: billed, never timed. + expect(at(threads.main, H).find((row) => row.model === "gpt-5.6-luna")).toMatchObject({ + requests: 1, + input: 7, + ...untimed, + }); + expect(at(threads.healed, H + 2 * HOUR)).toMatchObject([{ requests: 1, input: 11, output: 3, ...untimed }]); + // Its response ended before the crash: timed as it streamed. + expect(at(threads.recovered, H + 3 * HOUR)).toMatchObject([{ requests: 1, timed: 1, duration: 2_000 }]); + expect( + rows.tally + .filter((row) => row.threadId === threads.recovered && row.kind === "tool") + .map((row) => [row.name, row.count]), + ).toEqual([["echo", 1]]); + const name = new Map([ + [threads.main, "main"], + [threads.fork, "fork"], + [threads.subagent, "subagent"], + [threads.healed, "healed"], + [threads.recovered, "recovered"], + [sessionId, "S1"], + [projectId, "P1"], + ]); + const normal = < + A extends { hour: number; threadId: string; sessionId: string; projectId: string; rev: number }, + >( + row: A, + ) => { + const { rev: _rev, ...rest } = row; + return { + ...rest, + hour: new Date(row.hour).toISOString(), + threadId: name.get(row.threadId), + sessionId: name.get(row.sessionId), + projectId: name.get(row.projectId), + }; + }; + // Revisions move with unrelated writes; the order they give is checked, not their values. + expect(rows.usage.map((row) => row.rev)).toEqual(rows.usage.map((row) => row.rev).sort((a, b) => a - b)); + yield* artifact("usage.rollup.json", { + usage: byKey(rows.usage.map(normal)), + tally: byKey(rows.tally.map(normal)), + }); + }), + ); + + it( + "keeps a deleted session's rows", + Effect.gen(function* () { + const control = yield* Control.Service; + const sql = yield* SqlClient.SqlClient; + const { sessionId } = yield* scenario; + const before = yield* rollup; + expect(yield* control.deleteSession(sessionId)).toBe(true); + expect(yield* sql`SELECT id FROM thread WHERE session_id = ${sessionId}`).toEqual([]); + const after = yield* rollup; + expect(plain(after.usage)).toEqual(plain(before.usage)); + expect(plain(after.tally)).toEqual(plain(before.tally)); + expect(after.usage.every((row) => row.sessionId === sessionId)).toBe(true); + }), + ); + + it( + "backfills the same rows from the entries", + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const store = yield* Store.Service; + const { sessionId } = yield* scenario; + const live = yield* rollup; + yield* sql`DELETE FROM usage`; + yield* sql`DELETE FROM tally`; + const head = (yield* store.head).rev; + yield* store.write(Usage.backfill(store.rev)); + const filled = yield* rollup; + expect(byKey(plain(filled.usage))).toEqual(byKey(plain(live.usage))); + expect(byKey(plain(filled.tally))).toEqual(byKey(plain(live.tally))); + // One revision for everything it wrote. + expect(new Set([...filled.usage, ...filled.tally].map((row) => row.rev))).toEqual(new Set([head + 1])); + yield* verify(sessionId, filled); + }), + ); + + it( + "migrates entries written before the rollup, skipping an envelope it cannot read", + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const { sessionId, threads } = yield* scenario; + const live = yield* rollup; + yield* sql`DROP TABLE usage`; + yield* sql`DROP TABLE tally`; + yield* sql`UPDATE store SET rev = rev + 1`; + yield* sql` + INSERT INTO entry (id, session_id, thread_id, seq, committed_seq, type, state, data, created_at, updated_at, rev) + VALUES ('legacy', ${sessionId}, ${threads.main}, 1000000, 1000000, 'assistant', 'committed', '{}', 0, 0, + (SELECT rev FROM store)) + `; + yield* sql.withTransaction(migrations["202611140001_usage"]); + const migrated = yield* rollup; + expect(byKey(plain(migrated.usage))).toEqual(byKey(plain(live.usage))); + expect(byKey(plain(migrated.tally))).toEqual(byKey(plain(live.tally))); + }), + ); +});