diff --git a/apps/server/src/db.ts b/apps/server/src/db.ts index 27997f6a..4c38b794 100644 --- a/apps/server/src/db.ts +++ b/apps/server/src/db.ts @@ -343,6 +343,13 @@ const migrations = { yield* sql`ALTER TABLE machine DROP COLUMN name`; yield* sql`ALTER TABLE machine RENAME COLUMN alias TO name`; }), + + // `boot` is the store's latest open while the index was read up to `cursor`, as `rollup_boot` is + // to `rollup`. + "202611140003_boot": Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* sql`ALTER TABLE machine ADD COLUMN boot TEXT`; + }), }; const setup = Effect.gen(function* () { diff --git a/apps/server/src/sync.ts b/apps/server/src/sync.ts index 404000eb..b3baf333 100644 --- a/apps/server/src/sync.ts +++ b/apps/server/src/sync.ts @@ -187,6 +187,9 @@ export const layer = Layer.effect( Effect.gen(function* () { const [row] = yield* sql<{ cursor: number | null }>`SELECT cursor FROM thread WHERE id = ${threadId}`; if (row === undefined) return; + // A first open is the thread's line as the store has it now, so its first batch replaces the + // thread's own detail: after a restored store, entries of the lost line would hold its seqs. + let first = row.cursor === null; yield* batches( live.client("thread.sync", { sessionId, @@ -201,21 +204,30 @@ export const layer = Layer.effect( ? noteEntry(threadId, item.entry).pipe(Effect.as(true)) : Effect.succeed(true), (rows, synced) => - transact((append) => - Effect.gen(function* () { - const changes = yield* Mirror.thread(threadId, rows); - yield* sql` + Effect.gen(function* () { + const whole = first; + yield* transact((append) => + Effect.gen(function* () { + if (whole) { + yield* sql`DELETE FROM entry WHERE thread_id = ${threadId}`; + yield* sql`DELETE FROM inbox WHERE thread_id = ${threadId}`; + } + const changes = yield* Mirror.thread(threadId, rows); + yield* sql` UPDATE thread SET cursor = ${synced.rev}, floor = COALESCE(${synced.floor ?? null}, floor) WHERE id = ${threadId} `; - if (rows.length > 0 || synced.floor !== undefined) - yield* append({ - topic: syncTopics.changed, - source: "sync", - payload: changes.notice({ machineId, threadId, full: false, items: rows.length }), - }); - }), - ).pipe(Effect.andThen(Effect.suspend(() => readyOf(threadId)))), + if (rows.length > 0 || synced.floor !== undefined) + yield* append({ + topic: syncTopics.changed, + source: "sync", + payload: changes.notice({ machineId, threadId, full: false, items: rows.length }), + }); + }), + ); + first = false; + yield* readyOf(threadId); + }), ); }).pipe( Effect.catchCause((cause) => @@ -322,17 +334,28 @@ export const layer = Layer.effect( return yield* Effect.never; }); - /** One connection's sync: decides between resume and full sync, then follows the index. */ + /** + * One connection's sync: decides between resume and full sync, then follows the index. A read + * from zero is a full sync, whose first batch also sweeps what it did not send. A resume that left + * the store's line (by `boot`) comes back from zero with `reset`; a server older than boots is + * read from zero only while its head is behind the cursor. + */ const connected = (machineId: string, client: Connection) => Effect.gen(function* () { const info = yield* client("server.info", {}); yield* servers.setHost(machineId, info.host?.name ?? null); - const [machine] = yield* sql<{ storeId: string | null; cursor: number; namespace: string }>` - SELECT store_id, cursor, namespace FROM machine WHERE id = ${machineId} + const [machine] = yield* sql<{ + storeId: string | null; + cursor: number; + boot: string | null; + namespace: string; + }>` + SELECT store_id, cursor, boot, namespace FROM machine WHERE id = ${machineId} `; if (machine === undefined) return yield* Effect.never; yield* unclaimed(machineId, info.storeId); - let full = info.storeId !== machine.storeId || info.rev < machine.cursor; + const resumed = info.storeId === machine.storeId && info.rev >= machine.cursor; + let full = !resumed; const live: Live = { client, threads: yield* FiberMap.make() }; yield* Effect.addFinalizer(() => Effect.sync(() => { @@ -342,13 +365,16 @@ export const layer = Layer.effect( const daemons = yield* FiberHandle.make(); let following = false; yield* batches( - client("index.sync", { after: full ? 0 : machine.cursor }), + client("index.sync", { + after: resumed ? machine.cursor : 0, + ...(resumed && machine.boot !== null ? { boot: machine.boot } : {}), + }), (item) => item._tag === "Synced", () => Effect.succeed(true), (rows, synced) => Effect.gen(function* () { const now = yield* Clock.currentTimeMillis; - const wasFull = full; + const wasFull = full || synced.reset === true; let spaces = false; yield* transact((append) => Effect.gen(function* () { @@ -361,7 +387,9 @@ export const layer = Layer.effect( if (wasFull) yield* Mirror.unseen(machineId, changes, now); spaces = changes.space.size > 0; yield* sql` - UPDATE machine SET cursor = ${synced.rev}, store_id = ${synced.storeId} WHERE id = ${machineId} + UPDATE machine SET cursor = ${synced.rev}, store_id = ${synced.storeId}, + boot = ${synced.boot ?? null} + WHERE id = ${machineId} `; if (rows.length > 0 || wasFull) yield* append({ diff --git a/apps/server/test/__artifacts__/metrics.simulation.json b/apps/server/test/__artifacts__/metrics.simulation.json index 2612b80d..67511d82 100644 --- a/apps/server/test/__artifacts__/metrics.simulation.json +++ b/apps/server/test/__artifacts__/metrics.simulation.json @@ -2,11 +2,11 @@ "seeded": { "cw-a": { "sessions": 300, - "prompts": 2277, - "requests": 7321, - "aborted": 196, - "failed": 137, - "calls": 7778 + "prompts": 2245, + "requests": 7273, + "aborted": 209, + "failed": 140, + "calls": 7827 }, "cw-b": { "sessions": 120, @@ -20,24 +20,24 @@ "connected": { "before": { "all": { - "requests": 10160, - "tokens": 496714759, - "cost": 228.254801875, - "timed": 9866, - "prompts": 3158, - "tools": 10315 + "requests": 10112, + "tokens": 490774644, + "cost": 221.19770285, + "timed": 9805, + "prompts": 3126, + "tools": 10347 }, "sessions": 420, "deleted": 0 }, "after": { "all": { - "requests": 10160, - "tokens": 496714759, - "cost": 228.254801875, - "timed": 9866, - "prompts": 3158, - "tools": 10315 + "requests": 10112, + "tokens": 490774644, + "cost": 221.19770285, + "timed": 9805, + "prompts": 3126, + "tools": 10347 }, "sessions": 420, "deleted": 7 @@ -45,12 +45,12 @@ }, "live": { "all": { - "requests": 10166, - "tokens": 496714771, - "cost": 228.254801875, - "timed": 9872, - "prompts": 3161, - "tools": 10318 + "requests": 10118, + "tokens": 490774656, + "cost": 221.19770285, + "timed": 9811, + "prompts": 3129, + "tools": 10350 }, "sessions": 421, "deleted": 7 @@ -62,12 +62,12 @@ ], "totals": { "all": { - "requests": 10178, - "tokens": 496714795, - "cost": 228.254801875, - "timed": 9884, - "prompts": 3167, - "tools": 10324 + "requests": 10130, + "tokens": 490774680, + "cost": 221.19770285, + "timed": 9823, + "prompts": 3135, + "tools": 10356 }, "sessions": 423, "deleted": 7 @@ -76,24 +76,24 @@ "paused": { "paused": { "all": { - "requests": 10182, - "tokens": 496714803, - "cost": 228.254801875, - "timed": 9888, - "prompts": 3169, - "tools": 10326 + "requests": 10134, + "tokens": 490774688, + "cost": 221.19770285, + "timed": 9827, + "prompts": 3137, + "tools": 10358 }, "sessions": 424, "deleted": 7 }, "resumed": { "all": { - "requests": 10182, - "tokens": 496714803, - "cost": 228.254801875, - "timed": 9888, - "prompts": 3169, - "tools": 10326 + "requests": 10134, + "tokens": 490774688, + "cost": 221.19770285, + "timed": 9827, + "prompts": 3137, + "tools": 10358 }, "sessions": 424, "deleted": 7 @@ -101,76 +101,76 @@ }, "restartedAway": { "all": { - "requests": 10186, - "tokens": 496714811, - "cost": 228.254801875, - "timed": 9892, - "prompts": 3171, - "tools": 10328 + "requests": 10138, + "tokens": 490774696, + "cost": 221.19770285, + "timed": 9831, + "prompts": 3139, + "tools": 10360 }, "sessions": 425, "deleted": 7 }, "cut": { "all": { - "requests": 10188, - "tokens": 496714815, - "cost": 228.254801875, - "timed": 9894, - "prompts": 3172, - "tools": 10329 + "requests": 10140, + "tokens": 490774700, + "cost": 221.19770285, + "timed": 9833, + "prompts": 3140, + "tools": 10361 }, "sessions": 426, "deleted": 7 }, "restored": { "all": { - "requests": 10192, - "tokens": 496714823, - "cost": 228.254801875, - "timed": 9898, - "prompts": 3174, - "tools": 10331 + "requests": 10144, + "tokens": 490774708, + "cost": 221.19770285, + "timed": 9837, + "prompts": 3142, + "tools": 10363 }, "sessions": 428, - "deleted": 9 + "deleted": 7 }, "readded": { "kept": { "all": { - "requests": 10192, - "tokens": 496714823, - "cost": 228.254801875, - "timed": 9898, - "prompts": 3174, - "tools": 10331 + "requests": 10144, + "tokens": 490774708, + "cost": 221.19770285, + "timed": 9837, + "prompts": 3142, + "tools": 10363 }, "sessions": 428, - "deleted": 130 + "deleted": 128 }, "again": { "all": { - "requests": 10192, - "tokens": 496714823, - "cost": 228.254801875, - "timed": 9898, - "prompts": 3174, - "tools": 10331 + "requests": 10144, + "tokens": 490774708, + "cost": 221.19770285, + "timed": 9837, + "prompts": 3142, + "tools": 10363 }, "sessions": 428, - "deleted": 9 + "deleted": 7 } }, "recreated": { "all": { - "requests": 10196, - "tokens": 496714831, - "cost": 228.254801875, - "timed": 9902, - "prompts": 3176, - "tools": 10333 + "requests": 10148, + "tokens": 490774716, + "cost": 221.19770285, + "timed": 9841, + "prompts": 3144, + "tools": 10365 }, "sessions": 429, - "deleted": 130 + "deleted": 128 } } diff --git a/apps/server/test/__artifacts__/schema.json b/apps/server/test/__artifacts__/schema.json index 7ce0f74e..bd03305f 100644 --- a/apps/server/test/__artifacts__/schema.json +++ b/apps/server/test/__artifacts__/schema.json @@ -70,7 +70,7 @@ "type": "table", "name": "machine", "tbl_name": "machine", - "sql": "CREATE TABLE machine ( id TEXT PRIMARY KEY, kind TEXT NOT NULL, executable TEXT, home TEXT, args TEXT NOT NULL DEFAULT '', env TEXT NOT NULL DEFAULT '[]', url TEXT, namespace TEXT NOT NULL, is_default INTEGER NOT NULL DEFAULT 0, pos INTEGER NOT NULL, store_id TEXT UNIQUE, cursor INTEGER NOT NULL DEFAULT 0, created_at INTEGER NOT NULL , rollup INTEGER NOT NULL DEFAULT 0, rollup_store TEXT, rollup_boot TEXT, host TEXT, name TEXT)" + "sql": "CREATE TABLE machine ( id TEXT PRIMARY KEY, kind TEXT NOT NULL, executable TEXT, home TEXT, args TEXT NOT NULL DEFAULT '', env TEXT NOT NULL DEFAULT '[]', url TEXT, namespace TEXT NOT NULL, is_default INTEGER NOT NULL DEFAULT 0, pos INTEGER NOT NULL, store_id TEXT UNIQUE, cursor INTEGER NOT NULL DEFAULT 0, created_at INTEGER NOT NULL , rollup INTEGER NOT NULL DEFAULT 0, rollup_store TEXT, rollup_boot TEXT, host TEXT, name TEXT, boot TEXT)" }, { "type": "table", @@ -237,7 +237,8 @@ "rollup_store": null, "rollup_boot": null, "host": null, - "name": null + "name": null, + "boot": null } ], "organization": [ @@ -309,7 +310,8 @@ "rollup_store": null, "rollup_boot": null, "host": null, - "name": "Cloud tunnel" + "name": "Cloud tunnel", + "boot": null }, "cascade": { "before": { diff --git a/apps/server/test/__artifacts__/sync.passed.json b/apps/server/test/__artifacts__/sync.passed.json new file mode 100644 index 00000000..b00cbfb8 --- /dev/null +++ b/apps/server/test/__artifacts__/sync.passed.json @@ -0,0 +1,520 @@ +{ + "resumed": [ + { + "machineId": "", + "full": false, + "items": 5, + "project": [ + "" + ], + "space": [ + "" + ], + "session": [ + "" + ], + "thread": [ + "", + "" + ], + "entry": [], + "inbox": [], + "removed": [] + }, + { + "machineId": "", + "threadId": "", + "full": false, + "items": 3, + "project": [], + "space": [], + "session": [], + "thread": [], + "entry": [ + "", + "" + ], + "inbox": [ + "" + ], + "removed": [] + } + ], + "changed": [ + { + "machineId": "", + "full": true, + "items": 12, + "project": [ + "" + ], + "space": [ + "" + ], + "session": [ + "", + "", + "", + "", + "" + ], + "thread": [ + "", + "", + "", + "", + "" + ], + "entry": [], + "inbox": [], + "removed": [ + "", + "" + ] + }, + { + "machineId": "", + "threadId": "", + "full": false, + "items": 4, + "project": [], + "space": [], + "session": [], + "thread": [], + "entry": [ + "", + "", + "", + "" + ], + "inbox": [], + "removed": [] + } + ], + "lost": { + "entries": [ + "", + "", + "", + "" + ] + }, + "restored": { + "entries": [ + "", + "", + "", + "" + ] + }, + "session": [ + { + "id": "", + "machine_id": "", + "organization_id": "personal", + "space_id": "", + "slug": "", + "dir": "", + "hostdir": null, + "tag": null, + "metadata": null, + "color": null, + "created_at": "