From 44146cf5a467a07c953bdab93e84f3255ba3cd4b Mon Sep 17 00:00:00 2001 From: Innei Date: Sun, 25 Oct 2026 15:50:49 +0800 Subject: [PATCH 01/19] feat(draft): store Lexical drafts as one Loro working copy Content documents in Lexical keep a Loro doc as the source of truth with a linear checkpoint timeline. Adds HTTP sync, checkpoints, field patches, history, restore and undo endpoints; legacy draft writes apply as snapshots at their base version. Markdown keeps the branch flow. --- apps/core/package.json | 2 + .../src/common/adapters/fastify.adapter.ts | 8 + apps/core/src/common/errors/app-error-code.ts | 2 + .../common/errors/app-error-definitions.ts | 10 + .../src/common/errors/app-error-payload.ts | 8 + .../migrations/0043_loro_content_storage.sql | 24 + .../database/migrations/meta/_journal.json | 7 + .../ai/ai-writer/ai-slug-backfill.service.ts | 4 +- .../content-migration-commit.service.ts | 19 +- .../content-migration.schema.ts | 1 + .../content-migration.service.ts | 15 +- apps/core/src/modules/draft/draft-content.ts | 12 + apps/core/src/modules/draft/draft-crdt.doc.ts | 217 +++++ .../modules/draft/draft-crdt.repository.ts | 242 ++++++ .../src/modules/draft/draft-crdt.service.ts | 754 ++++++++++++++++++ .../src/modules/draft/draft.controller.ts | 113 ++- apps/core/src/modules/draft/draft.module.ts | 4 + .../src/modules/draft/draft.repository.ts | 6 +- apps/core/src/modules/draft/draft.schema.ts | 41 + apps/core/src/modules/draft/draft.service.ts | 125 ++- apps/core/src/modules/draft/draft.types.ts | 32 + apps/core/src/modules/note/note.controller.ts | 4 +- apps/core/src/modules/note/note.service.ts | 17 + apps/core/src/modules/post/post.controller.ts | 6 +- apps/core/src/modules/post/post.service.ts | 13 + .../src/modules/publish/publish.service.ts | 20 +- .../admin/drafts-admin.contract.spec.ts | 1 + .../admin/drafts-crdt-admin.contract.spec.ts | 146 ++++ .../test/src/contracts/draft.contract.spec.ts | 1 + .../content-migration-commit.service.spec.ts | 37 + .../content-migration.service.spec.ts | 34 + .../src/modules/draft/draft-crdt.doc.spec.ts | 225 ++++++ .../draft-crdt.repository.pg.e2e.spec.ts | 240 ++++++ .../draft/draft-crdt.service.pg.e2e.spec.ts | 464 +++++++++++ .../draft/draft.service.crdt.pg.e2e.spec.ts | 291 +++++++ .../src/modules/draft/draft.service.spec.ts | 4 + .../modules/publish/publish.service.spec.ts | 62 +- apps/core/vite.config.ts | 2 +- dockerfile | 4 +- .../2026-10-25-loro-content-storage-design.md | 464 +++++++++++ packages/db-schema/src/schema/content.ts | 43 + pnpm-lock.yaml | 6 + 42 files changed, 3692 insertions(+), 38 deletions(-) create mode 100644 apps/core/src/database/migrations/0043_loro_content_storage.sql create mode 100644 apps/core/src/modules/draft/draft-crdt.doc.ts create mode 100644 apps/core/src/modules/draft/draft-crdt.repository.ts create mode 100644 apps/core/src/modules/draft/draft-crdt.service.ts create mode 100644 apps/core/test/src/contracts/admin/drafts-crdt-admin.contract.spec.ts create mode 100644 apps/core/test/src/modules/draft/draft-crdt.doc.spec.ts create mode 100644 apps/core/test/src/modules/draft/draft-crdt.repository.pg.e2e.spec.ts create mode 100644 apps/core/test/src/modules/draft/draft-crdt.service.pg.e2e.spec.ts create mode 100644 apps/core/test/src/modules/draft/draft.service.crdt.pg.e2e.spec.ts create mode 100644 docs/superpowers/specs/2026-10-25-loro-content-storage-design.md diff --git a/apps/core/package.json b/apps/core/package.json index 21381de31f9..2e71c9c3c37 100644 --- a/apps/core/package.json +++ b/apps/core/package.json @@ -69,6 +69,7 @@ "@earendil-works/pi-ai": "0.99.1", "@fastify/multipart": "10.1.2", "@fastify/static": "10.1.5", + "@haklex/rich-collab-loro": "0.54.4", "@haklex/rich-headless": "0.54.4", "@innei/message-engine": "0.5.0", "@innei/next-async": "0.4.0", @@ -125,6 +126,7 @@ "keyv": "5.6.0", "lexical": "^0.52.0", "linkedom": "0.18.13", + "loro-crdt": "^1.16.4", "lru-cache": "11.5.3", "marked": "18.0.14", "mime-types": "^3.0.2", diff --git a/apps/core/src/common/adapters/fastify.adapter.ts b/apps/core/src/common/adapters/fastify.adapter.ts index 3572bfe36a2..030261bdc22 100644 --- a/apps/core/src/common/adapters/fastify.adapter.ts +++ b/apps/core/src/common/adapters/fastify.adapter.ts @@ -71,6 +71,14 @@ app.register(FastifyMultipart, { }, }) +app + .getInstance() + .addContentTypeParser( + 'application/octet-stream', + { parseAs: 'buffer' }, + (_request, body, done) => done(null, body), + ) + const DEFAULT_RAW_BODY_LIMIT = 1024 * 1024 export function isRawBodyRoute(request: FastifyRequest): boolean { diff --git a/apps/core/src/common/errors/app-error-code.ts b/apps/core/src/common/errors/app-error-code.ts index 74feae8b98e..f7ad3783e88 100644 --- a/apps/core/src/common/errors/app-error-code.ts +++ b/apps/core/src/common/errors/app-error-code.ts @@ -114,6 +114,8 @@ export enum AppErrorCode { DRAFT_HEAD_CONFLICT = 'DRAFT_HEAD_CONFLICT', DRAFT_SHARE_NOT_FOUND = 'DRAFT_SHARE_NOT_FOUND', PUBLISHED_REVISION_CHANGED = 'PUBLISHED_REVISION_CHANGED', + DRAFT_CRDT_UPDATE_REJECTED = 'DRAFT_CRDT_UPDATE_REJECTED', + DRAFT_CRDT_FORMAT_UNSUPPORTED = 'DRAFT_CRDT_FORMAT_UNSUPPORTED', // document / helper DOCUMENT_NOT_FOUND = 'DOCUMENT_NOT_FOUND', diff --git a/apps/core/src/common/errors/app-error-definitions.ts b/apps/core/src/common/errors/app-error-definitions.ts index d1f439877e5..32489979ad3 100644 --- a/apps/core/src/common/errors/app-error-definitions.ts +++ b/apps/core/src/common/errors/app-error-definitions.ts @@ -445,6 +445,16 @@ export const APP_ERROR_DEFINITIONS = { expectedPublishedRevisionId: p.expectedPublishedRevisionId, }), }, + [AppErrorCode.DRAFT_CRDT_UPDATE_REJECTED]: { + status: 400, + message: 'The collaborative update was rejected', + details: (p) => ({ branchId: p.branchId, reason: p.reason }), + }, + [AppErrorCode.DRAFT_CRDT_FORMAT_UNSUPPORTED]: { + status: 400, + message: 'This draft only accepts Lexical content', + details: (p) => ({ branchId: p.branchId, contentFormat: p.contentFormat }), + }, // document / helper [AppErrorCode.DOCUMENT_NOT_FOUND]: { diff --git a/apps/core/src/common/errors/app-error-payload.ts b/apps/core/src/common/errors/app-error-payload.ts index d843b567fb0..2312343b2ac 100644 --- a/apps/core/src/common/errors/app-error-payload.ts +++ b/apps/core/src/common/errors/app-error-payload.ts @@ -128,6 +128,14 @@ export type AppErrorPayloadMap = { documentId: string expectedPublishedRevisionId: string | null } + [AppErrorCode.DRAFT_CRDT_UPDATE_REJECTED]: { + branchId: string + reason: string + } + [AppErrorCode.DRAFT_CRDT_FORMAT_UNSUPPORTED]: { + branchId: string + contentFormat: string + } // document / helper [AppErrorCode.DOCUMENT_NOT_FOUND]: WithId diff --git a/apps/core/src/database/migrations/0043_loro_content_storage.sql b/apps/core/src/database/migrations/0043_loro_content_storage.sql new file mode 100644 index 00000000000..b58fc7708be --- /dev/null +++ b/apps/core/src/database/migrations/0043_loro_content_storage.sql @@ -0,0 +1,24 @@ +-- migration-lint:allow=no-bare-create-index reason=index targets the brand-new empty content_crdt_updates table; CONCURRENTLY cannot run inside the migration transaction +CREATE TABLE "content_crdt_docs" ( + "document_id" text PRIMARY KEY NOT NULL, + "snapshot" bytea NOT NULL, + "snapshot_version" bytea NOT NULL, + "synced_revision_id" text, + "fields" jsonb, + "created_at" timestamp with time zone DEFAULT now() NOT NULL, + "updated_at" timestamp with time zone +); +--> statement-breakpoint +CREATE TABLE "content_crdt_updates" ( + "id" text PRIMARY KEY NOT NULL, + "document_id" text NOT NULL, + "bytes" bytea NOT NULL, + "created_at" timestamp with time zone DEFAULT now() NOT NULL +); +--> statement-breakpoint +ALTER TABLE "content_revisions" ADD COLUMN "frontiers" jsonb;--> statement-breakpoint +ALTER TABLE "content_revisions" ADD COLUMN "kind" text;--> statement-breakpoint +ALTER TABLE "content_crdt_docs" ADD CONSTRAINT "content_crdt_docs_document_id_content_documents_id_fk" FOREIGN KEY ("document_id") REFERENCES "public"."content_documents"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "content_crdt_docs" ADD CONSTRAINT "content_crdt_docs_synced_revision_id_content_revisions_id_fk" FOREIGN KEY ("synced_revision_id") REFERENCES "public"."content_revisions"("id") ON DELETE set null ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "content_crdt_updates" ADD CONSTRAINT "content_crdt_updates_document_id_content_documents_id_fk" FOREIGN KEY ("document_id") REFERENCES "public"."content_documents"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +CREATE INDEX "content_crdt_updates_document_idx" ON "content_crdt_updates" USING btree ("document_id"); diff --git a/apps/core/src/database/migrations/meta/_journal.json b/apps/core/src/database/migrations/meta/_journal.json index 5890dfe687e..f1c34805636 100644 --- a/apps/core/src/database/migrations/meta/_journal.json +++ b/apps/core/src/database/migrations/meta/_journal.json @@ -302,6 +302,13 @@ "when": 1792900000000, "tag": "0042_agent_transcript", "breakpoints": true + }, + { + "idx": 43, + "version": "7", + "when": 1792950000000, + "tag": "0043_loro_content_storage", + "breakpoints": true } ] } diff --git a/apps/core/src/modules/ai/ai-writer/ai-slug-backfill.service.ts b/apps/core/src/modules/ai/ai-writer/ai-slug-backfill.service.ts index 17ac2f667f0..574cc68e3ee 100644 --- a/apps/core/src/modules/ai/ai-writer/ai-slug-backfill.service.ts +++ b/apps/core/src/modules/ai/ai-writer/ai-slug-backfill.service.ts @@ -143,9 +143,9 @@ export class AiSlugBackfillService implements OnModuleInit { continue } - const updated = await this.noteService.updateById(note.id, { + const updated = await this.noteService.patchById(note.id, { slug, - } as any) + }) if (!updated) { skipped++ await context.appendLog( diff --git a/apps/core/src/modules/content-migration/content-migration-commit.service.ts b/apps/core/src/modules/content-migration/content-migration-commit.service.ts index 87200f4f692..06b930f1bb3 100644 --- a/apps/core/src/modules/content-migration/content-migration-commit.service.ts +++ b/apps/core/src/modules/content-migration/content-migration-commit.service.ts @@ -310,7 +310,15 @@ export class ContentMigrationCommitService { ) .innerJoin( contentRevisions, - eq(contentRevisions.id, drafts.headRevisionId), + and( + eq( + contentRevisions.id, + parseEntityId( + branchPrecondition.headRevisionId ?? input.branchId, + ), + ), + eq(contentRevisions.documentId, drafts.documentId), + ), ) .where(eq(drafts.id, parseEntityId(input.branchId))) .limit(1) @@ -329,10 +337,11 @@ export class ContentMigrationCommitService { draft.revision.content ? sha256(draft.revision.content) : analyzeMigrationMarkdown(draft.revision.text, input).sourceHash - if ( - draft.branch.headRevisionId !== branchPrecondition.headRevisionId || - draftHash !== branchPrecondition.hash - ) { + // A checkpoint is immutable, so it may be published after the working copy moved on. + const headMoved = + !draft.revision.kind && + draft.branch.headRevisionId !== branchPrecondition.headRevisionId + if (headMoved || draftHash !== branchPrecondition.hash) { throw new ConflictException('Draft changed after the migration check') } } diff --git a/apps/core/src/modules/content-migration/content-migration.schema.ts b/apps/core/src/modules/content-migration/content-migration.schema.ts index 97d7b6115c7..b8a264c5315 100644 --- a/apps/core/src/modules/content-migration/content-migration.schema.ts +++ b/apps/core/src/modules/content-migration/content-migration.schema.ts @@ -26,6 +26,7 @@ export const MarkdownToLexicalDryRunSchema = z.object({ refType: z.enum(DraftRefType), refId: zEntityId, branchId: zEntityId.optional(), + revisionId: zEntityId.optional(), sourceText: z.string().max(2_000_000), profile: z.literal('yohaku-v1'), }) diff --git a/apps/core/src/modules/content-migration/content-migration.service.ts b/apps/core/src/modules/content-migration/content-migration.service.ts index 279ab8e87ab..b1b38046131 100644 --- a/apps/core/src/modules/content-migration/content-migration.service.ts +++ b/apps/core/src/modules/content-migration/content-migration.service.ts @@ -154,18 +154,23 @@ export class ContentMigrationService { }) if (draft) { - const draftResult = analyzeMxMarkdown(draft.headRevision.text, { + const revision = dto.revisionId + ? await this.draftService.findRevisionById(dto.revisionId) + : draft.headRevision + if (!revision || revision.documentId !== draft.documentId) { + throw new BadRequestException('Revision does not belong to this draft') + } + const draftResult = analyzeMxMarkdown(revision.text, { profile: 'yohaku-v1', }) preconditions.push({ kind: 'branch', id: String(draft.id), hash: - draft.headRevision.contentFormat === ContentFormat.Lexical && - draft.headRevision.content - ? sha256(draft.headRevision.content) + revision.contentFormat === ContentFormat.Lexical && revision.content + ? sha256(revision.content) : draftResult.sourceHash, - headRevisionId: draft.headRevisionId, + headRevisionId: dto.revisionId ?? draft.headRevisionId, }) } diff --git a/apps/core/src/modules/draft/draft-content.ts b/apps/core/src/modules/draft/draft-content.ts index 5968fb00183..2478fa4a0f1 100644 --- a/apps/core/src/modules/draft/draft-content.ts +++ b/apps/core/src/modules/draft/draft-content.ts @@ -76,3 +76,15 @@ export function sameRevisionContent( JSON.stringify(normalizeJson(left)) === JSON.stringify(normalizeJson(right)) ) } + +export function pickSnapshot(input: RevisionSnapshot): RevisionSnapshot { + return { + content: input.content, + contentFormat: input.contentFormat, + images: input.images, + meta: input.meta, + text: input.text, + title: input.title, + typeSpecificData: input.typeSpecificData, + } +} diff --git a/apps/core/src/modules/draft/draft-crdt.doc.ts b/apps/core/src/modules/draft/draft-crdt.doc.ts new file mode 100644 index 00000000000..9401914b303 --- /dev/null +++ b/apps/core/src/modules/draft/draft-crdt.doc.ts @@ -0,0 +1,217 @@ +import { + $reconcileRoot, + createLoroBinding, + TREE_NAME, +} from '@haklex/rich-collab-loro' +import { allHeadlessNodes } from '@haklex/rich-headless' +import { createHeadlessEditor } from '@lexical/headless' +import { + DecoratorNode, + type Klass, + type LexicalEditor, + type LexicalNode, + type NodeKey, + type SerializedEditorState, + type SerializedLexicalNode, +} from 'lexical' +import { LoroDoc, type OpId } from 'loro-crdt' + +import type { Frontiers } from './draft.types' + +export interface DocContent { + content: string + title: string +} + +const TITLE = 'title' + +const BUILTIN_TYPES = new Set([ + 'root', + 'paragraph', + 'text', + 'linebreak', + 'tab', + ...allHeadlessNodes.map((node) => + ('replace' in node ? (node.replace as Klass) : node).getType(), + ), +]) + +const passthroughCache = new Map>() + +// Blocks only the admin editor renders (map, stock, …); the server keeps their JSON verbatim. +const passthroughNode = (type: string): Klass => { + const cached = passthroughCache.get(type) + if (cached) return cached + class PassthroughNode extends DecoratorNode { + __json: SerializedLexicalNode + + static getType(): string { + return type + } + + static clone(node: PassthroughNode): PassthroughNode { + return new PassthroughNode(node.__json, node.__key) + } + + static importJSON(json: SerializedLexicalNode): PassthroughNode { + // The Loro binding rebuilds every node with `children: []`, which a leaf block never had. + const { children: _children, ...rest } = json as SerializedLexicalNode & { + children?: unknown + } + return new PassthroughNode(rest) + } + + constructor( + json: SerializedLexicalNode = { type, version: 1 }, + key?: NodeKey, + ) { + super(key) + this.__json = json + } + + exportJSON(): SerializedLexicalNode { + return this.__json + } + + createDOM(): never { + throw new Error(`${type} has no DOM on the server`) + } + + updateDOM(): boolean { + return false + } + + isInline(): boolean { + return false + } + + decorate(): null { + return null + } + } + passthroughCache.set(type, PassthroughNode) + return PassthroughNode +} + +const collectJsonTypes = (node: unknown, out: Set) => { + if (!node || typeof node !== 'object') return + const { children, type } = node as { children?: unknown; type?: unknown } + if (typeof type === 'string') out.add(type) + if (Array.isArray(children)) { + for (const child of children) collectJsonTypes(child, out) + } +} + +const collectDocTypes = (doc: LoroDoc, out: Set) => { + for (const node of doc.getTree(TREE_NAME).nodes()) { + const type = node.data.get('type') + if (typeof type === 'string') out.add(type) + } +} + +const createEditor = (types: Set): LexicalEditor => + createHeadlessEditor({ + nodes: [ + ...allHeadlessNodes, + ...[...types] + .filter((type) => !BUILTIN_TYPES.has(type)) + .map(passthroughNode), + ], + onError: (error) => { + throw error + }, + }) + +const parseState = (content: string): SerializedEditorState => + JSON.parse(content) as SerializedEditorState + +export const createDocument = (): LoroDoc => { + const doc = new LoroDoc() + doc.setRecordTimestamp(true) + return doc +} + +export function createContentDoc( + content: DocContent, + message: string, +): LoroDoc { + const doc = createDocument() + const state = parseState(content.content) + const types = new Set() + collectJsonTypes(state.root, types) + const editor = createEditor(types) + editor.setEditorState(editor.parseEditorState(state)) + doc.getText(TITLE).insert(0, content.title) + createLoroBinding(editor, doc, { commitMessage: message }).dispose() + doc.commit({ message }) + return doc +} + +export function readDocContent(doc: LoroDoc): DocContent { + const types = new Set() + collectDocTypes(doc, types) + const editor = createEditor(types) + if (doc.getTree(TREE_NAME).roots().length > 0) { + createLoroBinding(editor, doc).dispose() + } + return { + content: JSON.stringify(editor.getEditorState().toJSON()), + title: doc.getText(TITLE).toString(), + } +} + +export function readDocContentAt( + doc: LoroDoc, + frontiers: Frontiers, +): DocContent { + return readDocContent(doc.forkAt(frontiers)) +} + +export function editDocAt( + doc: LoroDoc, + frontiers: Frontiers, + target: DocContent, + message: string, +): Uint8Array { + const fork = doc.forkAt(frontiers) + fork.setRecordTimestamp(true) + const state = parseState(target.content) + const types = new Set() + collectDocTypes(fork, types) + collectJsonTypes(state.root, types) + const editor = createEditor(types) + const binding = createLoroBinding(editor, fork, { commitMessage: message }) + const from = fork.oplogVersion() + const title = fork.getText(TITLE) + if (title.toString() !== target.title) title.update(target.title) + editor.update(() => $reconcileRoot(state.root), { discrete: true }) + fork.commit({ message }) + binding.dispose() + return fork.export({ from, mode: 'update' }) +} + +export function revertChange( + doc: LoroDoc, + id: OpId, + before: OpId[], + message: string, +): Uint8Array { + const fork = doc.fork() + fork.setRecordTimestamp(true) + const from = fork.oplogVersion() + fork.applyDiff(doc.diff([id], before, false), { fullState: true }) + fork.commit({ message }) + return fork.export({ from, mode: 'update' }) +} + +export function vetUpdate(doc: LoroDoc, bytes: Uint8Array): LoroDoc { + const probe = doc.fork() + if (probe.import(bytes).pending) { + throw new Error('update depends on changes missing from the document') + } + if (probe.getTree(TREE_NAME).roots().length > 1) { + throw new Error('update belongs to a different document lineage') + } + readDocContent(probe) + return probe +} diff --git a/apps/core/src/modules/draft/draft-crdt.repository.ts b/apps/core/src/modules/draft/draft-crdt.repository.ts new file mode 100644 index 00000000000..f28d6655cf0 --- /dev/null +++ b/apps/core/src/modules/draft/draft-crdt.repository.ts @@ -0,0 +1,242 @@ +import { Inject, Injectable } from '@nestjs/common' +import { and, asc, eq, inArray, sql } from 'drizzle-orm' + +import { PG_DB_TOKEN } from '~/constants/system.constant' +import { + contentCrdtDocs, + contentCrdtUpdates, + contentRevisions, + drafts, +} from '~/database/schema' +import { + BaseRepository, + toEntityId, +} from '~/processors/database/base.repository' +import type { AppDatabase } from '~/processors/database/postgres.provider' +import { type EntityId, parseEntityId } from '~/shared/id/entity-id' +import { SnowflakeService } from '~/shared/id/snowflake.service' + +import { mapBranch, mapRevision } from './draft.repository' +import type { + CheckpointInput, + ContentRevisionRow, + CrdtDocRow, + CrdtFields, + CrdtFieldsPatch, + DraftBranchRow, +} from './draft.types' + +const mapDoc = (row: typeof contentCrdtDocs.$inferSelect): CrdtDocRow => ({ + ...row, + documentId: toEntityId(row.documentId)!, + syncedRevisionId: toEntityId(row.syncedRevisionId), +}) + +export interface CreateCrdtDocInput { + documentId: EntityId | string + fields: CrdtFields | null + snapshot: Uint8Array + snapshotVersion: Uint8Array + syncedRevisionId: EntityId | string | null +} + +export interface CompactInput { + expectedVersion: Uint8Array + foldedIds: EntityId[] + snapshot: Uint8Array + snapshotVersion: Uint8Array +} + +@Injectable() +export class DraftCrdtRepository extends BaseRepository { + constructor( + @Inject(PG_DB_TOKEN) db: AppDatabase, + private readonly snowflake: SnowflakeService, + ) { + super(db) + } + + // Holds a transaction open only to own the advisory lock; `run` uses its own connections. + async withDocumentLock( + documentId: EntityId | string, + run: () => Promise, + ): Promise { + return this.db.transaction(async (tx) => { + await tx.execute( + sql`select pg_advisory_xact_lock(hashtext(${`crdt-edit:${parseEntityId(documentId)}`}))`, + ) + return run() + }) + } + + async findDoc(documentId: EntityId | string): Promise { + const [row] = await this.db + .select() + .from(contentCrdtDocs) + .where(eq(contentCrdtDocs.documentId, parseEntityId(documentId))) + .limit(1) + return row ? mapDoc(row) : null + } + + async createDoc(input: CreateCrdtDocInput): Promise { + await this.db + .insert(contentCrdtDocs) + .values({ + documentId: parseEntityId(input.documentId), + fields: input.fields, + snapshot: input.snapshot, + snapshotVersion: input.snapshotVersion, + syncedRevisionId: input.syncedRevisionId + ? parseEntityId(input.syncedRevisionId) + : null, + }) + .onConflictDoNothing({ target: contentCrdtDocs.documentId }) + return (await this.findDoc(input.documentId))! + } + + async appendUpdate( + documentId: EntityId | string, + bytes: Uint8Array, + ): Promise { + const id = this.snowflake.nextId() + await this.db.insert(contentCrdtUpdates).values({ + bytes, + documentId: parseEntityId(documentId), + id, + }) + return toEntityId(id)! + } + + async findUpdates( + documentId: EntityId | string, + ): Promise<{ bytes: Uint8Array; id: EntityId }[]> { + const rows = await this.db + .select({ bytes: contentCrdtUpdates.bytes, id: contentCrdtUpdates.id }) + .from(contentCrdtUpdates) + .where(eq(contentCrdtUpdates.documentId, parseEntityId(documentId))) + .orderBy(asc(contentCrdtUpdates.createdAt), asc(contentCrdtUpdates.id)) + return rows.map((row) => ({ bytes: row.bytes, id: toEntityId(row.id)! })) + } + + async compact( + documentId: EntityId | string, + input: CompactInput, + ): Promise { + const documentIdValue = parseEntityId(documentId) + return this.db.transaction(async (tx) => { + const lock = await tx.execute<{ locked: boolean }>( + sql`select pg_try_advisory_xact_lock(hashtext(${`crdt-compact:${documentIdValue}`})) as locked`, + ) + if (!lock.rows[0]?.locked) return false + const updated = await tx + .update(contentCrdtDocs) + .set({ + snapshot: input.snapshot, + snapshotVersion: input.snapshotVersion, + updatedAt: new Date(), + }) + .where( + and( + eq(contentCrdtDocs.documentId, documentIdValue), + sql`${contentCrdtDocs.snapshotVersion} = ${Buffer.from(input.expectedVersion)}`, + ), + ) + .returning({ documentId: contentCrdtDocs.documentId }) + if (!updated.length) return false + if (input.foldedIds.length) { + await tx.delete(contentCrdtUpdates).where( + and( + eq(contentCrdtUpdates.documentId, documentIdValue), + inArray( + contentCrdtUpdates.id, + input.foldedIds.map((id) => parseEntityId(id)), + ), + ), + ) + } + return true + }) + } + + async setSyncedRevision( + documentId: EntityId | string, + revisionId: EntityId | string, + ): Promise { + await this.db + .update(contentCrdtDocs) + .set({ + syncedRevisionId: parseEntityId(revisionId), + updatedAt: new Date(), + }) + .where(eq(contentCrdtDocs.documentId, parseEntityId(documentId))) + } + + async mergeFields( + documentId: EntityId | string, + patch: CrdtFieldsPatch, + ): Promise { + const merge = (key: keyof CrdtFields) => + patch[key] + ? sql`(case when jsonb_typeof(${contentCrdtDocs.fields}->${key}) = 'object' then ${contentCrdtDocs.fields}->${key} else '{}'::jsonb end) || ${JSON.stringify(patch[key])}::jsonb` + : sql`coalesce(${contentCrdtDocs.fields}->${key}, 'null'::jsonb)` + const [row] = await this.db + .update(contentCrdtDocs) + .set({ + fields: sql`jsonb_build_object('meta', ${merge('meta')}, 'typeSpecificData', ${merge('typeSpecificData')})`, + updatedAt: new Date(), + }) + .where(eq(contentCrdtDocs.documentId, parseEntityId(documentId))) + .returning({ fields: contentCrdtDocs.fields }) + return row?.fields ?? { meta: null, typeSpecificData: null } + } + + async replaceFields( + documentId: EntityId | string, + fields: CrdtFields, + ): Promise { + await this.db + .update(contentCrdtDocs) + .set({ fields, updatedAt: new Date() }) + .where(eq(contentCrdtDocs.documentId, parseEntityId(documentId))) + } + + async appendCheckpoints( + branchId: EntityId | string, + inputs: CheckpointInput[], + ): Promise<{ branch: DraftBranchRow; revisions: ContentRevisionRow[] }> { + return this.db.transaction(async (tx) => { + const branchIdValue = parseEntityId(branchId) + const [branch] = await tx + .select() + .from(drafts) + .where(eq(drafts.id, branchIdValue)) + .for('update') + .limit(1) + if (!branch) throw new Error('DRAFT_NOT_FOUND') + + let parentRevisionId = branch.headRevisionId + const revisions: ContentRevisionRow[] = [] + for (const input of inputs) { + const [revision] = await tx + .insert(contentRevisions) + .values({ + ...input.snapshot, + documentId: branch.documentId, + frontiers: input.frontiers, + id: this.snowflake.nextId(), + kind: input.kind, + parentRevisionId, + }) + .returning() + revisions.push(mapRevision(revision)) + parentRevisionId = revision.id + } + const [updated] = await tx + .update(drafts) + .set({ headRevisionId: parentRevisionId, updatedAt: new Date() }) + .where(eq(drafts.id, branchIdValue)) + .returning() + return { branch: mapBranch(updated), revisions } + }) + } +} diff --git a/apps/core/src/modules/draft/draft-crdt.service.ts b/apps/core/src/modules/draft/draft-crdt.service.ts new file mode 100644 index 00000000000..5ad6ad145e1 --- /dev/null +++ b/apps/core/src/modules/draft/draft-crdt.service.ts @@ -0,0 +1,754 @@ +import { mxLexicalToMarkdown } from '@mx-space/editor' +import { Injectable, Logger } from '@nestjs/common' +import { type LoroDoc, type OpId, VersionVector } from 'loro-crdt' + +import { AppErrorCode, createAppException } from '~/common/errors' +import { FileReferenceType } from '~/modules/file/file-reference.enum' +import { FileReferenceService } from '~/modules/file/file-reference.service' +import type { EntityId } from '~/shared/id/entity-id' +import { ContentFormat } from '~/shared/types/content-format.type' + +import type { DraftRefType } from './draft.enum' +import { DraftRepository } from './draft.repository' +import type { + CheckpointInput, + CheckpointKind, + ContentDocumentRow, + ContentRevisionRow, + CrdtDocRow, + CrdtFields, + CrdtFieldsPatch, + DraftBranchRow, + RevisionSnapshot, +} from './draft.types' +import { + canonicalSnapshot, + pickSnapshot, + sameRevisionContent, +} from './draft-content' +import { + createContentDoc, + createDocument, + type DocContent, + editDocAt, + readDocContent, + revertChange, + vetUpdate, +} from './draft-crdt.doc' +import { DraftCrdtRepository } from './draft-crdt.repository' + +const COMPACT_THRESHOLD = 100 +const MAX_UPDATE_BYTES = 4 * 1024 * 1024 + +const EMPTY_STATE = JSON.stringify({ + root: { + children: [], + direction: null, + format: '', + indent: 0, + type: 'root', + version: 1, + }, +}) + +interface Session { + branch: DraftBranchRow + doc: LoroDoc + document: ContentDocumentRow + row: CrdtDocRow +} + +export interface CrdtChange { + deps: OpId[] + id: OpId + lamport: number + message: string | undefined + timestamp: number +} + +export type CrdtCheckpoint = Pick< + ContentRevisionRow, + | 'contentFormat' + | 'createdAt' + | 'frontiers' + | 'id' + | 'kind' + | 'parentRevisionId' + | 'title' +> + +export interface CrdtHistory { + changes: CrdtChange[] + checkpoints: CrdtCheckpoint[] +} + +export interface CheckpointResult { + branch: DraftBranchRow + revision: ContentRevisionRow +} + +const fieldsOf = (snapshot: RevisionSnapshot): CrdtFields => ({ + meta: snapshot.meta, + typeSpecificData: snapshot.typeSpecificData, +}) + +const sameBytes = (left: Uint8Array, right: Uint8Array) => + left.length === right.length && left.every((byte, i) => byte === right[i]) + +const changedKeys = ( + base: Record | null, + next: Record | null, +): Record | undefined => { + const patch: Record = {} + for (const key of new Set([ + ...Object.keys(base ?? {}), + ...Object.keys(next ?? {}), + ])) { + const value = next?.[key] ?? null + if (JSON.stringify(base?.[key] ?? null) !== JSON.stringify(value)) { + patch[key] = value + } + } + return Object.keys(patch).length ? patch : undefined +} + +@Injectable() +export class DraftCrdtService { + private readonly logger = new Logger(DraftCrdtService.name) + + constructor( + private readonly drafts: DraftRepository, + private readonly crdt: DraftCrdtRepository, + private readonly fileReferences: FileReferenceService, + ) {} + + async usesCrdt(branchId: EntityId | string): Promise { + const branch = await this.drafts.findBranchById(branchId) + if (!branch) return false + if (await this.crdt.findDoc(branch.documentId)) return true + const head = await this.drafts.findRevisionById(branch.headRevisionId) + return head?.contentFormat === ContentFormat.Lexical + } + + async workingCopy(branchId: EntityId | string): Promise { + return (await this.open(branchId)).branch + } + + async workingCopyOfDocument( + documentId: EntityId | string, + ): Promise { + const document = await this.drafts.findDocumentById(documentId) + if (!document) return null + if (!(await this.crdt.findDoc(document.id))) { + const [candidate] = await this.drafts.findBranchesByDocument(document.id) + const revisionId = + candidate?.headRevisionId ?? document.publishedRevisionId + if (!revisionId) return null + const head = await this.drafts.findRevisionById(revisionId) + if (head?.contentFormat !== ContentFormat.Lexical) return null + } + return (await this.openDocument(document)).branch + } + + async exportUpdates( + branchId: EntityId | string, + since?: Uint8Array, + ): Promise { + const { doc } = await this.open(branchId) + if (!since) return doc.export({ mode: 'snapshot' }) + let from: VersionVector + try { + from = VersionVector.decode(since) + } catch { + return doc.export({ mode: 'snapshot' }) + } + return doc.export({ from, mode: 'update' }) + } + + async importUpdate( + branchId: EntityId | string, + bytes: Uint8Array, + ): Promise<{ branch: DraftBranchRow; document: ContentDocumentRow }> { + const session = await this.open(branchId) + if (bytes.length > MAX_UPDATE_BYTES) { + this.reject(session.branch, 'update is too large') + } + try { + vetUpdate(session.doc, bytes) + } catch (error) { + this.reject(session.branch, (error as Error).message) + } + await this.crdt.appendUpdate(session.document.id, bytes) + return { branch: session.branch, document: session.document } + } + + async createCheckpoint( + branchId: EntityId | string, + kind: Extract, + ): Promise { + const session = await this.open(branchId) + const head = await this.revision(session.branch.headRevisionId) + const snapshot = this.materialize(session, readDocContent(session.doc)) + if (head.frontiers && sameRevisionContent(pickSnapshot(head), snapshot)) { + return { branch: session.branch, revision: head } + } + return this.checkpoint(session, kind, snapshot) + } + + async updateFields( + branchId: EntityId | string, + patch: CrdtFieldsPatch, + ): Promise { + const { document } = await this.open(branchId) + return this.crdt.mergeFields(document.id, patch) + } + + async mergeDocumentFields( + documentId: EntityId | string, + typeSpecificData: Record, + ): Promise { + if (!(await this.crdt.findDoc(documentId))) return + await this.crdt.mergeFields(documentId, { typeSpecificData }) + } + + async legacyWrite( + branchId: EntityId | string, + baseRevisionId: EntityId | string | null, + snapshot: RevisionSnapshot, + ): Promise { + return this.mutate(branchId, async (session) => { + if (snapshot.contentFormat !== ContentFormat.Lexical) { + throw createAppException(AppErrorCode.DRAFT_CRDT_FORMAT_UNSUPPORTED, { + branchId: session.branch.id, + contentFormat: snapshot.contentFormat, + }) + } + const base = baseRevisionId + ? await this.revision(baseRevisionId, session.document.id) + : null + const target: DocContent = { + content: snapshot.content ?? EMPTY_STATE, + title: snapshot.title, + } + const writeFields = fieldsOf(snapshot) + const message = `legacy-save ${base?.id ?? session.branch.headRevisionId}` + const at = this.knownFrontiers( + session.doc, + base ? (base.frontiers ?? (await this.importedFrontiers(base))) : null, + ) + + const bytes = editDocAt( + session.doc, + at ?? session.doc.frontiers(), + target, + message, + ) + let own: CheckpointInput + if (at) { + const fork = session.doc.forkAt(at) + fork.import(bytes) + own = { + frontiers: fork.frontiers(), + kind: 'legacy-save', + snapshot: this.materialize( + session, + readDocContent(fork), + writeFields, + ), + } + } else { + own = { + frontiers: null, + kind: 'legacy-save', + snapshot: this.materialize(session, target, writeFields), + } + } + + await this.apply(session, bytes) + const fields = await this.mergeChangedFields( + session, + base ? fieldsOf(base) : session.row.fields, + writeFields, + ) + const merged = this.materialize( + session, + readDocContent(session.doc), + fields, + ) + const inputs: CheckpointInput[] = sameRevisionContent( + own.snapshot, + merged, + ) + ? [ + { + frontiers: session.doc.frontiers(), + kind: 'legacy-save', + snapshot: merged, + }, + ] + : [ + own, + { + frontiers: session.doc.frontiers(), + kind: 'legacy-save', + snapshot: merged, + }, + ] + const { branch, revisions } = await this.appendCheckpoints( + session, + inputs, + ) + return { branch, revision: revisions[0] } + }) + } + + async restore( + branchId: EntityId | string, + revisionId: EntityId | string, + ): Promise { + return this.mutate(branchId, (session) => + this.restoreIn(session, revisionId), + ) + } + + async undo( + branchId: EntityId | string, + id: OpId, + before: OpId[], + ): Promise { + return this.mutate(branchId, async (session) => { + try { + session.doc.getChangeAt(id) + } catch { + this.reject(session.branch, 'unknown change') + } + await this.apply( + session, + revertChange(session.doc, id, before, `undo ${id.peer}@${id.counter}`), + ) + return this.checkpoint( + session, + 'undo', + this.materialize(session, readDocContent(session.doc)), + ) + }) + } + + async discard(branchId: EntityId | string): Promise { + return this.mutate(branchId, async (session) => { + if (session.document.publishedRevisionId) { + const { branch } = await this.restoreIn( + session, + session.document.publishedRevisionId, + ) + return branch + } + await this.drafts.archiveBranch(session.branch.id) + await this.fileReferences.removeReferencesForDocument( + session.branch.id, + FileReferenceType.Draft, + ) + return null + }) + } + + async history(branchId: EntityId | string): Promise { + const { doc, document } = await this.open(branchId) + const checkpoints = (await this.drafts.findRevisionsByDocument(document.id)) + .sort( + (left, right) => + right.createdAt.getTime() - left.createdAt.getTime() || + Number(BigInt(right.id) - BigInt(left.id)), + ) + .map((revision): CrdtCheckpoint => ({ + contentFormat: revision.contentFormat, + createdAt: revision.createdAt, + frontiers: revision.frontiers ?? null, + id: revision.id, + kind: revision.kind ?? null, + parentRevisionId: revision.parentRevisionId, + title: revision.title, + })) + const changes = [...doc.getAllChanges().values()] + .flat() + .sort((left, right) => right.lamport - left.lamport) + .map((change) => ({ + deps: change.deps, + id: { counter: change.counter + change.length - 1, peer: change.peer }, + lamport: change.lamport, + message: change.message, + timestamp: change.timestamp, + })) + return { changes, checkpoints } + } + + private async open( + branchId: EntityId | string, + locked = false, + ): Promise { + const requested = await this.drafts.findBranchById(branchId) + if (!requested) { + throw createAppException(AppErrorCode.DRAFT_NOT_FOUND, { + id: String(branchId), + }) + } + const document = await this.drafts.findDocumentById(requested.documentId) + if (!document) { + throw createAppException(AppErrorCode.DRAFT_NOT_FOUND, { + id: requested.id, + }) + } + return this.openDocument(document, locked) + } + + private async openDocument( + document: ContentDocumentRow, + locked = false, + ): Promise { + let row = await this.crdt.findDoc(document.id) + let branch = await this.ensureWorkingCopy(document, Boolean(row)) + if (!row) ({ branch, row } = await this.importDoc(document, branch)) + + let updates = await this.crdt.findUpdates(document.id) + // A compaction committed between the two reads has deleted rows our snapshot lacks. + for (;;) { + const current = await this.crdt.findDoc(document.id) + if (!current || sameBytes(current.snapshotVersion, row.snapshotVersion)) { + break + } + row = current + updates = await this.crdt.findUpdates(document.id) + } + const doc = createDocument() + doc.import(row.snapshot) + if (updates.length) doc.importBatch(updates.map((update) => update.bytes)) + const session: Session = { branch, doc, document, row } + + const head = await this.revision(branch.headRevisionId) + if (!head.frontiers && head.id !== row.syncedRevisionId) { + if (!locked) { + return this.crdt.withDocumentLock(document.id, () => + this.openDocument(document, true), + ) + } + await this.reconcile(session, head) + } + if (updates.length >= COMPACT_THRESHOLD) { + void this.compact( + session, + updates.map((update) => update.id), + ) + } + return session + } + + private mutate( + branchId: EntityId | string, + run: (session: Session) => Promise, + ): Promise { + return this.drafts.findBranchById(branchId).then((branch) => { + if (!branch) { + throw createAppException(AppErrorCode.DRAFT_NOT_FOUND, { + id: String(branchId), + }) + } + return this.crdt.withDocumentLock(branch.documentId, async () => + run(await this.open(branchId, true)), + ) + }) + } + + private async ensureWorkingCopy( + document: ContentDocumentRow, + imported: boolean, + ): Promise { + const active = await this.drafts.findBranchesByDocument(document.id) + if (active.length) { + const [keep, ...rest] = active + if (!imported) await this.assertLexical(keep) + for (const branch of rest) await this.retire(document, branch) + return keep + } + if (!document.publishedRevisionId) { + throw createAppException(AppErrorCode.DRAFT_NOT_FOUND, { + id: document.id, + }) + } + const published = await this.revision(document.publishedRevisionId) + const { branch } = await this.drafts.createBranch( + document.id, + published.id, + pickSnapshot(published), + true, + ) + if (!imported) await this.assertLexical(branch) + return branch + } + + private async assertLexical(branch: DraftBranchRow) { + const head = await this.revision(branch.headRevisionId) + if (head.contentFormat !== ContentFormat.Lexical) { + throw createAppException(AppErrorCode.DRAFT_CRDT_FORMAT_UNSUPPORTED, { + branchId: branch.id, + contentFormat: head.contentFormat, + }) + } + } + + private async retire(document: ContentDocumentRow, branch: DraftBranchRow) { + await this.drafts.archiveBranch(branch.id) + const share = await this.drafts.findShareByDocument(document.id) + if (share?.mode === 'follow' && share.draftId === branch.id) { + await this.drafts.upsertShare(document.id, share.token, { + mode: 'pinned', + revisionId: branch.headRevisionId, + }) + } + await this.fileReferences.removeReferencesForDocument( + branch.id, + FileReferenceType.Draft, + ) + } + + private async importDoc( + document: ContentDocumentRow, + branch: DraftBranchRow, + ): Promise<{ branch: DraftBranchRow; row: CrdtDocRow }> { + const head = await this.revision(branch.headRevisionId) + const doc = createContentDoc( + { content: head.content ?? EMPTY_STATE, title: head.title }, + `import ${head.id}`, + ) + const snapshot = doc.export({ mode: 'snapshot' }) + const fields = fieldsOf(head) + const row = await this.crdt.createDoc({ + documentId: document.id, + fields, + snapshot, + snapshotVersion: doc.oplogVersion().encode(), + syncedRevisionId: head.id, + }) + if (!sameBytes(row.snapshot, snapshot)) return { branch, row } + + const { branch: moved, revisions } = await this.crdt.appendCheckpoints( + branch.id, + [ + { + frontiers: doc.frontiers(), + kind: 'import', + snapshot: this.materializeFor( + document.refType, + readDocContent(doc), + fields, + ), + }, + ], + ) + await this.crdt.setSyncedRevision(document.id, revisions[0].id) + return { + branch: moved, + row: { ...row, syncedRevisionId: revisions[0].id }, + } + } + + private async reconcile(session: Session, head: ContentRevisionRow) { + if (head.contentFormat !== ContentFormat.Lexical) { + this.logger.warn( + `Skipping non-Lexical legacy head ${head.id} on CRDT document ${session.document.id}`, + ) + return + } + const synced = session.row.syncedRevisionId + ? await this.drafts.findRevisionById(session.row.syncedRevisionId) + : null + await this.apply( + session, + editDocAt( + session.doc, + this.knownFrontiers(session.doc, synced?.frontiers) ?? + session.doc.frontiers(), + { content: head.content ?? EMPTY_STATE, title: head.title }, + `legacy-save ${head.id}`, + ), + ) + const fields = await this.mergeChangedFields( + session, + synced ? fieldsOf(synced) : session.row.fields, + fieldsOf(head), + ) + const { branch } = await this.appendCheckpoints(session, [ + { + frontiers: session.doc.frontiers(), + kind: 'legacy-save', + snapshot: this.materialize( + session, + readDocContent(session.doc), + fields, + ), + }, + ]) + session.branch = branch + } + + private async restoreIn( + session: Session, + revisionId: EntityId | string, + ): Promise { + const revision = await this.revision(revisionId, session.document.id) + if (revision.contentFormat !== ContentFormat.Lexical) { + throw createAppException(AppErrorCode.DRAFT_CRDT_FORMAT_UNSUPPORTED, { + branchId: session.branch.id, + contentFormat: revision.contentFormat, + }) + } + await this.apply( + session, + editDocAt( + session.doc, + session.doc.frontiers(), + { content: revision.content ?? EMPTY_STATE, title: revision.title }, + `restore ${revision.id}`, + ), + ) + const fields = fieldsOf(revision) + await this.crdt.replaceFields(session.document.id, fields) + session.row = { ...session.row, fields } + return this.checkpoint( + session, + 'restore', + this.materialize(session, readDocContent(session.doc), fields), + ) + } + + private async apply(session: Session, bytes: Uint8Array) { + await this.crdt.appendUpdate(session.document.id, bytes) + session.doc.import(bytes) + } + + private async checkpoint( + session: Session, + kind: CheckpointKind, + snapshot: RevisionSnapshot, + ): Promise { + const { branch, revisions } = await this.appendCheckpoints(session, [ + { frontiers: session.doc.frontiers(), kind, snapshot }, + ]) + return { branch, revision: revisions[0] } + } + + private async appendCheckpoints(session: Session, inputs: CheckpointInput[]) { + const result = await this.crdt.appendCheckpoints(session.branch.id, inputs) + const head = result.revisions.at(-1)! + await this.crdt.setSyncedRevision(session.document.id, head.id) + session.row = { ...session.row, syncedRevisionId: head.id } + await this.fileReferences.updateReferencesForDocument( + head, + result.branch.id, + FileReferenceType.Draft, + ) + return result + } + + private async mergeChangedFields( + session: Session, + base: CrdtFields | null, + next: CrdtFields, + ): Promise { + const patch: CrdtFieldsPatch = { + meta: changedKeys(base?.meta ?? null, next.meta), + typeSpecificData: changedKeys( + base?.typeSpecificData ?? null, + next.typeSpecificData, + ), + } + const current = session.row.fields ?? { meta: null, typeSpecificData: null } + if (!patch.meta && !patch.typeSpecificData) return current + const fields = await this.crdt.mergeFields(session.document.id, patch) + session.row = { ...session.row, fields } + return fields + } + + private async compact(session: Session, foldedIds: EntityId[]) { + try { + await this.crdt.compact(session.document.id, { + expectedVersion: session.row.snapshotVersion, + foldedIds, + snapshot: session.doc.export({ mode: 'snapshot' }), + snapshotVersion: session.doc.oplogVersion().encode(), + }) + } catch (error) { + this.logger.warn( + `Compaction of ${session.document.id} failed: ${(error as Error).message}`, + ) + } + } + + // A legacy revision has no version of its own, but the import checkpoint made from it does. + private async importedFrontiers(revision: ContentRevisionRow) { + const checkpoints = await this.drafts.findRevisionsByDocument( + revision.documentId, + ) + return ( + checkpoints.find( + (row) => row.kind === 'import' && row.parentRevisionId === revision.id, + )?.frontiers ?? null + ) + } + + private knownFrontiers( + doc: LoroDoc, + frontiers: CheckpointInput['frontiers'] | undefined, + ) { + if (!frontiers?.length) return null + try { + doc.frontiersToVV(frontiers) + return frontiers + } catch { + return null + } + } + + private materialize( + session: Session, + content: DocContent, + fields: CrdtFields | null = session.row.fields, + ): RevisionSnapshot { + return this.materializeFor(session.document.refType, content, fields) + } + + private materializeFor( + refType: DraftRefType, + content: DocContent, + fields: CrdtFields | null, + ): RevisionSnapshot { + return canonicalSnapshot(refType, { + content: content.content, + contentFormat: ContentFormat.Lexical, + images: null, + meta: fields?.meta ?? null, + text: mxLexicalToMarkdown(content.content), + title: content.title, + typeSpecificData: fields?.typeSpecificData ?? null, + }) + } + + private async revision( + id: EntityId | string, + documentId?: EntityId, + ): Promise { + const revision = await this.drafts.findRevisionById(id) + if (!revision || (documentId && revision.documentId !== documentId)) { + throw createAppException(AppErrorCode.CONTENT_REVISION_NOT_FOUND, { + id: String(id), + }) + } + return revision + } + + private reject(branch: DraftBranchRow, reason: string): never { + throw createAppException(AppErrorCode.DRAFT_CRDT_UPDATE_REJECTED, { + branchId: branch.id, + reason, + }) + } +} diff --git a/apps/core/src/modules/draft/draft.controller.ts b/apps/core/src/modules/draft/draft.controller.ts index 3f5f77cdf35..528bebfc6fa 100644 --- a/apps/core/src/modules/draft/draft.controller.ts +++ b/apps/core/src/modules/draft/draft.controller.ts @@ -1,7 +1,23 @@ -import { Body, Delete, Get, Param, Post, Put, Query } from '@nestjs/common' +import { + Body, + Delete, + Get, + HttpCode, + Param, + Patch, + Post, + Put, + Query, + Req, + Res, +} from '@nestjs/common' +import type { FastifyReply, FastifyRequest } from 'fastify' +import type { OpId } from 'loro-crdt' import { ApiController } from '~/common/decorators/api-controller.decorator' import { Auth } from '~/common/decorators/auth.decorator' +import { WithFastifyRouteOptions } from '~/common/decorators/fastify-route-options.decorator' +import { HTTPDecorators } from '~/common/decorators/http.decorator' import { AppErrorCode, createAppException } from '~/common/errors' import { withMeta } from '~/common/response/envelope.types' import { MetaObjectBuilder } from '~/common/response/meta-builder' @@ -11,16 +27,26 @@ import { DraftRefType } from './draft.enum' import { type CreateDraftDto, CreateDraftSchema, + type DraftCheckpointDto, + DraftCheckpointSchema, + type DraftCrdtQueryDto, + DraftCrdtQuerySchema, type DraftDocumentIdDto, DraftDocumentIdSchema, + type DraftFieldsDto, + DraftFieldsSchema, type DraftPagerDto, DraftPagerSchema, type DraftRefTypeAndIdDto, DraftRefTypeAndIdSchema, type DraftRefTypeDto, DraftRefTypeSchema, + type DraftRestoreDto, + DraftRestoreSchema, type DraftShareTokenDto, DraftShareTokenSchema, + type DraftUndoDto, + DraftUndoSchema, type RevisionComparisonDto, RevisionComparisonSchema, type SetDraftShareDto, @@ -31,6 +57,8 @@ import { import { DraftService } from './draft.service' import { DraftViews } from './draft.views' +const CRDT_UPDATE_BODY_LIMIT = 4 * 1024 * 1024 + @ApiController('drafts') export class DraftController { constructor(private readonly draftService: DraftService) {} @@ -131,7 +159,7 @@ export class DraftController { @Get('/:id') @Auth() async getById(@Param({ schema: EntityIdSchema }) params: EntityIdDto) { - const draft = await this.draftService.findById(params.id) + const draft = await this.draftService.findForEditing(params.id) if (!draft) { throw createAppException(AppErrorCode.DRAFT_NOT_FOUND, { id: params.id }) } @@ -144,6 +172,87 @@ export class DraftController { return this.draftService.getBranchRevisions(params.id) } + @Get('/:id/crdt') + @Auth() + @HTTPDecorators.RawResponse + async exportCrdt( + @Param({ schema: EntityIdSchema }) params: EntityIdDto, + @Query({ schema: DraftCrdtQuerySchema }) query: DraftCrdtQueryDto, + @Res({ passthrough: true }) reply: FastifyReply, + ) { + const bytes = await this.draftService.exportCrdt( + params.id, + query.since + ? new Uint8Array(Buffer.from(query.since, 'base64')) + : undefined, + ) + reply.type('application/octet-stream') + return Buffer.from(bytes) + } + + @Post('/:id/crdt') + @Auth() + @HttpCode(204) + @WithFastifyRouteOptions({ bodyLimit: CRDT_UPDATE_BODY_LIMIT }) + async importCrdt( + @Param({ schema: EntityIdSchema }) params: EntityIdDto, + @Req() request: FastifyRequest, + ) { + if (!Buffer.isBuffer(request.body)) { + throw createAppException(AppErrorCode.DRAFT_CRDT_UPDATE_REJECTED, { + branchId: params.id, + reason: 'expected an application/octet-stream body', + }) + } + await this.draftService.importCrdt(params.id, new Uint8Array(request.body)) + } + + @Post('/:id/checkpoints') + @Auth() + createCheckpoint( + @Param({ schema: EntityIdSchema }) params: EntityIdDto, + @Body({ schema: DraftCheckpointSchema }) body: DraftCheckpointDto, + ) { + return this.draftService.createCheckpoint(params.id, body.kind) + } + + @Patch('/:id/fields') + @Auth() + updateFields( + @Param({ schema: EntityIdSchema }) params: EntityIdDto, + @Body({ schema: DraftFieldsSchema }) body: DraftFieldsDto, + ) { + return this.draftService.updateFields(params.id, body) + } + + @Get('/:id/history') + @Auth() + history(@Param({ schema: EntityIdSchema }) params: EntityIdDto) { + return this.draftService.history(params.id) + } + + @Post('/:id/restore') + @Auth() + restore( + @Param({ schema: EntityIdSchema }) params: EntityIdDto, + @Body({ schema: DraftRestoreSchema }) body: DraftRestoreDto, + ) { + return this.draftService.restore(params.id, body.revisionId) + } + + @Post('/:id/undo') + @Auth() + undo( + @Param({ schema: EntityIdSchema }) params: EntityIdDto, + @Body({ schema: DraftUndoSchema }) body: DraftUndoDto, + ) { + return this.draftService.undo( + params.id, + body.id as OpId, + body.before as OpId[], + ) + } + @Put('/:id') @Auth() update( diff --git a/apps/core/src/modules/draft/draft.module.ts b/apps/core/src/modules/draft/draft.module.ts index 98a08e28e1f..4e32c9a0b89 100644 --- a/apps/core/src/modules/draft/draft.module.ts +++ b/apps/core/src/modules/draft/draft.module.ts @@ -5,11 +5,15 @@ import { DRAFT_SERVICE_TOKEN } from '~/constants/injection.constant' import { DraftController } from './draft.controller' import { DraftRepository } from './draft.repository' import { DraftService } from './draft.service' +import { DraftCrdtRepository } from './draft-crdt.repository' +import { DraftCrdtService } from './draft-crdt.service' @Module({ controllers: [DraftController], providers: [ DraftRepository, + DraftCrdtRepository, + DraftCrdtService, DraftService, { provide: DRAFT_SERVICE_TOKEN, useExisting: DraftService }, ], diff --git a/apps/core/src/modules/draft/draft.repository.ts b/apps/core/src/modules/draft/draft.repository.ts index bfa00f9b82e..5da763831dc 100644 --- a/apps/core/src/modules/draft/draft.repository.ts +++ b/apps/core/src/modules/draft/draft.repository.ts @@ -30,6 +30,7 @@ import { type EntityId, parseEntityId } from '~/shared/id/entity-id' import { SnowflakeService } from '~/shared/id/snowflake.service' import type { + CheckpointKind, ContentDocumentRow, ContentPublicationEventRow, ContentRevisionRow, @@ -53,12 +54,13 @@ const mapDocument = ( refType: row.refType as DraftRefType, }) -const mapRevision = ( +export const mapRevision = ( row: typeof contentRevisions.$inferSelect, ): ContentRevisionRow => ({ ...row, documentId: toEntityId(row.documentId)!, id: toEntityId(row.id)!, + kind: row.kind as CheckpointKind | null, parentRevisionId: toEntityId(row.parentRevisionId), }) @@ -83,7 +85,7 @@ const mapShare = ( revisionId: toEntityId(row.revisionId), }) -const mapBranch = (row: typeof drafts.$inferSelect): DraftBranchRow => ({ +export const mapBranch = (row: typeof drafts.$inferSelect): DraftBranchRow => ({ ...row, baseRevisionId: toEntityId(row.baseRevisionId)!, documentId: toEntityId(row.documentId)!, diff --git a/apps/core/src/modules/draft/draft.schema.ts b/apps/core/src/modules/draft/draft.schema.ts index a27c7d74f14..ca48b17c934 100644 --- a/apps/core/src/modules/draft/draft.schema.ts +++ b/apps/core/src/modules/draft/draft.schema.ts @@ -96,3 +96,44 @@ export const SetDraftShareSchema = z.discriminatedUnion('mode', [ ]) export type SetDraftShareDto = z.infer + +export const DraftCrdtQuerySchema = z.object({ + since: z.string().max(65_536).optional(), +}) + +export type DraftCrdtQueryDto = z.infer + +export const DraftCheckpointSchema = z.object({ + kind: z.enum(['save', 'publish']), +}) + +export type DraftCheckpointDto = z.infer + +export const DraftFieldsSchema = z + .object({ + meta: z.record(z.string(), z.unknown()).optional(), + typeSpecificData: z.record(z.string(), z.unknown()).optional(), + }) + .refine((value) => value.meta || value.typeSpecificData, { + message: 'Nothing to update', + }) + +export type DraftFieldsDto = z.infer + +export const DraftRestoreSchema = z.object({ + revisionId: zEntityId, +}) + +export type DraftRestoreDto = z.infer + +const OpIdSchema = z.object({ + counter: z.number().int().nonnegative(), + peer: z.string().regex(/^\d+$/), +}) + +export const DraftUndoSchema = z.object({ + before: z.array(OpIdSchema).max(64), + id: OpIdSchema, +}) + +export type DraftUndoDto = z.infer diff --git a/apps/core/src/modules/draft/draft.service.ts b/apps/core/src/modules/draft/draft.service.ts index a47cb862937..c9676b0acb2 100644 --- a/apps/core/src/modules/draft/draft.service.ts +++ b/apps/core/src/modules/draft/draft.service.ts @@ -1,6 +1,7 @@ import { randomBytes } from 'node:crypto' import { Injectable } from '@nestjs/common' +import type { OpId } from 'loro-crdt' import { AppErrorCode, createAppException } from '~/common/errors' import { BusinessEvents, EventScope } from '~/constants/business-event.constant' @@ -14,6 +15,7 @@ import { DraftRefType } from './draft.enum' import { DraftRepository } from './draft.repository' import type { CreateDraftDto, + DraftCheckpointDto, SetDraftShareDto, UpdateDraftDto, } from './draft.schema' @@ -21,6 +23,7 @@ import type { ContentDocumentRow, ContentPublicationEventRow, ContentRevisionRow, + CrdtFieldsPatch, DocumentShareRow, DraftBranchRow, DraftBranchView, @@ -33,6 +36,7 @@ import type { VersionTreeNode, } from './draft.types' import { canonicalSnapshot, sameRevisionContent } from './draft-content' +import { type CheckpointResult, DraftCrdtService } from './draft-crdt.service' const projectVersionTree = ( document: ContentDocumentRow, @@ -142,6 +146,7 @@ export class DraftService { private readonly fileReferenceService: FileReferenceService, private readonly databaseService: DatabaseService, private readonly eventManager: EventManagerService, + private readonly crdtService: DraftCrdtService, ) {} get repository() { @@ -167,6 +172,7 @@ export class DraftService { refId: string, ): Promise { const document = await this.ensurePublishedDocument(refType, refId) + await this.crdtService.workingCopyOfDocument(document.id) return this.contextForDocument(document) } @@ -229,6 +235,17 @@ export class DraftService { base = created.revision } + if (dto.baseRevisionId || dto.refId) { + const workingCopy = await this.crdtService.workingCopyOfDocument( + document.id, + ) + if (workingCopy) { + return this.legacyView( + await this.crdtService.legacyWrite(workingCopy.id, base.id, snapshot), + ) + } + } + const created = await this.draftRepository.createBranch( document.id, base.id, @@ -257,6 +274,15 @@ export class DraftService { throw createAppException(AppErrorCode.DRAFT_NOT_FOUND, { id }) } const snapshot = canonicalSnapshot(document.refType, dto.data) + if (await this.crdtService.usesCrdt(id)) { + return this.legacyView( + await this.crdtService.legacyWrite( + id, + dto.expectedHeadRevisionId, + snapshot, + ), + ) + } const result = await this.draftRepository.saveBranch( id, dto.expectedHeadRevisionId, @@ -288,24 +314,108 @@ export class DraftService { * as a failed request — the client still gets the new head in the response. */ private broadcastUpdate(view: DraftBranchView) { + this.broadcast(view, view.document) + } + + private broadcast(branch: DraftBranchRow, document: ContentDocumentRow) { void this.eventManager.emit( BusinessEvents.DRAFT_UPDATE, { - branchId: view.id, - documentId: view.documentId, - headRevisionId: view.headRevisionId, - refId: view.document.refId, - refType: view.document.refType, + branchId: branch.id, + documentId: branch.documentId, + headRevisionId: branch.headRevisionId, + refId: document.refId, + refType: document.refType, } satisfies DraftUpdateEventPayload, { scope: EventScope.TO_ADMIN }, ) } async findById(id: string): Promise { + if (await this.crdtService.usesCrdt(id)) { + return this.hydrate(await this.crdtService.workingCopy(id)) + } const branch = await this.draftRepository.findBranchById(id) return branch ? this.hydrate(branch) : null } + async mergeArticleFields( + refType: DraftRefType, + refId: string, + typeSpecificData: Record, + ): Promise { + if (!Object.keys(typeSpecificData).length) return + const document = await this.draftRepository.findDocumentByRef( + refType, + refId, + ) + if (document) { + await this.crdtService.mergeDocumentFields(document.id, typeSpecificData) + } + } + + async findForEditing(id: string): Promise { + if (await this.crdtService.usesCrdt(id)) { + return this.createCheckpoint(id, 'save') + } + return this.findById(id) + } + + usesCrdt(id: string): Promise { + return this.crdtService.usesCrdt(id) + } + + exportCrdt(id: string, since?: Uint8Array): Promise { + return this.crdtService.exportUpdates(id, since) + } + + async importCrdt(id: string, bytes: Uint8Array): Promise { + const { branch, document } = await this.crdtService.importUpdate(id, bytes) + this.broadcast(branch, document) + } + + async createCheckpoint( + id: string, + kind: DraftCheckpointDto['kind'], + ): Promise { + return this.checkpointView( + await this.crdtService.createCheckpoint(id, kind), + ) + } + + updateFields(id: string, patch: CrdtFieldsPatch) { + return this.crdtService.updateFields(id, patch) + } + + history(id: string) { + return this.crdtService.history(id) + } + + async restore(id: string, revisionId: string): Promise { + return this.checkpointView(await this.crdtService.restore(id, revisionId)) + } + + async undo(id: string, change: OpId, before: OpId[]) { + return this.checkpointView(await this.crdtService.undo(id, change, before)) + } + + private async checkpointView( + result: CheckpointResult, + ): Promise { + const view = await this.hydrate(result.branch) + this.broadcastUpdate(view) + return view + } + + private async legacyView(result: CheckpointResult): Promise { + const view = await this.checkpointView(result) + return { + ...view, + headRevision: result.revision, + headRevisionId: result.revision.id, + } + } + async findRevisionById(id: string): Promise { const revision = await this.draftRepository.findRevisionById(id) if (!revision) { @@ -460,6 +570,11 @@ export class DraftService { } async delete(id: string): Promise { + if (await this.crdtService.usesCrdt(id)) { + const branch = await this.crdtService.discard(id) + if (branch) this.broadcastUpdate(await this.hydrate(branch)) + return + } const branch = await this.draftRepository.archiveBranch(id) if (!branch) { throw createAppException(AppErrorCode.DRAFT_NOT_FOUND, { id }) diff --git a/apps/core/src/modules/draft/draft.types.ts b/apps/core/src/modules/draft/draft.types.ts index 997cf4cf240..365525b7e34 100644 --- a/apps/core/src/modules/draft/draft.types.ts +++ b/apps/core/src/modules/draft/draft.types.ts @@ -29,13 +29,45 @@ export interface ContentDocumentRow { updatedAt: Date | null } +export type Frontiers = { counter: number; peer: `${number}` }[] + +export type CheckpointKind = + 'save' | 'publish' | 'restore' | 'undo' | 'import' | 'legacy-save' + export interface ContentRevisionRow extends RevisionSnapshot { createdAt: Date documentId: EntityId + frontiers?: Frontiers | null id: EntityId + kind?: CheckpointKind | null parentRevisionId: EntityId | null } +export interface CrdtFields { + meta: Record | null + typeSpecificData: Record | null +} + +export type CrdtFieldsPatch = { + [K in keyof CrdtFields]?: Record +} + +export interface CrdtDocRow { + createdAt: Date + documentId: EntityId + fields: CrdtFields | null + snapshot: Uint8Array + snapshotVersion: Uint8Array + syncedRevisionId: EntityId | null + updatedAt: Date | null +} + +export interface CheckpointInput { + frontiers: Frontiers | null + kind: CheckpointKind + snapshot: RevisionSnapshot +} + export interface ContentPublicationEventRow { createdAt: Date documentId: EntityId diff --git a/apps/core/src/modules/note/note.controller.ts b/apps/core/src/modules/note/note.controller.ts index 1b64682defb..e65c82cc6a9 100644 --- a/apps/core/src/modules/note/note.controller.ts +++ b/apps/core/src/modules/note/note.controller.ts @@ -592,9 +592,9 @@ export class NoteController { @Body({ schema: PartialNoteSchema }) body: PartialNoteDto, @Param({ schema: EntityIdSchema }) params: EntityIdDto, ) { - await this.noteService.updateById( + await this.noteService.patchById( params.id, - body as unknown as Partial, + body as Parameters[1], ) } diff --git a/apps/core/src/modules/note/note.service.ts b/apps/core/src/modules/note/note.service.ts index 189c816ac47..aea095def74 100644 --- a/apps/core/src/modules/note/note.service.ts +++ b/apps/core/src/modules/note/note.service.ts @@ -353,6 +353,23 @@ export class NoteService { return note } + public async patchById( + id: string, + patch: Partial< + Pick + >, + ) { + const updated = await this.updateById(id, patch) + await this.draftService.mergeArticleFields( + DraftRefType.Note, + id, + Object.fromEntries( + Object.entries(patch).filter(([, value]) => value !== undefined), + ), + ) + return updated + } + public async updateById( id: string, data: Partial & { diff --git a/apps/core/src/modules/post/post.controller.ts b/apps/core/src/modules/post/post.controller.ts index 76e58d38187..688d7c509ff 100644 --- a/apps/core/src/modules/post/post.controller.ts +++ b/apps/core/src/modules/post/post.controller.ts @@ -57,7 +57,6 @@ import { SetPostPublishStatusSchema, } from './post.schema' import { PostService } from './post.service' -import type { PostModel } from './post.types' import { PostMetaBuilder } from './post-meta-builder' import { readPaywallMeta } from './post-paywall.util' @@ -560,10 +559,7 @@ export class PostController { @Param({ schema: EntityIdSchema }) params: EntityIdDto, @Body({ schema: PartialPostSchema }) body: PartialPostDto, ) { - await this.postService.updateById( - params.id, - body as unknown as Partial, - ) + await this.postService.patchById(params.id, body) } @Delete('/:id') diff --git a/apps/core/src/modules/post/post.service.ts b/apps/core/src/modules/post/post.service.ts index 42c2ce82b8e..1730995e6b2 100644 --- a/apps/core/src/modules/post/post.service.ts +++ b/apps/core/src/modules/post/post.service.ts @@ -347,6 +347,19 @@ export class PostService implements OnApplicationBootstrap { return trackedPost } + async patchById( + id: string, + patch: { categoryId?: string; pinAt?: Date | null }, + ) { + await this.updateById(id, patch as Partial) + await this.draftService.mergeArticleFields(DraftRefType.Post, id, { + ...(patch.categoryId === undefined + ? {} + : { categoryId: patch.categoryId }), + ...(patch.pinAt === undefined ? {} : { pin: Boolean(patch.pinAt) }), + }) + } + async updateById( id: string, data: Partial & { diff --git a/apps/core/src/modules/publish/publish.service.ts b/apps/core/src/modules/publish/publish.service.ts index 787f5c498d3..47ab58d8ab4 100644 --- a/apps/core/src/modules/publish/publish.service.ts +++ b/apps/core/src/modules/publish/publish.service.ts @@ -79,7 +79,17 @@ export class PublishService implements OnModuleInit { id: dto.branchId, }) } - if (branch.headRevisionId !== dto.revisionId) { + const workingCopy = await this.drafts.usesCrdt(branch.id) + let revision = branch.headRevision + if (workingCopy) { + const selected = await this.drafts.findRevisionById(dto.revisionId) + if (!selected || selected.documentId !== branch.documentId) { + throw createAppException(AppErrorCode.CONTENT_REVISION_NOT_FOUND, { + id: dto.revisionId, + }) + } + revision = selected + } else if (branch.headRevisionId !== dto.revisionId) { throw createAppException(AppErrorCode.DRAFT_HEAD_CONFLICT, { actualHeadRevisionId: branch.headRevisionId, branchId: branch.id, @@ -95,6 +105,7 @@ export class PublishService implements OnModuleInit { }) } if ( + !workingCopy && branch.relationToPublished && ['descendant', 'diverged'].includes(branch.relationToPublished) && !dto.confirmDiverged @@ -114,12 +125,13 @@ export class PublishService implements OnModuleInit { let migration: PublishTaskPayload['migration'] if ( existing?.contentFormat === ContentFormat.Markdown && - branch.headRevision.contentFormat === ContentFormat.Lexical + revision.contentFormat === ContentFormat.Lexical ) { const result = await this.contentMigrations.dryRunMarkdownToLexical({ branchId: branch.id, profile: 'yohaku-v1', refId: existing.id, + revisionId: revision.id, refType: branch.document.refType, sourceText: existing.text ?? '', }) @@ -160,8 +172,8 @@ export class PublishService implements OnModuleInit { : 'republish', refId: branch.document.refId, refType: branch.document.refType, - revisionId: branch.headRevision.id, - snapshot: structuredClone(branch.headRevision), + revisionId: revision.id, + snapshot: structuredClone(revision), } return this.tasks.createTask({ diff --git a/apps/core/test/src/contracts/admin/drafts-admin.contract.spec.ts b/apps/core/test/src/contracts/admin/drafts-admin.contract.spec.ts index 39378ef54d0..d7067139a8d 100644 --- a/apps/core/test/src/contracts/admin/drafts-admin.contract.spec.ts +++ b/apps/core/test/src/contracts/admin/drafts-admin.contract.spec.ts @@ -46,6 +46,7 @@ describe('DraftController Admin branch contract (e2e)', () => { provide: DraftService, useValue: { findById: async () => fixture, + findForEditing: async () => fixture, list: async () => ({ data: [fixture], pagination: { currentPage: 1, size: 10, total: 1, totalPage: 1 }, diff --git a/apps/core/test/src/contracts/admin/drafts-crdt-admin.contract.spec.ts b/apps/core/test/src/contracts/admin/drafts-crdt-admin.contract.spec.ts new file mode 100644 index 00000000000..0f80aa0fd42 --- /dev/null +++ b/apps/core/test/src/contracts/admin/drafts-crdt-admin.contract.spec.ts @@ -0,0 +1,146 @@ +import { describe, expect, test, vi } from 'vitest' + +import { apiRoutePrefix } from '~/common/decorators/api-controller.decorator' +import { DraftController } from '~/modules/draft/draft.controller' +import { DraftService } from '~/modules/draft/draft.service' + +import { createE2EApp } from '../../../helper/create-e2e-app' +import { authPassHeader } from '../../../mock/guard/auth.guard' + +const id = '7000000000000000010' +const view = { headRevisionId: '7000000000000000011', id } + +const service = { + createCheckpoint: vi.fn(async () => view), + exportCrdt: vi.fn(async () => new Uint8Array([1, 2, 3])), + history: vi.fn(async () => ({ + changes: [ + { + deps: [], + id: { counter: 4, peer: '99' }, + lamport: 4, + message: 'human:owner', + timestamp: 1, + }, + ], + checkpoints: [{ id: view.headRevisionId, kind: 'save' }], + })), + importCrdt: vi.fn(async () => undefined), + restore: vi.fn(async () => view), + undo: vi.fn(async () => view), + updateFields: vi.fn(async () => ({ + meta: null, + typeSpecificData: { categoryId: 'c' }, + })), +} + +describe('DraftController CRDT endpoints (e2e)', () => { + const proxy = createE2EApp({ + controllers: [DraftController], + providers: [{ provide: DraftService, useValue: service }], + }) + + test('GET /drafts/:id/crdt returns raw bytes and decodes ?since', async () => { + const since = Buffer.from([7, 8]).toString('base64') + const res = await proxy.app.inject({ + headers: authPassHeader, + method: 'GET', + url: `${apiRoutePrefix}/drafts/${id}/crdt?since=${encodeURIComponent(since)}`, + }) + + expect(res.statusCode).toBe(200) + expect(res.headers['content-type']).toContain('application/octet-stream') + expect([...res.rawPayload]).toEqual([1, 2, 3]) + expect([ + ...((service.exportCrdt.mock.calls[0] as unknown[])[1] as Uint8Array), + ]).toEqual([7, 8]) + }) + + test('POST /drafts/:id/crdt imports an octet-stream body', async () => { + const res = await proxy.app.inject({ + headers: { + ...authPassHeader, + 'content-type': 'application/octet-stream', + }, + method: 'POST', + payload: Buffer.from([5, 6]), + url: `${apiRoutePrefix}/drafts/${id}/crdt`, + }) + + expect(res.statusCode).toBe(204) + const [calledId, bytes] = service.importCrdt.mock.calls[0] as unknown as [ + string, + Uint8Array, + ] + expect(calledId).toBe(id) + expect([...bytes]).toEqual([5, 6]) + }) + + test('POST /drafts/:id/checkpoints creates a checkpoint of a kind', async () => { + const res = await proxy.app.inject({ + headers: authPassHeader, + method: 'POST', + payload: { kind: 'publish' }, + url: `${apiRoutePrefix}/drafts/${id}/checkpoints`, + }) + + expect(res.statusCode).toBe(201) + expect(res.json().data).toMatchObject({ + head_revision_id: view.headRevisionId, + }) + expect(service.createCheckpoint).toHaveBeenCalledWith(id, 'publish') + }) + + test('PATCH /drafts/:id/fields keeps nested keys verbatim', async () => { + const res = await proxy.app.inject({ + headers: authPassHeader, + method: 'PATCH', + payload: { typeSpecificData: { categoryId: 'c' } }, + url: `${apiRoutePrefix}/drafts/${id}/fields`, + }) + + expect(res.statusCode).toBe(200) + expect(service.updateFields).toHaveBeenCalledWith(id, { + typeSpecificData: { categoryId: 'c' }, + }) + }) + + test('GET /drafts/:id/history lists checkpoints and changes', async () => { + const res = await proxy.app.inject({ + headers: authPassHeader, + method: 'GET', + url: `${apiRoutePrefix}/drafts/${id}/history`, + }) + + expect(res.statusCode).toBe(200) + expect(res.json().data.changes[0]).toMatchObject({ + id: { counter: 4, peer: '99' }, + message: 'human:owner', + }) + }) + + test('POST /drafts/:id/restore and /undo apply as new commits', async () => { + const restore = await proxy.app.inject({ + headers: authPassHeader, + method: 'POST', + payload: { revisionId: view.headRevisionId }, + url: `${apiRoutePrefix}/drafts/${id}/restore`, + }) + const undo = await proxy.app.inject({ + headers: authPassHeader, + method: 'POST', + payload: { + before: [{ counter: 1, peer: '99' }], + id: { counter: 4, peer: '99' }, + }, + url: `${apiRoutePrefix}/drafts/${id}/undo`, + }) + + expect(restore.statusCode).toBe(201) + expect(undo.statusCode).toBe(201) + expect(service.restore).toHaveBeenCalledWith(id, view.headRevisionId) + expect(service.undo).toHaveBeenCalledWith(id, { counter: 4, peer: '99' }, [ + { counter: 1, peer: '99' }, + ]) + }) +}) diff --git a/apps/core/test/src/contracts/draft.contract.spec.ts b/apps/core/test/src/contracts/draft.contract.spec.ts index 87d33d55c4d..e047982999f 100644 --- a/apps/core/test/src/contracts/draft.contract.spec.ts +++ b/apps/core/test/src/contracts/draft.contract.spec.ts @@ -59,6 +59,7 @@ const provider = { right: revision(), }), findById: async () => branch, + findForEditing: async () => branch, findNewDrafts: async () => [branch], findRevisionById: async () => revision(), getBranchRevisions: async () => [revision()], diff --git a/apps/core/test/src/modules/content-migration/content-migration-commit.service.spec.ts b/apps/core/test/src/modules/content-migration/content-migration-commit.service.spec.ts index 22b39911fcd..e96d17ddcda 100644 --- a/apps/core/test/src/modules/content-migration/content-migration-commit.service.spec.ts +++ b/apps/core/test/src/modules/content-migration/content-migration-commit.service.spec.ts @@ -270,4 +270,41 @@ describe('ContentMigrationCommitService', () => { ) expect(database.state).toEqual(before) }) + + it('accepts a checkpoint precondition after the working copy head moved on', async () => { + const rows = branchRows() + const database = createTransactionalDatabase({ + source: sourceRow(), + ...rows, + branch: { ...rows.branch, headRevisionId: '501' }, + revision: { ...rows.revision, kind: 'publish' }, + translations: [], + }) + const service = new ContentMigrationCommitService( + database as never, + new LexicalService(), + ) + + await service.commitMarkdownToLexical(buildCommitInput([])) + + expect(database.state.source.contentFormat).toBe(ContentFormat.Lexical) + }) + + it('still rejects a moved head when the precondition revision is not a checkpoint', async () => { + const rows = branchRows() + const database = createTransactionalDatabase({ + source: sourceRow(), + ...rows, + branch: { ...rows.branch, headRevisionId: '501' }, + translations: [], + }) + const service = new ContentMigrationCommitService( + database as never, + new LexicalService(), + ) + + await expect( + service.commitMarkdownToLexical(buildCommitInput([])), + ).rejects.toBeInstanceOf(ConflictException) + }) }) diff --git a/apps/core/test/src/modules/content-migration/content-migration.service.spec.ts b/apps/core/test/src/modules/content-migration/content-migration.service.spec.ts index 5539ea84abf..36718a1a536 100644 --- a/apps/core/test/src/modules/content-migration/content-migration.service.spec.ts +++ b/apps/core/test/src/modules/content-migration/content-migration.service.spec.ts @@ -33,6 +33,7 @@ function createService( options: { source?: Record | null draft?: Record | null + revisions?: Record> translations?: Record[] } = {}, ) { @@ -53,6 +54,7 @@ function createService( const pageService = { findById: vi.fn() } const draftService = { findById: vi.fn().mockResolvedValue(options.draft ?? null), + findRevisionById: vi.fn(async (id: string) => options.revisions?.[id]), } const translationRepository = { listByRefId: vi @@ -208,4 +210,36 @@ describe('ContentMigrationService', () => { service.dryRunMarkdownToLexical({ ...request, branchId: '300' }), ).rejects.toBeInstanceOf(BadRequestException) }) + + it('pins the branch precondition to the checkpoint being published', async () => { + const lexicalContent = JSON.stringify({ root: { children: [] } }) + const { service } = createService({ + draft: { + document: { id: '400', refId: '100', refType: DraftRefType.Post }, + documentId: '400', + headRevision: { content: null, contentFormat: 'markdown', text: 'x' }, + headRevisionId: '501', + id: '300', + }, + revisions: { + '500': { + content: lexicalContent, + contentFormat: 'lexical', + documentId: '400', + id: '500', + text: '', + }, + }, + }) + + const result = await service.dryRunMarkdownToLexical({ + ...request, + branchId: '300', + revisionId: '500', + }) + + expect( + result.preconditions.find((item) => item.kind === 'branch'), + ).toMatchObject({ headRevisionId: '500' }) + }) }) diff --git a/apps/core/test/src/modules/draft/draft-crdt.doc.spec.ts b/apps/core/test/src/modules/draft/draft-crdt.doc.spec.ts new file mode 100644 index 00000000000..76146b26f39 --- /dev/null +++ b/apps/core/test/src/modules/draft/draft-crdt.doc.spec.ts @@ -0,0 +1,225 @@ +import { LoroDoc } from 'loro-crdt' +import { describe, expect, it } from 'vitest' + +import { + createContentDoc, + editDocAt, + readDocContent, + revertChange, + vetUpdate, +} from '~/modules/draft/draft-crdt.doc' + +const text = (value: string) => ({ + detail: 0, + format: 0, + mode: 'normal', + style: '', + text: value, + type: 'text', + version: 1, +}) + +const paragraph = (value: string) => ({ + children: [text(value)], + direction: null, + format: '', + indent: 0, + textFormat: 0, + textStyle: '', + type: 'paragraph', + version: 1, +}) + +const state = (...children: object[]) => + JSON.stringify({ + root: { + children, + direction: null, + format: '', + indent: 0, + type: 'root', + version: 1, + }, + }) + +const texts = (content: string) => + ( + JSON.parse(content) as { + root: { children: { children?: { text?: string }[]; type: string }[] } + } + ).root.children.map((child) => + child.children + ? child.children.map((leaf) => leaf.text).join('') + : child.type, + ) + +describe('draft CRDT document', () => { + it('round-trips title and body, keeping blocks the headless set does not know', () => { + const map = { type: 'map', version: 1, lat: 1, lng: 2, zoom: 3 } + const mystery = { type: 'mystery-block', version: 1, payload: { a: [1] } } + const doc = createContentDoc( + { content: state(paragraph('Hello'), map, mystery), title: 'Title' }, + 'import 1', + ) + + const read = readDocContent(doc) + const children = JSON.parse(read.content).root.children + + expect(read.title).toBe('Title') + expect(texts(read.content)).toEqual(['Hello', 'map', 'mystery-block']) + expect(children[1]).toMatchObject(map) + expect(children[2]).toMatchObject(mystery) + }) + + it('keeps block ids and nested lists through the tree', () => { + const list = { + children: [ + { + children: [text('item')], + direction: null, + format: '', + indent: 0, + type: 'listitem', + value: 1, + version: 1, + }, + ], + direction: null, + format: '', + indent: 0, + listType: 'bullet', + start: 1, + tag: 'ul', + type: 'list', + version: 1, + } + const doc = createContentDoc( + { + content: state({ ...paragraph('p'), $: { blockId: 'blk1' } }, list), + title: '', + }, + 'import 1', + ) + + const children = JSON.parse(readDocContent(doc).content).root.children + + expect(children[0].$).toEqual({ blockId: 'blk1' }) + expect(children[1].children[0].children[0].text).toBe('item') + }) + + it('merges a whole-document write made at an older version with newer edits', () => { + const doc = createContentDoc( + { content: state(paragraph('one'), paragraph('two')), title: 'T' }, + 'import 1', + ) + const base = doc.frontiers() + doc.import( + editDocAt( + doc, + doc.frontiers(), + { content: state(paragraph('one'), paragraph('two!')), title: 'T' }, + 'human:owner', + ), + ) + + doc.import( + editDocAt( + doc, + base, + { content: state(paragraph('ONE'), paragraph('two')), title: 'T2' }, + 'legacy-save 9', + ), + ) + + const read = readDocContent(doc) + expect(texts(read.content)).toEqual(['ONE', 'two!']) + expect(read.title).toBe('T2') + }) + + it('reverts one change while edits before and after it survive', () => { + const doc = createContentDoc( + { content: state(paragraph('a'), paragraph('b')), title: 'T' }, + 'import 1', + ) + const step = (content: string, message: string) => { + doc.import( + editDocAt(doc, doc.frontiers(), { content, title: 'T' }, message), + ) + } + step(state(paragraph('a1'), paragraph('b')), 'human:owner') + const before = doc.frontiers() + step( + state(paragraph('a1'), paragraph('b'), paragraph('agent')), + 'agent:add', + ) + const agentHead = doc.frontiers() + step( + state(paragraph('a1'), paragraph('b2'), paragraph('agent')), + 'human:owner', + ) + + doc.import(revertChange(doc, agentHead[0], before, 'undo agent')) + + expect(texts(readDocContent(doc).content)).toEqual(['a1', 'b2']) + }) + + it('rejects an update from a different document lineage', () => { + const doc = createContentDoc( + { content: state(paragraph('a')), title: 'T' }, + 'import 1', + ) + const stranger = createContentDoc( + { content: state(paragraph('b')), title: 'T' }, + 'import 2', + ) + + expect(() => + vetUpdate(doc, stranger.export({ mode: 'update' })), + ).toThrowError(/lineage/) + }) + + it('rejects an update whose dependencies the document lacks', () => { + const doc = createContentDoc( + { content: state(paragraph('a')), title: 'T' }, + 'import 1', + ) + const peer = new LoroDoc() + peer.import(doc.export({ mode: 'snapshot' })) + const edit = (content: string) => + peer.import( + editDocAt(peer, peer.frontiers(), { content, title: 'T' }, 'human'), + ) + edit(state(paragraph('a'), paragraph('b'))) + const afterFirst = peer.oplogVersion() + edit(state(paragraph('a'), paragraph('b'), paragraph('c'))) + + expect(() => + vetUpdate(doc, peer.export({ from: afterFirst, mode: 'update' })), + ).toThrowError(/missing/) + }) + + it('accepts a well-formed peer update and returns the merged probe', () => { + const doc = createContentDoc( + { content: state(paragraph('a')), title: 'T' }, + 'import 1', + ) + const peer = new LoroDoc() + peer.import(doc.export({ mode: 'snapshot' })) + peer.import( + editDocAt( + peer, + peer.frontiers(), + { content: state(paragraph('a'), paragraph('b')), title: 'T' }, + 'human:owner', + ), + ) + + const probe = vetUpdate( + doc, + peer.export({ from: doc.oplogVersion(), mode: 'update' }), + ) + + expect(texts(readDocContent(probe).content)).toEqual(['a', 'b']) + expect(texts(readDocContent(doc).content)).toEqual(['a']) + }) +}) diff --git a/apps/core/test/src/modules/draft/draft-crdt.repository.pg.e2e.spec.ts b/apps/core/test/src/modules/draft/draft-crdt.repository.pg.e2e.spec.ts new file mode 100644 index 00000000000..37441f896c0 --- /dev/null +++ b/apps/core/test/src/modules/draft/draft-crdt.repository.pg.e2e.spec.ts @@ -0,0 +1,240 @@ +import { SnowflakeGenerator } from '@mx-space/db-schema/id' +import * as schema from '@mx-space/db-schema/schema' +import { eq } from 'drizzle-orm' +import { drizzle } from 'drizzle-orm/node-postgres' +import { Pool } from 'pg' +import { createIsolatedPgDatabase } from 'test/helper/pg-testcontainer' +import { afterAll, beforeAll, describe, expect, it } from 'vitest' + +import { DraftRefType } from '~/modules/draft/draft.enum' +import { DraftRepository } from '~/modules/draft/draft.repository' +import { DraftCrdtRepository } from '~/modules/draft/draft-crdt.repository' +import type { AppDatabase } from '~/processors/database/postgres.provider' +import type { SnowflakeService } from '~/shared/id/snowflake.service' + +const snapshot = (title: string) => ({ + content: null, + contentFormat: 'lexical', + images: null, + meta: null, + text: title, + title, + typeSpecificData: null, +}) + +const bytes = (...values: number[]) => new Uint8Array(values) + +describe('DraftCrdtRepository (real PG)', () => { + let pool: Pool + let db: AppDatabase + let database: Awaited> + let drafts: DraftRepository + let repository: DraftCrdtRepository + + const seedDocument = async () => { + const { document, revision } = await drafts.createDocumentWithRoot( + DraftRefType.Post, + null, + snapshot('Root'), + false, + ) + const { branch } = await drafts.createBranch( + document.id, + revision.id, + snapshot('Root'), + true, + ) + return { branch, document, revision } + } + + beforeAll(async () => { + database = await createIsolatedPgDatabase() + pool = new Pool({ connectionString: database.getConnectionUri(), max: 4 }) + db = drizzle(pool, { schema }) as unknown as AppDatabase + const snowflake = new SnowflakeGenerator({ + workerId: 44, + }) as SnowflakeService + drafts = new DraftRepository(db, snowflake) + repository = new DraftCrdtRepository(db, snowflake) + }, 120_000) + + afterAll(async () => { + await pool?.end() + await database?.drop() + }) + + it('creates a doc once and returns the stored row on a second create', async () => { + const { document, revision } = await seedDocument() + + const first = await repository.createDoc({ + documentId: document.id, + fields: { meta: null, typeSpecificData: { slug: 'a' } }, + snapshot: bytes(1, 2, 3), + snapshotVersion: bytes(9), + syncedRevisionId: revision.id, + }) + const second = await repository.createDoc({ + documentId: document.id, + fields: null, + snapshot: bytes(7), + snapshotVersion: bytes(7), + syncedRevisionId: null, + }) + + expect(first.snapshot).toEqual(bytes(1, 2, 3)) + expect(second.snapshot).toEqual(bytes(1, 2, 3)) + expect(second.syncedRevisionId).toBe(revision.id) + expect(await repository.findDoc(document.id)).toMatchObject({ + fields: { meta: null, typeSpecificData: { slug: 'a' } }, + snapshotVersion: bytes(9), + }) + }) + + it('appends updates and compacts exactly the folded rows', async () => { + const { document } = await seedDocument() + await repository.createDoc({ + documentId: document.id, + fields: null, + snapshot: bytes(0), + snapshotVersion: bytes(0), + syncedRevisionId: null, + }) + const a = await repository.appendUpdate(document.id, bytes(1)) + const b = await repository.appendUpdate(document.id, bytes(2)) + + const loaded = await repository.findUpdates(document.id) + expect(loaded.map((row) => row.bytes)).toEqual([bytes(1), bytes(2)]) + + const late = await repository.appendUpdate(document.id, bytes(3)) + const compacted = await repository.compact(document.id, { + expectedVersion: bytes(0), + foldedIds: [a, b], + snapshot: bytes(1, 2), + snapshotVersion: bytes(2), + }) + + expect(compacted).toBe(true) + expect( + (await repository.findUpdates(document.id)).map((row) => row.id), + ).toEqual([late]) + expect((await repository.findDoc(document.id))?.snapshot).toEqual( + bytes(1, 2), + ) + }) + + it('refuses a compaction built from a stale snapshot', async () => { + const { document } = await seedDocument() + await repository.createDoc({ + documentId: document.id, + fields: null, + snapshot: bytes(0), + snapshotVersion: bytes(5), + syncedRevisionId: null, + }) + const a = await repository.appendUpdate(document.id, bytes(1)) + + const compacted = await repository.compact(document.id, { + expectedVersion: bytes(4), + foldedIds: [a], + snapshot: bytes(1), + snapshotVersion: bytes(6), + }) + + expect(compacted).toBe(false) + expect(await repository.findUpdates(document.id)).toHaveLength(1) + expect((await repository.findDoc(document.id))?.snapshot).toEqual(bytes(0)) + }) + + it('merges fields per top-level key without clobbering other keys', async () => { + const { document } = await seedDocument() + await repository.createDoc({ + documentId: document.id, + fields: { + meta: { cover: 'a.png', keep: 1 }, + typeSpecificData: { slug: 'old', tags: ['x'] }, + }, + snapshot: bytes(0), + snapshotVersion: bytes(0), + syncedRevisionId: null, + }) + + await repository.mergeFields(document.id, { + typeSpecificData: { slug: 'new' }, + }) + const merged = await repository.mergeFields(document.id, { + meta: { cover: null }, + }) + + expect(merged).toEqual({ + meta: { cover: null, keep: 1 }, + typeSpecificData: { slug: 'new', tags: ['x'] }, + }) + }) + + it('merges into a section that is still null', async () => { + const { document } = await seedDocument() + await repository.createDoc({ + documentId: document.id, + fields: { meta: null, typeSpecificData: null }, + snapshot: bytes(0), + snapshotVersion: bytes(0), + syncedRevisionId: null, + }) + + expect( + await repository.mergeFields(document.id, { meta: { cover: 'c' } }), + ).toEqual({ meta: { cover: 'c' }, typeSpecificData: null }) + }) + + it('chains checkpoints linearly and moves the working copy head to the last', async () => { + const { branch, revision } = await seedDocument() + const frontiers = [{ counter: 3, peer: '12' as const }] + + const { branch: moved, revisions } = await repository.appendCheckpoints( + branch.id, + [ + { frontiers, kind: 'legacy-save', snapshot: snapshot('Own') }, + { frontiers: null, kind: 'save', snapshot: snapshot('Head') }, + ], + ) + + const [own, head] = revisions + expect(own).toMatchObject({ + frontiers, + kind: 'legacy-save', + parentRevisionId: revision.id, + title: 'Own', + }) + expect(head.parentRevisionId).toBe(own.id) + expect(moved.headRevisionId).toBe(head.id) + const [row] = await db + .select() + .from(schema.drafts) + .where(eq(schema.drafts.id, branch.id)) + expect(row.headRevisionId).toBe(head.id) + }) + + it('serializes work on the same document under the document lock', async () => { + const { document } = await seedDocument() + const order: string[] = [] + let release!: () => void + const gate = new Promise((resolve) => { + release = resolve + }) + + const first = repository.withDocumentLock(document.id, async () => { + order.push('first:start') + await gate + order.push('first:end') + }) + await new Promise((resolve) => setTimeout(resolve, 50)) + const second = repository.withDocumentLock(document.id, async () => { + order.push('second') + }) + await new Promise((resolve) => setTimeout(resolve, 100)) + release() + await Promise.all([first, second]) + + expect(order).toEqual(['first:start', 'first:end', 'second']) + }) +}) diff --git a/apps/core/test/src/modules/draft/draft-crdt.service.pg.e2e.spec.ts b/apps/core/test/src/modules/draft/draft-crdt.service.pg.e2e.spec.ts new file mode 100644 index 00000000000..e47afaec5c2 --- /dev/null +++ b/apps/core/test/src/modules/draft/draft-crdt.service.pg.e2e.spec.ts @@ -0,0 +1,464 @@ +import { SnowflakeGenerator } from '@mx-space/db-schema/id' +import * as schema from '@mx-space/db-schema/schema' +import { eq } from 'drizzle-orm' +import { drizzle } from 'drizzle-orm/node-postgres' +import { LoroDoc } from 'loro-crdt' +import { Pool } from 'pg' +import { createIsolatedPgDatabase } from 'test/helper/pg-testcontainer' +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' + +import { AppErrorCode } from '~/common/errors/app-error-code' +import { DraftRefType } from '~/modules/draft/draft.enum' +import { DraftRepository } from '~/modules/draft/draft.repository' +import type { RevisionSnapshot } from '~/modules/draft/draft.types' +import { editDocAt, readDocContent } from '~/modules/draft/draft-crdt.doc' +import { DraftCrdtRepository } from '~/modules/draft/draft-crdt.repository' +import { DraftCrdtService } from '~/modules/draft/draft-crdt.service' +import type { AppDatabase } from '~/processors/database/postgres.provider' +import type { SnowflakeService } from '~/shared/id/snowflake.service' + +const paragraph = (value: string) => ({ + children: [ + { + detail: 0, + format: 0, + mode: 'normal', + style: '', + text: value, + type: 'text', + version: 1, + }, + ], + direction: null, + format: '', + indent: 0, + textFormat: 0, + textStyle: '', + type: 'paragraph', + version: 1, +}) + +const body = (...values: string[]) => + JSON.stringify({ + root: { + children: values.map(paragraph), + direction: null, + format: '', + indent: 0, + type: 'root', + version: 1, + }, + }) + +const texts = (content: string | null) => + ( + JSON.parse(content ?? '{"root":{"children":[]}}') as { + root: { children: { children: { text: string }[] }[] } + } + ).root.children.map((child) => child.children.map((t) => t.text).join('')) + +const lexical = ( + title: string, + values: string[], + typeSpecificData: Record | null = { slug: 's' }, +): RevisionSnapshot => ({ + content: body(...values), + contentFormat: 'lexical', + images: null, + meta: null, + text: values.join('\n\n'), + title, + typeSpecificData, +}) + +describe('DraftCrdtService (real PG)', () => { + let pool: Pool + let db: AppDatabase + let database: Awaited> + let drafts: DraftRepository + let crdt: DraftCrdtRepository + let service: DraftCrdtService + const fileReferences = { + removeReferencesForDocument: vi.fn(), + updateReferencesForDocument: vi.fn(), + } + + const seed = async (published = lexical('Pub', ['p1', 'p2'])) => { + const { document, revision } = await drafts.createDocumentWithRoot( + DraftRefType.Post, + null, + published, + true, + ) + const { branch } = await drafts.createBranch( + document.id, + revision.id, + published, + true, + ) + return { branch, document, published: revision } + } + + const clientOf = async (branchId: string) => { + const doc = new LoroDoc() + doc.import(await service.exportUpdates(branchId)) + return doc + } + + const clientEdit = async ( + branchId: string, + doc: LoroDoc, + title: string, + values: string[], + ) => { + const from = doc.oplogVersion() + doc.import( + editDocAt( + doc, + doc.frontiers(), + { content: body(...values), title }, + 'human:owner', + ), + ) + await service.importUpdate(branchId, doc.export({ from, mode: 'update' })) + } + + beforeAll(async () => { + database = await createIsolatedPgDatabase() + pool = new Pool({ connectionString: database.getConnectionUri(), max: 4 }) + db = drizzle(pool, { schema }) as unknown as AppDatabase + const snowflake = new SnowflakeGenerator({ + workerId: 45, + }) as SnowflakeService + drafts = new DraftRepository(db, snowflake) + crdt = new DraftCrdtRepository(db, snowflake) + service = new DraftCrdtService(drafts, crdt, fileReferences as never) + }, 120_000) + + afterAll(async () => { + await pool?.end() + await database?.drop() + }) + + it('imports a document lazily with an import checkpoint the doc matches', async () => { + const { branch, document } = await seed() + + const workingCopy = await service.workingCopy(branch.id) + + const head = await drafts.findRevisionById(workingCopy.headRevisionId) + expect(head).toMatchObject({ kind: 'import', title: 'Pub' }) + expect(head?.frontiers?.length).toBeGreaterThan(0) + expect(texts(head!.content)).toEqual(['p1', 'p2']) + const row = await crdt.findDoc(document.id) + expect(row?.syncedRevisionId).toBe(head!.id) + expect(row?.fields).toEqual({ meta: null, typeSpecificData: { slug: 's' } }) + expect(readDocContent(await clientOf(branch.id)).title).toBe('Pub') + }) + + it('keeps one working copy, archiving older branches and pinning their follow shares', async () => { + const { branch: older, document, published } = await seed() + const { branch: newer } = await drafts.createBranch( + document.id, + published.id, + lexical('Newer', ['n']), + false, + ) + await drafts.upsertShare(document.id, 'tok-older', { + draftId: older.id, + mode: 'follow', + }) + + const workingCopy = await service.workingCopy(older.id) + + expect(workingCopy.id).toBe(newer.id) + expect((await drafts.findBranchById(older.id))?.status).toBe('archived') + expect(await drafts.findShareByToken('tok-older')).toMatchObject({ + draftId: null, + mode: 'pinned', + revisionId: older.headRevisionId, + }) + }) + + it('creates a working copy from the published revision when none is active', async () => { + const { branch, document } = await seed() + await drafts.archiveBranch(branch.id) + + const workingCopy = await service.workingCopy(branch.id) + + expect(workingCopy.id).not.toBe(branch.id) + expect(workingCopy.status).toBe('active') + expect(workingCopy.documentId).toBe(document.id) + }) + + it('syncs peer updates and checkpoints the merged content, skipping no-op saves', async () => { + const { branch } = await seed() + const tabA = await clientOf(branch.id) + const tabB = await clientOf(branch.id) + + await clientEdit(branch.id, tabA, 'Pub', ['p1 from A', 'p2']) + await clientEdit(branch.id, tabB, 'Pub', ['p1', 'p2 from B']) + const { revision } = await service.createCheckpoint(branch.id, 'save') + const again = await service.createCheckpoint(branch.id, 'save') + + expect(revision.kind).toBe('save') + expect(texts(revision.content)).toEqual(['p1 from A', 'p2 from B']) + expect(revision.text).toContain('p1 from A') + expect(again.revision.id).toBe(revision.id) + expect(fileReferences.updateReferencesForDocument).toHaveBeenCalled() + }) + + it('serves only the updates a client lacks', async () => { + const { branch } = await seed() + const tab = await clientOf(branch.id) + const since = tab.oplogVersion().encode() + const other = await clientOf(branch.id) + await clientEdit(branch.id, other, 'Pub', ['p1', 'p2', 'p3']) + + tab.import(await service.exportUpdates(branch.id, since)) + + expect(texts(readDocContent(tab).content)).toEqual(['p1', 'p2', 'p3']) + }) + + it('compacts a long update log into the snapshot without losing edits', async () => { + const { branch, document } = await seed() + const tab = await clientOf(branch.id) + for (let i = 0; i < 100; i++) { + await clientEdit(branch.id, tab, 'Pub', ['p1', `p2 ${i}`]) + } + + await service.workingCopy(branch.id) + await vi.waitFor(async () => { + expect(await crdt.findUpdates(document.id)).toHaveLength(0) + }) + + const fresh = await clientOf(branch.id) + expect(texts(readDocContent(fresh).content)).toEqual(['p1', 'p2 99']) + }) + + it('reloads when another replica compacts between reading the snapshot and the updates', async () => { + const { branch, document } = await seed() + const tab = await clientOf(branch.id) + await clientEdit(branch.id, tab, 'Pub', ['p1', 'p2 edited']) + const racing = Object.create(crdt) as DraftCrdtRepository + let raced = false + racing.findUpdates = async (documentId) => { + if (!raced) { + raced = true + const row = await crdt.findDoc(documentId) + const updates = await crdt.findUpdates(documentId) + const full = new LoroDoc() + full.import(row!.snapshot) + full.importBatch(updates.map((update) => update.bytes)) + await crdt.compact(documentId, { + expectedVersion: row!.snapshotVersion, + foldedIds: updates.map((update) => update.id), + snapshot: full.export({ mode: 'snapshot' }), + snapshotVersion: full.oplogVersion().encode(), + }) + } + return crdt.findUpdates(documentId) + } + const replica = new DraftCrdtService( + drafts, + racing, + fileReferences as never, + ) + + const { revision } = await replica.createCheckpoint(branch.id, 'save') + + expect(raced).toBe(true) + expect(texts(revision.content)).toEqual(['p1', 'p2 edited']) + expect(await crdt.findUpdates(document.id)).toHaveLength(0) + }) + + it('rejects an update from another document lineage without storing it', async () => { + const { branch, document } = await seed() + await service.workingCopy(branch.id) + const { branch: strangerBranch } = await seed(lexical('X', ['x'])) + const stranger = await clientOf(strangerBranch.id) + + await expect( + service.importUpdate(branch.id, stranger.export({ mode: 'update' })), + ).rejects.toMatchObject({ + code: AppErrorCode.DRAFT_CRDT_UPDATE_REJECTED, + }) + expect(await crdt.findUpdates(document.id)).toHaveLength(0) + }) + + it('applies a legacy write at its base version and returns its own checkpoint', async () => { + const { branch } = await seed() + const workingCopy = await service.workingCopy(branch.id) + const base = workingCopy.headRevisionId + const tab = await clientOf(branch.id) + await clientEdit(branch.id, tab, 'Pub', ['p1', 'p2 unfinished']) + await service.createCheckpoint(branch.id, 'save') + + const { branch: after, revision } = await service.legacyWrite( + branch.id, + base, + lexical('Pub', ['p1 cli', 'p2'], { slug: 'cli' }), + ) + + expect(texts(revision.content)).toEqual(['p1 cli', 'p2']) + expect(revision.typeSpecificData).toEqual({ slug: 'cli' }) + expect(revision.kind).toBe('legacy-save') + const head = await drafts.findRevisionById(after.headRevisionId) + expect(texts(head!.content)).toEqual(['p1 cli', 'p2 unfinished']) + expect(head!.typeSpecificData).toEqual({ slug: 'cli' }) + expect(head!.parentRevisionId).toBe(revision.id) + }) + + it('three-way merges a write based on the revision the doc was imported from', async () => { + const { branch, published } = await seed() + await service.workingCopy(branch.id) + const tab = await clientOf(branch.id) + await clientEdit(branch.id, tab, 'Pub', ['p1', 'p2 owner']) + + const { branch: after } = await service.legacyWrite( + branch.id, + published.id, + lexical('Pub', ['p1 cli', 'p2']), + ) + + const head = await drafts.findRevisionById(after.headRevisionId) + expect(texts(head!.content)).toEqual(['p1 cli', 'p2 owner']) + }) + + it('rejects a Markdown legacy write to a Loro document', async () => { + const { branch } = await seed() + await service.workingCopy(branch.id) + + await expect( + service.legacyWrite(branch.id, null, { + ...lexical('M', []), + content: null, + contentFormat: 'markdown', + text: '# md', + }), + ).rejects.toMatchObject({ + code: AppErrorCode.DRAFT_CRDT_FORMAT_UNSUPPORTED, + }) + }) + + it('folds in a save an old pod made outside the Loro doc', async () => { + const { branch } = await seed() + const workingCopy = await service.workingCopy(branch.id) + const tab = await clientOf(branch.id) + await clientEdit(branch.id, tab, 'Pub', ['p1', 'p2 new']) + const saved = await drafts.saveBranch( + branch.id, + workingCopy.headRevisionId, + lexical('Old pod', ['p1 old pod', 'p2']), + ) + expect(saved.kind).toBe('ok') + + const reconciled = await service.workingCopy(branch.id) + + const head = await drafts.findRevisionById(reconciled.headRevisionId) + expect(head?.kind).toBe('legacy-save') + expect(head?.title).toBe('Old pod') + expect(texts(head!.content)).toEqual(['p1 old pod', 'p2 new']) + }) + + it('restores a checkpoint as a new commit, fields included', async () => { + const { branch } = await seed() + const imported = await service.workingCopy(branch.id) + const tab = await clientOf(branch.id) + await clientEdit(branch.id, tab, 'Changed', ['changed']) + await service.updateFields(branch.id, { typeSpecificData: { slug: 'x' } }) + await service.createCheckpoint(branch.id, 'save') + + const { revision } = await service.restore( + branch.id, + imported.headRevisionId, + ) + + expect(revision.kind).toBe('restore') + expect(revision.title).toBe('Pub') + expect(texts(revision.content)).toEqual(['p1', 'p2']) + expect(revision.typeSpecificData).toEqual({ slug: 's' }) + }) + + it('undoes one step while later edits survive', async () => { + const { branch } = await seed() + await service.workingCopy(branch.id) + const tab = await clientOf(branch.id) + await clientEdit(branch.id, tab, 'Pub', ['p1', 'p2', 'agent']) + const { changes } = await service.history(branch.id) + const step = changes.find((change) => change.message === 'human:owner')! + await clientEdit(branch.id, tab, 'Pub', ['p1 later', 'p2', 'agent']) + + const { revision } = await service.undo(branch.id, step.id, step.deps) + + expect(revision.kind).toBe('undo') + expect(texts(revision.content)).toEqual(['p1 later', 'p2']) + }) + + it('merges metadata fields into the next checkpoint', async () => { + const { branch } = await seed() + await service.workingCopy(branch.id) + + await service.updateFields(branch.id, { + meta: { cover: 'c.png' }, + typeSpecificData: { tags: ['a'] }, + }) + const { revision } = await service.createCheckpoint(branch.id, 'save') + + expect(revision.meta).toEqual({ cover: 'c.png' }) + expect(revision.typeSpecificData).toEqual({ slug: 's', tags: ['a'] }) + }) + + it('lists checkpoints and change log newest first', async () => { + const { branch } = await seed() + await service.workingCopy(branch.id) + const tab = await clientOf(branch.id) + await clientEdit(branch.id, tab, 'Pub', ['p1', 'p2', 'p3']) + await service.createCheckpoint(branch.id, 'save') + + const { changes, checkpoints } = await service.history(branch.id) + + expect(checkpoints.map((row) => row.kind)).toEqual(['save', 'import', null]) + expect(checkpoints[0]).not.toHaveProperty('content') + expect(checkpoints[0]).toMatchObject({ title: 'Pub' }) + expect(changes[0].message).toBe('human:owner') + expect(changes.at(-1)?.message).toMatch(/^import /) + }) + + it('discards the working copy back to the published revision', async () => { + const { branch, published } = await seed() + await service.workingCopy(branch.id) + const tab = await clientOf(branch.id) + await clientEdit(branch.id, tab, 'Draft', ['unpublished']) + + const discarded = await service.discard(branch.id) + + expect(discarded?.status).toBe('active') + const head = await drafts.findRevisionById(discarded!.headRevisionId) + expect(head?.kind).toBe('restore') + expect(head?.title).toBe(published.title) + expect(texts(head!.content)).toEqual(['p1', 'p2']) + }) + + it('archives a never-published working copy on discard', async () => { + const { document, revision } = await drafts.createDocumentWithRoot( + DraftRefType.Note, + null, + lexical('New', ['n']), + false, + ) + const { branch } = await drafts.createBranch( + document.id, + revision.id, + lexical('New', ['n']), + true, + ) + await service.workingCopy(branch.id) + + expect(await service.discard(branch.id)).toBeNull() + const [row] = await db + .select() + .from(schema.drafts) + .where(eq(schema.drafts.id, branch.id)) + expect(row.status).toBe('archived') + }) +}) diff --git a/apps/core/test/src/modules/draft/draft.service.crdt.pg.e2e.spec.ts b/apps/core/test/src/modules/draft/draft.service.crdt.pg.e2e.spec.ts new file mode 100644 index 00000000000..9341646147b --- /dev/null +++ b/apps/core/test/src/modules/draft/draft.service.crdt.pg.e2e.spec.ts @@ -0,0 +1,291 @@ +import { SnowflakeGenerator } from '@mx-space/db-schema/id' +import * as schema from '@mx-space/db-schema/schema' +import { drizzle } from 'drizzle-orm/node-postgres' +import { LoroDoc } from 'loro-crdt' +import { Pool } from 'pg' +import { createIsolatedPgDatabase } from 'test/helper/pg-testcontainer' +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' + +import { AppErrorCode } from '~/common/errors/app-error-code' +import { BusinessEvents } from '~/constants/business-event.constant' +import { DraftRefType } from '~/modules/draft/draft.enum' +import { DraftRepository } from '~/modules/draft/draft.repository' +import { DraftService } from '~/modules/draft/draft.service' +import type { RevisionSnapshot } from '~/modules/draft/draft.types' +import { editDocAt } from '~/modules/draft/draft-crdt.doc' +import { DraftCrdtRepository } from '~/modules/draft/draft-crdt.repository' +import { DraftCrdtService } from '~/modules/draft/draft-crdt.service' +import type { AppDatabase } from '~/processors/database/postgres.provider' +import type { SnowflakeService } from '~/shared/id/snowflake.service' + +const body = (...values: string[]) => + JSON.stringify({ + root: { + children: values.map((value) => ({ + children: [ + { + detail: 0, + format: 0, + mode: 'normal', + style: '', + text: value, + type: 'text', + version: 1, + }, + ], + direction: null, + format: '', + indent: 0, + textFormat: 0, + textStyle: '', + type: 'paragraph', + version: 1, + })), + direction: null, + format: '', + indent: 0, + type: 'root', + version: 1, + }, + }) + +const texts = (content: string | null) => + ( + JSON.parse(content!) as { + root: { children: { children: { text: string }[] }[] } + } + ).root.children.map((child) => child.children.map((t) => t.text).join('')) + +const lexical = (title: string, values: string[]): RevisionSnapshot => ({ + content: body(...values), + contentFormat: 'lexical', + images: null, + meta: null, + text: values.join('\n\n'), + title, + typeSpecificData: { slug: 's' }, +}) + +const markdown = (title: string): RevisionSnapshot => ({ + content: null, + contentFormat: 'markdown', + images: null, + meta: null, + text: title, + title, + typeSpecificData: null, +}) + +describe('DraftService on the Loro path (real PG)', () => { + let pool: Pool + let database: Awaited> + let drafts: DraftRepository + let service: DraftService + let refCounter = 7_100_000_000_000_000_000n + const emit = vi.fn() + const fileReferences = { + removeReferencesForDocument: vi.fn(), + updateReferencesForDocument: vi.fn(), + } + + const seed = async (published: RevisionSnapshot) => { + const refId = String(refCounter++) + const { document, revision } = await drafts.createDocumentWithRoot( + DraftRefType.Post, + refId, + published, + true, + ) + const { branch } = await drafts.createBranch( + document.id, + revision.id, + published, + true, + ) + return { branch, document, published: revision, refId } + } + + beforeAll(async () => { + database = await createIsolatedPgDatabase() + pool = new Pool({ connectionString: database.getConnectionUri(), max: 6 }) + const db = drizzle(pool, { schema }) as unknown as AppDatabase + const snowflake = new SnowflakeGenerator({ + workerId: 46, + }) as SnowflakeService + drafts = new DraftRepository(db, snowflake) + const crdt = new DraftCrdtService( + drafts, + new DraftCrdtRepository(db, snowflake), + fileReferences as never, + ) + service = new DraftService( + drafts, + fileReferences as never, + {} as never, + { emit } as never, + crdt, + ) + }, 120_000) + + afterAll(async () => { + await pool?.end() + await database?.drop() + }) + + it('never conflicts on a stale PUT and returns the write’s own checkpoint', async () => { + const { branch, published } = await seed(lexical('Pub', ['a', 'b'])) + await service.update(branch.id, { + data: lexical('Pub', ['a', 'b owner']), + expectedHeadRevisionId: published.id, + } as never) + + const view = await service.update(branch.id, { + data: lexical('Pub', ['a cli', 'b']), + expectedHeadRevisionId: published.id, + } as never) + + expect(texts(view.headRevision.content)).toEqual(['a cli', 'b']) + const stored = await drafts.findBranchById(view.id) + const head = await drafts.findRevisionById(stored!.headRevisionId) + expect(texts(head!.content)).toEqual(['a cli', 'b owner']) + expect(emit).toHaveBeenCalledWith( + BusinessEvents.DRAFT_UPDATE, + expect.objectContaining({ headRevisionId: stored!.headRevisionId }), + expect.anything(), + ) + }) + + it('writes a CLI save against the published base into the working copy', async () => { + const { branch, published, refId } = await seed(lexical('Pub', ['a'])) + + const view = await service.create({ + baseRevisionId: published.id, + data: lexical('Pub', ['a', 'cli']), + refId, + refType: DraftRefType.Post, + } as never) + + expect(view.id).toBe(branch.id) + expect(texts(view.headRevision.content)).toEqual(['a', 'cli']) + expect(await drafts.findBranchesByDocument(branch.documentId)).toHaveLength( + 1, + ) + }) + + it('keeps the branch flow and its conflicts for Markdown documents', async () => { + const { branch, published } = await seed(markdown('Md')) + await service.update(branch.id, { + data: markdown('Md 2'), + expectedHeadRevisionId: published.id, + } as never) + + await expect( + service.update(branch.id, { + data: markdown('Md 3'), + expectedHeadRevisionId: published.id, + } as never), + ).rejects.toMatchObject({ code: AppErrorCode.DRAFT_HEAD_CONFLICT }) + }) + + it('collapses a Lexical document to one branch when its context is read', async () => { + const { document, published, refId } = await seed(lexical('Pub', ['a'])) + await drafts.createBranch( + document.id, + published.id, + lexical('Other', ['o']), + false, + ) + + const context = await service.getContext(DraftRefType.Post, refId) + + expect(context.branches).toHaveLength(1) + expect(context.branches[0].headRevision.kind).toBe('import') + }) + + it('discards a Lexical working copy back to the published content', async () => { + const { branch } = await seed(lexical('Pub', ['a'])) + await service.update(branch.id, { + data: lexical('Draft', ['changed']), + expectedHeadRevisionId: branch.headRevisionId, + } as never) + + await service.delete(branch.id) + + const view = await service.findById(branch.id) + expect(view?.status).toBe('active') + expect(view?.headRevision).toMatchObject({ kind: 'restore', title: 'Pub' }) + }) + + it('syncs binary updates and broadcasts the working copy', async () => { + const { branch } = await seed(lexical('Pub', ['a'])) + const tab = new LoroDoc() + tab.import(await service.exportCrdt(branch.id)) + const from = tab.oplogVersion() + tab.import( + editDocAt( + tab, + tab.frontiers(), + { content: body('a', 'typed'), title: 'Pub' }, + 'human:owner', + ), + ) + emit.mockClear() + + await service.importCrdt(branch.id, tab.export({ from, mode: 'update' })) + const { headRevision } = await service.createCheckpoint(branch.id, 'save') + + expect(texts(headRevision.content)).toEqual(['a', 'typed']) + expect(emit).toHaveBeenCalledWith( + BusinessEvents.DRAFT_UPDATE, + expect.objectContaining({ branchId: branch.id }), + expect.anything(), + ) + }) + + it('merges live article edits into the working copy fields', async () => { + const { branch, refId } = await seed(lexical('Pub', ['a'])) + await service.findById(branch.id) + + await service.mergeArticleFields(DraftRefType.Post, refId, { + categoryId: 'cat-2', + pin: true, + }) + const view = await service.createCheckpoint(branch.id, 'save') + + expect(view.headRevision.typeSpecificData).toEqual({ + categoryId: 'cat-2', + pin: true, + slug: 's', + }) + }) + + it('leaves documents without a Loro doc alone on live edits', async () => { + const { branch, refId } = await seed(markdown('Md')) + + await service.mergeArticleFields(DraftRefType.Post, refId, { pin: true }) + + const view = await service.findById(branch.id) + expect(view?.headRevision.typeSpecificData).toBeNull() + }) + + it('opens a Lexical draft for editing at a fresh checkpoint', async () => { + const { branch } = await seed(lexical('Pub', ['a'])) + await service.findById(branch.id) + await service.updateFields(branch.id, { meta: { cover: 'late.png' } }) + + const view = await service.findForEditing(branch.id) + + expect(view?.headRevision.meta).toEqual({ cover: 'late.png' }) + expect((await service.findForEditing(branch.id))?.headRevisionId).toBe( + view?.headRevisionId, + ) + }) + + it('opens a Markdown draft for editing without touching it', async () => { + const { branch } = await seed(markdown('Md')) + + const view = await service.findForEditing(branch.id) + + expect(view?.headRevisionId).toBe(branch.headRevisionId) + }) +}) diff --git a/apps/core/test/src/modules/draft/draft.service.spec.ts b/apps/core/test/src/modules/draft/draft.service.spec.ts index e0a091563cc..660c318cdc9 100644 --- a/apps/core/test/src/modules/draft/draft.service.spec.ts +++ b/apps/core/test/src/modules/draft/draft.service.spec.ts @@ -74,6 +74,10 @@ const harness = () => { fileReferences as never, {} as never, { emit: vi.fn() } as never, + { + usesCrdt: vi.fn(async () => false), + workingCopyOfDocument: vi.fn(async () => null), + } as never, ) return { fileReferences, repository, service } } diff --git a/apps/core/test/src/modules/publish/publish.service.spec.ts b/apps/core/test/src/modules/publish/publish.service.spec.ts index 71a8e1a4dff..2229301b168 100644 --- a/apps/core/test/src/modules/publish/publish.service.spec.ts +++ b/apps/core/test/src/modules/publish/publish.service.spec.ts @@ -42,13 +42,27 @@ const makeBranch = () => ({ status: 'active', }) -const harness = (options: { online?: boolean } = {}) => { +const harness = (options: { crdt?: boolean; online?: boolean } = {}) => { const branch = makeBranch() + const checkpoints = new Map>([ + [ + 'checkpoint-1', + { + ...snapshot, + documentId: 'document-1', + id: 'checkpoint-1', + kind: 'publish', + title: 'Checkpoint title', + }, + ], + ]) const order: string[] = [] let handler: TaskHandler | undefined let payload: PublishTaskPayload | undefined const drafts = { findById: vi.fn(async () => branch), + findRevisionById: vi.fn(async (id: string) => checkpoints.get(id) ?? null), + usesCrdt: vi.fn(async () => options.crdt ?? false), linkDocument: vi.fn(), recordPublication: vi.fn(async () => ({ kind: 'ok' })), setPublishAiResources: vi.fn(), @@ -104,13 +118,17 @@ const harness = (options: { online?: boolean } = {}) => { ) service.onModuleInit() - const create = (confirmDiverged = false, aiResources?: unknown) => + const create = ( + confirmDiverged = false, + aiResources?: unknown, + revisionId: string = branch.headRevisionId, + ) => service.create({ aiResources, branchId: branch.id, confirmDiverged, expectedPublishedRevisionId: branch.document.publishedRevisionId, - revisionId: branch.headRevisionId, + revisionId, } as never) const run = async (signal = new AbortController().signal) => { const setResult = vi.fn() @@ -142,6 +160,44 @@ const harness = (options: { online?: boolean } = {}) => { } } +describe('PublishService on a Loro working copy', () => { + it('publishes a checkpoint that is not the working copy head', async () => { + const { create, payload } = harness({ crdt: true }) + + await create(false, undefined, 'checkpoint-1') + + expect(payload()).toMatchObject({ + revisionId: 'checkpoint-1', + snapshot: { title: 'Checkpoint title' }, + }) + }) + + it('ignores the divergence confirmation for a working copy', async () => { + const { branch, create, tasks } = harness({ crdt: true }) + branch.relationToPublished = 'diverged' + + await create(false, undefined, 'checkpoint-1') + + expect(tasks.createTask).toHaveBeenCalled() + }) + + it('rejects a revision from another document', async () => { + const { create } = harness({ crdt: true }) + + await expect( + create(false, undefined, 'unknown-revision'), + ).rejects.toMatchObject({ code: AppErrorCode.CONTENT_REVISION_NOT_FOUND }) + }) + + it('still requires the head revision for a Markdown branch', async () => { + const { create } = harness() + + await expect( + create(false, undefined, 'checkpoint-1'), + ).rejects.toMatchObject({ code: AppErrorCode.DRAFT_HEAD_CONFLICT }) + }) +}) + describe('PublishService tree selection', () => { it('requires confirmation when publishing a branch from an older base', async () => { const { branch, create, tasks } = harness() diff --git a/apps/core/vite.config.ts b/apps/core/vite.config.ts index 92cf8c152f7..16f08ee320b 100644 --- a/apps/core/vite.config.ts +++ b/apps/core/vite.config.ts @@ -89,7 +89,7 @@ export default defineConfig(({ command }) => { entryFileNames: '[name].mjs', chunkFileNames: 'chunks/[name]-[hash].mjs', }, - external: ['sharp', 'ws'], + external: ['sharp', 'ws', 'loro-crdt'], }, }, }), diff --git a/dockerfile b/dockerfile index b05cb6a5b38..a37dc8e185f 100644 --- a/dockerfile +++ b/dockerfile @@ -47,9 +47,9 @@ COPY --from=builder /app/out . RUN npm i sharp -g RUN npm i sharp -# WsAdapter and sandbox workers load these packages outside the Vite bundle. +# WsAdapter, sandbox workers and loro-crdt (reads its wasm next to itself) load outside the Vite bundle. # The production image only copies the Vite bundle, so install it next to sharp. -RUN npm i ws@8.21.3 undici@8.11.2 +RUN npm i ws@8.21.3 undici@8.11.2 loro-crdt@1.16.4 COPY --chmod=755 docker-entrypoint.sh . COPY docker-browser-watchdog.mjs . diff --git a/docs/superpowers/specs/2026-10-25-loro-content-storage-design.md b/docs/superpowers/specs/2026-10-25-loro-content-storage-design.md new file mode 100644 index 00000000000..8d97cbe96c1 --- /dev/null +++ b/docs/superpowers/specs/2026-10-25-loro-content-storage-design.md @@ -0,0 +1,464 @@ +# Loro as the content store for drafts + +Date: 2026-10-25 +Status: draft, pending approval +Phase 1 of 4 in the collaborative editing track (see "Track"). + +## Intent + +The author wants articles to be co-written live: the author, invited readers +(co-authors) and AI agents all editing one document, each with a visible +cursor, every change attributed, and any single step revertible. That needs a +CRDT underneath the editor. haklex already ships one binding, +`@haklex/rich-collab-loro` (Lexical ↔ LoroTree), and `mxs author` already runs +a human + agent session on it. + +Today's draft system cannot carry that. It is a git-like model built to +survive conflicts: `content_revisions` is a snapshot chain, `drafts` are +branches with base/head pointers, saves use an optimistic lock +(`expectedHeadRevisionId` / `DRAFT_HEAD_CONFLICT`), and the admin write page +carries a three-way merge, a conflict dialog and a version tree to cope. In +the author's words, draft branches only exist because of conflicts. A CRDT +makes conflicts disappear, and with them the reason for branches. Two +histories (Loro's oplog and our revision chain) both claiming to be the truth +is the clash this phase removes. + +This phase swaps the storage under the editor and nothing else: one user (the +owner), one admin write page, HTTP sync. Realtime transport, the server-side +AI peer, and the co-author page come in later phases and build on this one. + +## Track + +1. **This document.** Loro doc per content document is the source of truth; + history becomes a linear timeline of checkpoints; branches and the + optimistic lock go away. +2. Realtime room over WebSocket (`collab.*` events on the existing + `@mx-space/ws-client` envelope), presence and cursors. +3. Server-side AI peer that writes straight into the doc. +4. Co-author page (`#/collab/:documentId` inside admin), invite links, + reader OAuth login, per-document AI quota, byline. + +Each phase gets its own design. Phase 2 and 3 protocol work is being aligned +with the Write Agent WebSocket design in a parallel session; nothing here may +assume the transport. + +## Decisions made by the author + +- Loro's history replaces our own. The existing draft history conflicts with + it and should not be maintained alongside it. +- Draft branches are not useful and go away. The one thing branches did that + matters, "go back to an old version, keep editing, and the newer history + still exists", is covered by Loro's revert-as-new-ops (`revertTo`, or + restoring a checkpoint as a new commit). History stays linear and nothing + is lost. +- Reverting works at two grains: the whole document back to a point, and a + single step (one AI turn, one human edit run) undone on its own while later + edits survive. +- AI edits land directly in the document like a human's and are reverted after + the fact if unwanted. (Matters from phase 3; this phase must not block it.) +- Markdown is deprecated. Nothing new is built for Markdown content. +- A CLI save-and-publish publishes only what the CLI changed on top of the + published version, never the owner's unfinished work in the admin. The CLI + change is also merged into the working copy, so a later publish from the + admin keeps it. +- The author asked for a prototype first. It was built and confirmed the + interaction (see "Prototype findings"). + +## Goals + +- Every Lexical content document (post, note, page) has exactly one Loro doc, + stored in PostgreSQL, which is the source of truth for its working content. +- The admin write page edits through the Loro binding. Saving never conflicts. + Two admin tabs (or devices) editing the same document converge without a + dialog. +- History is one linear timeline per document: checkpoints (saves, publishes, + restores, undos, imports) plus the Loro change log, attributed by peer. +- Restore a checkpoint, and undo a single step, both as new commits. +- Everything outside the editor keeps working on materialized snapshots: + publish, share links, Yohaku `/preview/:token`, the dashboard desk, AI cover, + file references, the CLI. +- No existing revision is lost. + +## Non-goals + +- Realtime transport, presence, cursors (phase 2). Phase 1 syncs over HTTP. +- Server-side AI peer, co-authors, invites, reader auth (phases 3 and 4). +- Shallow snapshots / history trimming. +- Replaying pre-existing revisions into the Loro oplog. Old revisions stay as + checkpoints with materialized content only. +- Any change to Markdown documents. +- Changing the public API consumed by Yohaku (`GET /drafts/shared/:token`). + +## Anti-goals + +- Publishing content that differs from what the owner saw when they pressed + publish. +- A document whose Loro doc and visible checkpoints disagree with no way back + (every import or legacy write must leave a checkpoint). +- Losing an edit during the rolling-deploy window, when old pods still accept + legacy `PUT /drafts/:id` saves. +- A second history system creeping back (e.g. the client keeping its own + revision chain or merge logic). + +## Prototype findings + +Throwaway branch `spike/collab-room` (worktree `../mx-core-spike-collab`) +extended the `mxs author` local server into a multi-client room. Verified with +two independent browsers plus an agent editing the file on disk: + +- Two people typing in different paragraphs converge; an agent rewrite merged + while a human kept typing, both kept. +- Single-step undo of the agent turn: on a fork, + `fork.applyDiff(doc.diff([lastIdOfStep], depsOfStep), { fullState: true })`, + commit, export the update and import it into the main doc. The agent's + paragraph came back; human edits made before and after it survived; both + clients synced; the file persisted. This is the undo used below. +- Not covered: undoing a grouped row when another peer's changes interleave + inside the group (they would be reverted too). +- The spike's agent writes through `streamAtVersion`, which forks and + therefore gets a fresh PeerID per turn. Phase 3 must give each agent session + a fixed PeerID. Not a phase 1 concern, but phase 1 must store commit + messages and peers so phase 3 can attribute. + +## Design + +### What lives in the Loro doc + +One `LoroDoc` per `content_documents` row: + +- `lexical` tree — the body, via `createLoroBinding` (`TREE_NAME`). +- `title` — `LoroText`. + +Nothing else. `meta` and `typeSpecificData` (slug, category, tags, note +`password`, paywall settings in post `meta`, …) stay **out** of the CRDT and +live on the working copy (see "Metadata fields"). (Designer's call.) Reason: +Loro history cannot be redacted, and from phase 4 co-authors sync the whole +doc, so every value ever stored in it would reach them; co-authors also do not +edit metadata. Deciding this now avoids rewriting history later. + +`text` (Markdown rendering) and `images` are not stored; they are derived when +materializing. `contentFormat` is always `lexical`. + +### Metadata fields + +- New nullable column `content_crdt_docs.fields jsonb`: the working copy's + current `{ meta, typeSpecificData }`, normalized with `canonicalSnapshot` + rules. +- Written by the owner through a JSON endpoint that takes only the keys that + changed and merges them per top-level key (last writer wins per key), so + two tabs editing different fields do not clobber each other. +- Every checkpoint copies `fields` into its `meta` / `typeSpecificData` + columns, so restore, compare, share and publish see them as today. +- Metadata changes appear in the timeline as checkpoints, not as Loro steps, + and are not undoable step by step. Restoring a checkpoint restores its + fields too. +- **Edits made outside drafts flow in.** `PATCH /posts/:id` (`categoryId`, + `pinAt`), `PATCH /notes/:id` (`mood`, `weather`, `topicId`, `bookmark`) and + the AI slug backfill change the live article directly. When the document + has a working copy, the same service call also merges those keys into + `fields`. Otherwise the long-lived working copy would undo them on the next + publish. (Designer's call.) + +Commit messages carry attribution: `human:`, `agent:`, +`restore `, `undo @`, `import `, +`legacy-save `. Phase 1 only produces the owner's `human:` plus +the server-side kinds; the format must not need changing for phases 3 and 4. + +### Storage + +All additive (single release, expand only): + +- `content_crdt_docs`: `document_id` (PK, FK → `content_documents`, cascade), + `snapshot bytea`, `snapshot_version bytea` (encoded version vector covered + by the snapshot), `synced_revision_id` (FK → `content_revisions`, the + checkpoint whose content the doc last matched or produced), `fields jsonb` + (see "Metadata fields"), `updated_at`. +- `content_crdt_updates`: `id` (snowflake), `document_id` (FK, cascade), + `bytes bytea`, `created_at`. Append-only. Loro merges are commutative, so + two replicas appending concurrently need no lock. +- Compaction: when a document's pending updates pass a threshold, a replica + takes a PostgreSQL advisory lock on the document, imports snapshot + updates, + writes a new snapshot and deletes exactly the update rows it folded in. + Loading is snapshot + every remaining update row. Do not filter by + `id >` some maximum: snowflake ids from two replicas do not commit in id + order, so a lower id can commit after compaction. Re-importing an update + Loro already has is a no-op. +- `content_revisions`: add nullable `frontiers jsonb` and `kind text` + (`save | publish | restore | undo | import | legacy-save`). Existing rows keep + `frontiers = null`, `kind = null` (read as legacy). `parent_revision_id` + links each new checkpoint to the previous one, so the chain is linear from + here on. + +### The working copy + +Most consumers key on a branch id: publish jobs (`branchId`), file references +(`FileReferenceType.Draft`), share links in follow mode (`draft_id`), the +`?draftId=` URL, AI cover, content migration, the desk, the CLI. To keep this +phase's blast radius small, the `drafts` table stays, with a new invariant: + +- **At most one active `drafts` row per document: the working copy.** Its id + is the stable handle for the Loro doc. Its `head_revision_id` is the latest + checkpoint. `base_revision_id` is kept as-is and no longer means anything. +- Existing documents with several active branches: on first load through the + new path, the most recently updated active branch becomes the working copy; + the others are archived. Their revisions remain and are restorable from the + timeline. A follow-mode share pointing at an archived branch is converted to + a pinned share at that branch's head, so its readers keep seeing the same + content. (Designer's call.) +- A document with no active branch (all archived, or published-only) gets a + new working copy seeded from the published revision on first load. + +**Discarding the draft.** `DELETE /drafts/:id` (write page, drafts list bulk +delete, `mxs draft delete`) on a working copy restores the published +checkpoint as a new commit (`restore`, as below) instead of archiving: the +working copy goes back to what is live, and the discarded work stays in the +timeline. For a document that was never published, it archives the working +copy and the document drops out of draft lists, as today. (Designer's call, +following "revert is a new commit, nothing is lost".) + +Renaming or retiring the `drafts` table is a later cleanup, not this phase. + +### Checkpoints + +A checkpoint is a `content_revisions` row with `frontiers` set and the full +materialized snapshot (title, text, content, images, meta, typeSpecificData). +Everything that reads revisions today keeps reading these rows. + +Created by: + +- The write page: on ⌘S, and on the existing 10-second idle timer (which today + does `PUT /drafts/:id`). Skipped when the materialized content equals the + previous checkpoint (`sameRevisionContent`). +- The server: before a publish job, on restore, on undo, on import, on a + legacy write. + +File references for the working copy are refreshed when a checkpoint is +created, not on every update. + +### Materializing on the server + +`apps/core` gains `loro-crdt` (Node build) and `@haklex/rich-collab-loro`, and +binds a headless Lexical editor (from `@haklex/rich-headless`) to the doc to +read the body as Lexical JSON, then derives `text` with the existing +`helper.lexical.service` `$toMarkdown` path. + +Risk the builder must cover: the headless node set must register every node +type the admin editor can produce (including admin-only ones such as +`CatalogDynamicEditNode` (`dynamic`) and `AgentDiffEditNode`), or binding and +import fail. `mxs author`'s `createAuthorHeadlessEditor` already solves this +for the CLI and is the reference. + +### Sync API (phase 1, HTTP) + +Owner-only (`@Auth()`), binary bodies, `@HTTPDecorators.RawResponse`: + +- `GET /drafts/:id/crdt` → full snapshot. +- `GET /drafts/:id/crdt?since=` → updates the caller + lacks. +- `POST /drafts/:id/crdt` (octet-stream) → import. The server validates on a + fork before appending: size limit, the update does not add a second root to + the `lexical` tree (the `mxs author` lineage check), and the result still + materializes. Rejected updates return an `AppException`. + +A rejected update is already in the client's local doc, and every later local +update depends on it, so that tab can never sync again. On rejection the +client reloads the doc from the server and re-applies its own materialized +content as one legacy write (below), then continues on the fresh doc. The +user's text survives; their step history for that session collapses into one +`legacy-save`. (Designer's call.) + +No client-side persistence of unsent updates (IndexedDB) in phase 1. Updates +leave the browser in ~100 ms batches, which is more durable than today's +10-second `PUT`. A failed send stays queued in memory and is retried, and the +page warns on unload while anything is unsent, as `mxs author` does. + +After a successful append the server emits the existing `draft.update` admin +broadcast (payload unchanged in shape; `headRevisionId` is the latest +checkpoint). Other tabs react by pulling `?since=`. Phase 2 replaces both the +POST/GET pair and this signal with `collab.*` frames. + +Endpoint names are the builder's call; the semantics are not. + +### Legacy whole-document writes + +`POST /drafts` and `PUT /drafts/:id` keep their request shapes, because the CLI +(and old admin builds during cutover) use them. They become "apply this whole +snapshot to the working copy at a known version": + +- `PUT /drafts/:id {data, expectedHeadRevisionId}`: take the frontiers of the + `expectedHeadRevisionId` checkpoint and apply the body there with + `editAtVersion` (a three-way merge against everything since). `editAtVersion` + only touches the `lexical` tree, so the title is applied at the same version + separately. For metadata, merge only the keys whose value differs from that + checkpoint into `fields`. Commit as `legacy-save`, create a checkpoint, + return the working copy view. `DRAFT_HEAD_CONFLICT` is no longer thrown. +- A legacy write carrying `contentFormat: markdown` to a document on the Loro + path is rejected (400). (Designer's call.) +- If the base checkpoint has no frontiers (a legacy revision), apply at the + doc's current frontiers instead (last writer wins for that write). +- `POST /drafts` with a `refId` or `baseRevisionId` on a document that already + has a working copy writes into it the same way, with the base checkpoint as + the version. Without a document it creates the document, the working copy + and the Loro doc. + +**A legacy write produces two checkpoints, and returns the first.** The CLI +saves `published + patch` with `baseRevisionId = published` +(`cli/draft/_shared.ts:124`), then publishes the returned `headRevisionId` +(`publishSavedDraft`, `:198`; used by `note edit --publish` and friends). The +author decided that this must publish only what the CLI changed, never the +owner's unfinished work in the working copy, while the working copy must also +keep the CLI's change. So every legacy write: + +1. Creates **the write's own checkpoint**: the base plus this write, nothing + else. When the base checkpoint has frontiers, it is the fork state right + after `editAtVersion` at those frontiers. The ops are real, so the + checkpoint stores those frontiers and is a genuine version of the doc. + When the base has no frontiers (a legacy revision), the server builds the + content from the base's materialized content plus the write without Loro, + and the checkpoint stores no frontiers. Either way it carries the full + materialized snapshot, so publish can use it. +2. Imports the same ops into the doc, which merges them with everything since, + and creates a `legacy-save` checkpoint at the new head. That checkpoint + becomes the working copy's `head_revision_id`. +3. Returns the working copy view with `headRevision` / `headRevisionId` set to + checkpoint 1, the revision this write produced. This differs from the + working copy's stored head, and the response documents it as such. + +The CLI therefore keeps its request shapes and publishes exactly +`published + patch`. The owner's unpublished work stays unpublished and now +includes the CLI change. The existing `newerDraftChanges` report then tells +the publish caller that the working copy holds more than what went live. + +### Lazy import and the cutover window + +No batch migration. A document gets its Loro doc the first time the new path +touches it (open in the write page, a legacy write, a publish): + +1. Pick the working copy (above). +2. Create the doc from its head revision's content in one commit, + `import `, and set `synced_revision_id`. + +During the rolling deploy, old pods still accept `PUT /drafts/:id` and move the +branch head without touching the Loro doc. So on every load, if the working +copy's `head_revision_id` is a revision with no frontiers and differs from +`synced_revision_id`, the server applies that revision's content as a legacy +write at the synced checkpoint's frontiers, then updates `synced_revision_id`. +This closes the window without coordination. + +### History, restore and undo + +- `GET /drafts/:id/history` returns the timeline: checkpoints (all kinds, + including legacy revisions without frontiers) merged with the Loro change + log (peer, counter, lamport, timestamp, commit message), newest first. Human + changes group by author and idle gap, agent changes by turn, as in the + `mxs author` history panel. +- **Restore a checkpoint**: `editAtVersion(doc, doc.frontiers(), …, checkpoint + content)`, commit `restore `, new checkpoint of kind `restore`. Works for + legacy checkpoints too, because it uses content, not frontiers. +- **Undo one step**: the fork + `applyDiff` method from the prototype, commit + `undo …`, new checkpoint of kind `undo`. Offered for timeline entries that + come from the change log. +- `GET /drafts/compare/:leftId/:rightId` keeps working on checkpoint content. + `relation` is now `same`, `ancestor` or `descendant` along the linear chain; + `diverged` can only come from archived legacy branches. + +### Publish + +`POST /publish-jobs` keeps its shape. The write page creates a `publish` +checkpoint at the current head and passes it as `revisionId`, with +`branchId` = the working copy. Changes in `publish.service.ts`: + +- Drop the `headRevisionId === revisionId` check (`:82`). Any checkpoint of the + document may be published; edits made after it are reported through the + existing `hasNewerBranchHead` / `newerDraftChanges` path. +- `confirmDiverged` is accepted and ignored for working copies. +- `PUBLISHED_REVISION_CHANGED` stays. + +The snapshot frozen into the job is the checkpoint's materialized content, so +what is published is exactly what was checkpointed. + +### Share links + +No schema change. Pinned shares point at a checkpoint. Follow shares point at +the working copy row and resolve to its `head_revision_id`, the latest +checkpoint, which the idle timer keeps fresh. Yohaku and `api-client` are +untouched. + +### Admin write page + +For Lexical documents: + +- The editor binds to the Loro doc (a `LoroSyncPlugin` like `apps/admin/src/author/`, + using the HTTP endpoints above). The title binds to the `title` text. The + metadata form saves changed keys through the fields endpoint. +- The save layer stops sending snapshots. The 10-second timer and ⌘S create + checkpoints instead. +- Removed: `DRAFT_HEAD_CONFLICT` handling and `reconcileWithRemote`, + `merge-draft-conflict.ts`, `DraftConflictDialog`, the `remoteUpdateStale` + banner, `DraftRecoveryReview`, branch selection (`availableDraft`, + `continueDraft` / `applyDraft`), the `?baseRevisionId` fork-on-save flow. +- `VersionTreePanel` and `version-tree/*` are replaced by a timeline panel with + restore and undo-this-step, plus the existing share bar. +- `features/drafts/*`: the list shows one row per document; "Restore" restores + in place instead of opening a fork. +- The dashboard desk keeps reading `relationToPublished` and `headRevision`. + +The write-page AI agent's accept/reject review goes away in this phase. Its +pending `agent-diff` nodes are stripped before every save today +(`stripPendingAgentDiffs`, WRVC:5109) because they must never be persisted; +with the editor bound to Loro, every editor update is persisted and synced, so +those nodes cannot exist. Agent operations (`insert` / `replace` / `delete`) +apply straight to the editor, inside one Loro commit whose message is +`agent:` (set the next commit message before applying), so the turn +shows as one undoable step. This is the author's "AI edits land directly" +decision brought forward to phase 1 for the client-side agent. (Designer's +call.) The agent loop moving to the server (parallel Write Agent WS design) +does not change this: whoever applies the operation commits it as one +`agent:` step. + +**Markdown boundary.** (Designer's call throughout; Markdown gets no new work.) + +- The signal is the working copy head's `contentFormat`. Lexical → Loro path. + Markdown → the current flow, unchanged, including several branches and + `DRAFT_HEAD_CONFLICT`. The legacy endpoints therefore behave by format; the + one-working-copy rule applies to Lexical documents. +- The staged conversion (published article is Markdown, draft is Lexical) is + on the Loro path, because the working copy is Lexical. The migration commit + currently requires the branch head and its hash to be unchanged + (`content-migration-commit.service.ts:331`). Idle checkpoints move the head, + so that precondition must check the checkpoint being published instead of + the live head. +- Markdown code paths are removed when Markdown is removed, not here. + +### CLI and other server consumers + +- CLI: no request-shape change. Legacy writes return the write's own + checkpoint, so CLI save-and-publish publishes only the CLI's change (above). + `post/apply.ts`'s "more than one active branch" failure can no longer trigger + after import. +- `ai-image` cover, aggregate desk, content migration, delete cascades + (`deleteByRef`): unchanged, they read the working copy row and checkpoints. +- `file-reference-usage.repository.ts` keeps scanning `content_revisions`. + +## Constraints + +- Rolling deploy with two replicas: everything in this phase is additive. No + drop, rename or type change. The `drafts` table and its columns stay. +- Schema migrations follow the `mx-migration-author` skill and pass + `lint:migrations`. +- API envelope rules from `AGENTS.md`: binary endpoints use `RawResponse`; JSON + endpoints return bare values or `withMeta`. +- `loro-crdt` must load in the core runtime (Node, wasm) and in the Docker + image. + +## Left to the builder + +- Endpoint paths and the exact checkpoint skip rule. +- Compaction thresholds and whether to cache loaded docs in memory per pod + (if cached, reload updates `id >` last seen before each read). +- Timeline panel visuals. +- How the admin decides "this document uses the Loro path" (presence of a + `content_crdt_docs` row after import is the obvious signal). +- Tests: repository specs for the new tables; e2e for sync, legacy writes at a + version, lazy import, the cutover reconciliation, restore, undo, publish of a + non-head checkpoint; admin component tests replacing the conflict-merge ones. diff --git a/packages/db-schema/src/schema/content.ts b/packages/db-schema/src/schema/content.ts index 26f40f5daf0..069e8b35edf 100644 --- a/packages/db-schema/src/schema/content.ts +++ b/packages/db-schema/src/schema/content.ts @@ -3,6 +3,7 @@ import type { AnyPgColumn } from 'drizzle-orm/pg-core' import { boolean, check, + customType, index, integer, jsonb, @@ -15,6 +16,12 @@ import { import { readers } from './auth' import { createdAt, pkText, refText, tsCol, updatedAt } from './columns' +const bytea = customType<{ data: Uint8Array; driverData: Buffer }>({ + dataType: () => 'bytea', + toDriver: (value) => Buffer.from(value), + fromDriver: (value) => new Uint8Array(value), +}) + export const categories = pgTable( 'categories', { @@ -259,6 +266,10 @@ export const contentRevisions = pgTable( string, unknown > | null>(), + frontiers: jsonb('frontiers').$type< + { counter: number; peer: `${number}` }[] | null + >(), + kind: text('kind'), }, (table) => [ index('content_revisions_document_idx').on(table.documentId), @@ -346,6 +357,38 @@ export const contentPublicationEvents = pgTable( ], ) +export const contentCrdtDocs = pgTable('content_crdt_docs', { + documentId: refText('document_id') + .primaryKey() + .notNull() + .references(() => contentDocuments.id, { onDelete: 'cascade' }), + snapshot: bytea('snapshot').notNull(), + snapshotVersion: bytea('snapshot_version').notNull(), + syncedRevisionId: refText('synced_revision_id').references( + () => contentRevisions.id, + { onDelete: 'set null' }, + ), + fields: jsonb('fields').$type<{ + meta: Record | null + typeSpecificData: Record | null + } | null>(), + createdAt: createdAt(), + updatedAt: updatedAt(), +}) + +export const contentCrdtUpdates = pgTable( + 'content_crdt_updates', + { + id: pkText(), + documentId: refText('document_id') + .notNull() + .references(() => contentDocuments.id, { onDelete: 'cascade' }), + bytes: bytea('bytes').notNull(), + createdAt: createdAt(), + }, + (table) => [index('content_crdt_updates_document_idx').on(table.documentId)], +) + /** * Self-referential thread structure plus polymorphic ref to content (Post/Note/Page/Recently). */ diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6fe7e4674ae..908abf4c898 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -718,6 +718,9 @@ importers: '@fastify/static': specifier: 10.1.5 version: 10.1.5 + '@haklex/rich-collab-loro': + specifier: 0.54.4 + version: 0.54.4(lexical@0.52.0(typescript@6.0.3))(loro-crdt@1.16.4) '@haklex/rich-headless': specifier: 0.54.4 version: 0.54.4(lexical@0.52.0(typescript@6.0.3))(typescript@6.0.3) @@ -886,6 +889,9 @@ importers: linkedom: specifier: 0.18.13 version: 0.18.13 + loro-crdt: + specifier: ^1.16.4 + version: 1.16.4 lru-cache: specifier: 11.5.3 version: 11.5.3 From a72d3cbad66c5698f66da60e2cf5c4844a968105 Mon Sep 17 00:00:00 2001 From: Innei Date: Sun, 25 Oct 2026 15:50:55 +0800 Subject: [PATCH 02/19] feat(admin): edit Lexical drafts through the shared Loro doc The write page binds the editor to the draft's Loro doc, syncs over HTTP, patches fields separately and replaces the branch picker with a checkpoint timeline that can restore and undo. Agent edits land directly. --- apps/admin/src/api/drafts.ts | 63 +++- apps/admin/src/api/http.ts | 32 ++ .../drafts/components/DraftDetail.tsx | 21 +- .../write/components/TimelinePanel.test.tsx | 144 +++++++ .../write/components/TimelinePanel.tsx | 284 ++++++++++++++ .../components/WriteRouteViewsContent.tsx | 354 ++++++++++++++---- .../timeline/build-timeline.test.ts | 97 +++++ .../components/timeline/build-timeline.ts | 96 +++++ .../write/crdt/crdt-draft-session.test.ts | 185 +++++++++ .../features/write/crdt/crdt-draft-session.ts | 95 +++++ .../src/features/write/crdt/crdt-sync.test.ts | 208 ++++++++++ .../src/features/write/crdt/crdt-sync.ts | 122 ++++++ .../features/write/crdt/draft-fields.test.ts | 38 ++ .../src/features/write/crdt/draft-fields.ts | 38 ++ .../write/crdt/resync-rejected.test.ts | 30 ++ .../features/write/crdt/resync-rejected.ts | 13 + .../write/crdt/use-crdt-agent-turns.test.ts | 54 +++ .../write/crdt/use-crdt-agent-turns.ts | 61 +++ .../write/crdt/use-crdt-draft.test.ts | 92 +++++ .../src/features/write/crdt/use-crdt-draft.ts | 141 +++++++ apps/admin/src/i18n/resources/en-US.ts | 31 +- apps/admin/src/i18n/resources/zh-CN.ts | 28 +- apps/admin/src/models/draft.ts | 39 ++ apps/admin/src/query/keys.ts | 1 + apps/admin/vite.config.mts | 3 + 25 files changed, 2199 insertions(+), 71 deletions(-) create mode 100644 apps/admin/src/features/write/components/TimelinePanel.test.tsx create mode 100644 apps/admin/src/features/write/components/TimelinePanel.tsx create mode 100644 apps/admin/src/features/write/components/timeline/build-timeline.test.ts create mode 100644 apps/admin/src/features/write/components/timeline/build-timeline.ts create mode 100644 apps/admin/src/features/write/crdt/crdt-draft-session.test.ts create mode 100644 apps/admin/src/features/write/crdt/crdt-draft-session.ts create mode 100644 apps/admin/src/features/write/crdt/crdt-sync.test.ts create mode 100644 apps/admin/src/features/write/crdt/crdt-sync.ts create mode 100644 apps/admin/src/features/write/crdt/draft-fields.test.ts create mode 100644 apps/admin/src/features/write/crdt/draft-fields.ts create mode 100644 apps/admin/src/features/write/crdt/resync-rejected.test.ts create mode 100644 apps/admin/src/features/write/crdt/resync-rejected.ts create mode 100644 apps/admin/src/features/write/crdt/use-crdt-agent-turns.test.ts create mode 100644 apps/admin/src/features/write/crdt/use-crdt-agent-turns.ts create mode 100644 apps/admin/src/features/write/crdt/use-crdt-draft.test.ts create mode 100644 apps/admin/src/features/write/crdt/use-crdt-draft.ts diff --git a/apps/admin/src/api/drafts.ts b/apps/admin/src/api/drafts.ts index d2e07bca651..5b657c71cce 100644 --- a/apps/admin/src/api/drafts.ts +++ b/apps/admin/src/api/drafts.ts @@ -1,6 +1,9 @@ import type { Image, PaginateResult } from '~/models/base' import type { ContentRevision, + CrdtOpId, + DraftFields, + DraftHistory, DraftModel, DraftRefType, DraftShare, @@ -9,7 +12,14 @@ import type { VersionContext, } from '~/models/draft' -import { deleteJson, getJson, postJson, putJson } from './http' +import { + deleteJson, + getJson, + patchJson, + postJson, + putJson, + requestBytes, +} from './http' export type DraftSortOrder = 'asc' | 'desc' @@ -112,3 +122,54 @@ export function updateDraft(id: string, data: UpdateDraftData) { export function deleteDraft(id: string) { return deleteJson<{ success: boolean }>(`/drafts/${id}`) } + +const encodeBase64 = (bytes: Uint8Array) => + btoa(Array.from(bytes, (byte) => String.fromCharCode(byte)).join('')) + +export function getDraftCrdt(id: string, since?: Uint8Array) { + const query = since ? `?since=${encodeURIComponent(encodeBase64(since))}` : '' + return requestBytes(`/drafts/${id}/crdt${query}`, { method: 'GET' }) +} + +export async function postDraftCrdt(id: string, bytes: Uint8Array) { + await requestBytes(`/drafts/${id}/crdt`, { + body: bytes as Uint8Array, + headers: { 'content-type': 'application/octet-stream' }, + method: 'POST', + }) +} + +export function createDraftCheckpoint(id: string, kind: 'publish' | 'save') { + return postJson( + `/drafts/${id}/checkpoints`, + { kind }, + ) +} + +export function updateDraftFields( + id: string, + patch: Partial>>, +) { + return patchJson(`/drafts/${id}/fields`, patch) +} + +export function getDraftHistory(id: string) { + return getJson(`/drafts/${id}/history`) +} + +export function restoreDraftRevision(id: string, revisionId: string) { + return postJson(`/drafts/${id}/restore`, { + revisionId, + }) +} + +export function undoDraftChange( + id: string, + change: CrdtOpId, + before: CrdtOpId[], +) { + return postJson( + `/drafts/${id}/undo`, + { before, id: change }, + ) +} diff --git a/apps/admin/src/api/http.ts b/apps/admin/src/api/http.ts index ab189f88504..71694abd438 100644 --- a/apps/admin/src/api/http.ts +++ b/apps/admin/src/api/http.ts @@ -111,6 +111,38 @@ export async function requestJson( return responseData as TResponse } +export async function requestBytes( + path: string, + init: RequestInit = {}, +): Promise { + const response = await fetch(`${API_URL}${path}`, { + credentials: 'include', + ...init, + headers: buildAdminRequestHeaders(init.headers), + }) + + if (!response.ok) { + const responseData = normalizeResponseData( + camelcaseKeys(await readResponseData(response)), + ) + if (isUnauthorizedResponse(response, responseData)) { + handleUnauthorized() + } + const message = + responseData?.error?.message || + responseData?.message || + response.statusText + throw new ApiRequestError( + Array.isArray(message) ? message.join(', ') : message || 'Request failed', + responseData?.error?.code ?? responseData?.code, + response.status, + responseData?.error?.details, + ) + } + + return new Uint8Array(await response.arrayBuffer()) +} + export async function requestBlob( path: string, init: RequestInit = {}, diff --git a/apps/admin/src/features/drafts/components/DraftDetail.tsx b/apps/admin/src/features/drafts/components/DraftDetail.tsx index 479074dcb16..007f0eb8750 100644 --- a/apps/admin/src/features/drafts/components/DraftDetail.tsx +++ b/apps/admin/src/features/drafts/components/DraftDetail.tsx @@ -1,9 +1,10 @@ -import { useQuery } from '@tanstack/react-query' +import { useMutation, useQuery } from '@tanstack/react-query' import { ChevronLeft, GitCompare, Loader2, Pencil, Trash2 } from 'lucide-react' import { useEffect, useMemo, useState } from 'react' import { useNavigate } from 'react-router' +import { toast } from 'sonner' -import { getDraftRevisions } from '~/api/drafts' +import { getDraftRevisions, restoreDraftRevision } from '~/api/drafts' import { APP_SHELL_HEADER_HEIGHT_CLASS } from '~/constants/layout' import { useI18n } from '~/i18n' import type { DraftModel } from '~/models/draft' @@ -47,6 +48,13 @@ export function DraftDetail(props: { null, ) const versionsScopeId = `draft-versions-${props.draft.id}` + const restoresInPlace = props.draft.headRevision.contentFormat === 'lexical' + const restoreMutation = useMutation({ + mutationFn: (revisionId: string) => + restoreDraftRevision(props.draft.id, revisionId), + onError: () => toast.error(t('write.timeline.failed')), + onSuccess: (draft) => navigate(getEditPathForDraft(draft)), + }) useScopeArrowNav({ itemSelector: '[data-scope-item="row"]', @@ -164,10 +172,15 @@ export function DraftDetail(props: { item={item} key={item.id} onRestore={() => - navigate(getEditPathForRevision(props.draft, item.id)) + restoresInPlace + ? restoreMutation.mutate(item.id) + : navigate(getEditPathForRevision(props.draft, item.id)) } onSelect={() => setSelectedRevisionId(item.id)} - restorePending={false} + restorePending={ + restoreMutation.isPending && + restoreMutation.variables === item.id + } selected={selectedRevisionId === item.id} /> ))} diff --git a/apps/admin/src/features/write/components/TimelinePanel.test.tsx b/apps/admin/src/features/write/components/TimelinePanel.test.tsx new file mode 100644 index 00000000000..62c75a8ffa8 --- /dev/null +++ b/apps/admin/src/features/write/components/TimelinePanel.test.tsx @@ -0,0 +1,144 @@ +import { QueryClient, QueryClientProvider } from '@tanstack/react-query' +import { act, createElement } from 'react' +import type { Root } from 'react-dom/client' +import { createRoot } from 'react-dom/client' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +import { + getDraftHistory, + getDraftShare, + restoreDraftRevision, + undoDraftChange, +} from '~/api/drafts' +import { I18nProvider } from '~/i18n' +import type { DraftHistory } from '~/models/draft' + +import { TimelinePanel } from './TimelinePanel' + +vi.mock('~/api/drafts', () => ({ + deleteDraftShare: vi.fn(), + getDraftHistory: vi.fn(), + getDraftShare: vi.fn(), + restoreDraftRevision: vi.fn(), + setDraftShare: vi.fn(), + undoDraftChange: vi.fn(), +})) +vi.mock('~/ui/feedback/confirm', () => ({ + confirmDialog: vi.fn(async () => true), +})) + +const history: DraftHistory = { + changes: [ + { + deps: [{ counter: 3, peer: '1' }], + id: { counter: 4, peer: '1' }, + length: 1, + message: 'human:owner', + timestamp: 1_792_900_000, + }, + ], + checkpoints: [ + { + createdAt: '2026-10-25T00:00:00.000Z', + frontiers: [], + id: 'rev-1', + kind: 'save', + parentRevisionId: null, + title: 'Saved', + }, + ], +} as unknown as DraftHistory + +let container: HTMLDivElement +let root: Root +const calls: string[] = [] +const onBeforeChange = vi.fn(async () => { + calls.push('flush') +}) +const onChanged = vi.fn(async (change: { restored: boolean }) => { + calls.push(change.restored ? 'reload-fields' : 'pull') +}) + +const render = async () => { + const client = new QueryClient({ + defaultOptions: { queries: { retry: false } }, + }) + await act(async () => { + root.render( + createElement( + QueryClientProvider, + { client }, + createElement( + I18nProvider, + null, + createElement(TimelinePanel, { + documentId: null, + draftId: 'd1', + onBeforeChange, + onChanged, + onClose: () => undefined, + publishedRevisionId: null, + }), + ), + ), + ) + }) + await act(async () => { + await Promise.resolve() + }) +} + +const click = async (label: string) => { + const button = await vi.waitFor(() => { + const found = container.querySelector( + `button[aria-label="${label}"]`, + ) + expect(found).not.toBeNull() + return found! + }) + await act(async () => { + button.click() + }) + await vi.waitFor(() => expect(onChanged).toHaveBeenCalled()) +} + +beforeEach(() => { + calls.length = 0 + onChanged.mockClear() + container = document.createElement('div') + document.body.append(container) + root = createRoot(container) + vi.mocked(getDraftHistory).mockResolvedValue(history) + vi.mocked(getDraftShare).mockResolvedValue(null as never) + vi.mocked(restoreDraftRevision).mockImplementation(async () => { + calls.push('restore') + return {} as never + }) + vi.mocked(undoDraftChange).mockImplementation(async () => { + calls.push('undo') + return {} as never + }) +}) + +afterEach(() => { + act(() => root.unmount()) + document.body.innerHTML = '' +}) + +describe('TimelinePanel', () => { + it('sends unsent edits before restoring and reloads fields after', async () => { + await render() + + await click('恢复到此版本') + + expect(calls).toEqual(['flush', 'restore', 'reload-fields']) + }) + + it('sends unsent edits before undoing a step', async () => { + await render() + + await click('撤销此步') + + expect(calls).toEqual(['flush', 'undo', 'pull']) + }) +}) diff --git a/apps/admin/src/features/write/components/TimelinePanel.tsx b/apps/admin/src/features/write/components/TimelinePanel.tsx new file mode 100644 index 00000000000..40b26fb5f7a --- /dev/null +++ b/apps/admin/src/features/write/components/TimelinePanel.tsx @@ -0,0 +1,284 @@ +import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' +import { History, Loader2, RotateCcw, Undo2 } from 'lucide-react' +import { useEffect, useMemo } from 'react' +import { toast } from 'sonner' + +import type { SetDraftShareData } from '~/api/drafts' +import { + deleteDraftShare, + getDraftHistory, + getDraftShare, + restoreDraftRevision, + setDraftShare, + undoDraftChange, +} from '~/api/drafts' +import { useI18n } from '~/i18n' +import type { TranslationKey } from '~/i18n/types' +import type { CheckpointKind, DraftShare } from '~/models/draft' +import { adminQueryKeys } from '~/query/keys' +import { subscribeDraftUpdate } from '~/socket/draft-update-signal' +import { confirmDialog } from '~/ui/feedback/confirm' +import { AsidePanel } from '~/ui/layout/content-layout' +import { EmptyState } from '~/ui/patterns/EmptyState' +import { Button } from '~/ui/primitives/button' +import { Scroll } from '~/ui/primitives/scroll' +import { cn } from '~/utils/cn' + +import { + buildTimeline, + type StepKind, + type TimelineCheckpoint, + type TimelineRow, + type TimelineStep, +} from './timeline/build-timeline' +import { ShareBar } from './version-tree/ShareBar' + +const ROW_LIMIT = 200 + +const checkpointLabels: Record = { + import: 'write.timeline.checkpoint.import', + legacy: 'write.timeline.checkpoint.legacy', + 'legacy-save': 'write.timeline.checkpoint.legacySave', + publish: 'write.timeline.checkpoint.publish', + restore: 'write.timeline.checkpoint.restore', + save: 'write.timeline.checkpoint.save', + undo: 'write.timeline.checkpoint.undo', +} + +const stepLabels: Record = { + agent: 'write.timeline.step.agent', + human: 'write.timeline.step.human', + other: 'write.timeline.step.other', + restore: 'write.timeline.step.restore', + undo: 'write.timeline.step.undo', +} + +export interface TimelinePanelProps { + documentId: string | null + draftId: string + onBeforeChange: () => Promise + onChanged: (change: { restored: boolean }) => Promise | void + onClose: () => void + publishedRevisionId: string | null +} + +export function TimelinePanel(props: TimelinePanelProps) { + const { format, t } = useI18n() + const queryClient = useQueryClient() + const { draftId, documentId } = props + const historyKey = adminQueryKeys.drafts.timeline(draftId) + const historyQuery = useQuery({ + enabled: Boolean(draftId), + queryFn: () => getDraftHistory(draftId), + queryKey: historyKey, + }) + + useEffect( + () => + subscribeDraftUpdate((payload) => { + if (payload.branchId !== draftId) return + void queryClient.invalidateQueries({ + queryKey: adminQueryKeys.drafts.timeline(draftId), + }) + }), + [draftId, queryClient], + ) + + const rows = useMemo( + () => + historyQuery.data + ? buildTimeline(historyQuery.data, props.publishedRevisionId).slice( + 0, + ROW_LIMIT, + ) + : [], + [historyQuery.data, props.publishedRevisionId], + ) + + const afterChange = async (restored: boolean) => { + await props.onChanged({ restored }) + await queryClient.invalidateQueries({ queryKey: historyKey }) + } + const restoreMutation = useMutation({ + mutationFn: async (revisionId: string) => { + await props.onBeforeChange() + return restoreDraftRevision(draftId, revisionId) + }, + onError: () => toast.error(t('write.timeline.failed')), + onSuccess: async () => { + await afterChange(true) + toast.success(t('write.timeline.restored')) + }, + }) + const undoMutation = useMutation({ + mutationFn: async (step: TimelineStep) => { + await props.onBeforeChange() + return undoDraftChange(draftId, step.change, step.before) + }, + onError: () => toast.error(t('write.timeline.failed')), + onSuccess: async () => { + await afterChange(false) + toast.success(t('write.timeline.undone')) + }, + }) + const busy = restoreMutation.isPending || undoMutation.isPending + + const restore = async (row: TimelineCheckpoint) => { + const confirmed = await confirmDialog({ + confirmText: t('write.timeline.restore'), + description: t('write.timeline.restoreDescription'), + title: t('write.timeline.restoreTitle'), + }) + if (confirmed) restoreMutation.mutate(row.id) + } + + const shareKey = adminQueryKeys.drafts.share(documentId ?? '') + const shareQuery = useQuery({ + enabled: Boolean(documentId), + queryFn: () => getDraftShare(documentId!), + queryKey: shareKey, + }) + const writeShare = (share: DraftShare | null) => { + queryClient.setQueryData(shareKey, share) + } + const setShareMutation = useMutation({ + mutationFn: (data: SetDraftShareData) => setDraftShare(documentId!, data), + onSuccess: writeShare, + }) + const revokeShareMutation = useMutation({ + mutationFn: () => deleteDraftShare(documentId!), + onSuccess: () => { + writeShare(null) + toast.success(t('write.share.revoked')) + }, + }) + const followDraft = () => setShareMutation.mutate({ draftId, mode: 'follow' }) + + return ( + + {documentId ? ( + revokeShareMutation.mutate()} + pending={setShareMutation.isPending || revokeShareMutation.isPending} + share={shareQuery.data ?? null} + /> + ) : null} + {historyQuery.isLoading ? ( +
+
+ ) : rows.length === 0 ? ( +
+ +
+ ) : ( + +
    + {rows.map((row) => ( + void restore(row as TimelineCheckpoint)} + onUndo={() => undoMutation.mutate(row as TimelineStep)} + row={row} + time={format.relativeTime(row.time)} + /> + ))} +
+ + )} + + ) +} + +const rowKey = (row: TimelineRow) => + row.type === 'checkpoint' + ? `checkpoint:${row.id}` + : `step:${row.change.peer}:${row.change.counter}` + +function TimelineItem(props: { + busy: boolean + onRestore: () => void + onUndo: () => void + row: TimelineRow + time: string +}) { + const { t } = useI18n() + const { row } = props + + if (row.type === 'checkpoint') { + return ( +
  • + +
    +
    + + {t(checkpointLabels[row.kind ?? 'legacy'])} + + {row.published ? ( + + {t('write.timeline.published')} + + ) : null} +
    +
    + {row.title || t('write.editor.untitled')} · {props.time} +
    +
    + +
    • + ) + } + + const summary = + row.kind === 'agent' ? row.message.replace(/^agent:?\s*/, '') : '' + return ( +
  • +
    +
    + {t(stepLabels[row.kind], { count: row.count, summary })} +
    +
    {props.time}
    +
    + {row.undoable ? ( + + ) : null} +
    • + ) +} diff --git a/apps/admin/src/features/write/components/WriteRouteViewsContent.tsx b/apps/admin/src/features/write/components/WriteRouteViewsContent.tsx index 583e594e8fd..e55b53ba24c 100644 --- a/apps/admin/src/features/write/components/WriteRouteViewsContent.tsx +++ b/apps/admin/src/features/write/components/WriteRouteViewsContent.tsx @@ -2,7 +2,7 @@ import { projectAgentDiffNodesToFactualState } from '@haklex/rich-ext-ai-agent/s import { analyzeMxMarkdown, mxLexicalToMarkdown } from '@mx-space/editor' import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' import { load } from 'js-yaml' -import type { SerializedEditorState } from 'lexical' +import type { LexicalEditor, SerializedEditorState } from 'lexical' import type { LucideIcon } from 'lucide-react' import { ArrowLeftRight, @@ -65,12 +65,14 @@ import type { DraftWriteData } from '~/api/drafts' import { compareRevisions, createDraft, + createDraftCheckpoint, deleteDraft, getDraftById, getDraftContext, getNewDrafts, getRevision, updateDraft, + updateDraftFields, } from '~/api/drafts' import { uploadFile, uploadFileWithProgress } from '~/api/files' import { ApiRequestError } from '~/api/http' @@ -130,8 +132,16 @@ import { import { PublishConfirmationDialog } from '~/features/write/components/PublishConfirmationDialog' import { openPublishProcessDock } from '~/features/write/components/PublishProcessDock' import { SkillPicker } from '~/features/write/components/SkillPicker' +import { TimelinePanel } from '~/features/write/components/TimelinePanel' import { TtsGenerationEntry } from '~/features/write/components/tts/TtsGenerationEntry' import { VersionTreePanel } from '~/features/write/components/VersionTreePanel' +import { diffDraftFields } from '~/features/write/crdt/draft-fields' +import { resyncRejectedDraft } from '~/features/write/crdt/resync-rejected' +import { useCrdtAgentTurns } from '~/features/write/crdt/use-crdt-agent-turns' +import { + type CrdtDraftController, + useCrdtDraft, +} from '~/features/write/crdt/use-crdt-draft' import { MetaPresetSection } from '~/features/write/meta-presets' import type { DraftMergeConflict } from '~/features/write/utils/merge-draft-conflict' import { mergeDraftConflict } from '~/features/write/utils/merge-draft-conflict' @@ -144,6 +154,7 @@ import type { TranslationKey } from '~/i18n/types' import type { Amap, AMapSearch } from '~/models/amap' import type { Image as ImageModel } from '~/models/base' import type { + DraftFields, DraftModel, RevisionSnapshot, VersionTreeNode, @@ -445,6 +456,14 @@ function WritePageRoute(props: { kind: WriteKind }) { return } +const FIELDS_RETRY_MS = 2000 + +const fieldsOf = (data: DraftWriteData): DraftFields => ({ + meta: data.meta ?? null, + typeSpecificData: (data.typeSpecificData ?? + null) as DraftFields['typeSpecificData'], +}) + function useDraftSession(options: { baseRevisionId: string | null data: DraftWriteData @@ -473,19 +492,95 @@ function useDraftSession(options: { const [conflict, setConflict] = useState(null) const [conflictDialogOpen, setConflictDialogOpen] = useState(false) const [resolvingConflict, setResolvingConflict] = useState(false) + const ackedFieldsRef = useRef(null) + const fieldsTimerRef = useRef(null) + const fieldsInFlightRef = useRef | null>(null) + const crdt = useCrdtDraft({ + draftId, + enabled: options.state.contentFormat === 'lexical', + onRejected: async () => { + if (!draftId) return + try { + const draft = await resyncRejectedDraft({ + checkpoint: () => createDraftCheckpoint(draftId, 'save'), + data: latestDataRef.current, + update: (data, expectedHeadRevisionId) => + updateDraft(draftId, { data, expectedHeadRevisionId }), + }) + acceptedDraftRef.current = draft + toast.info(t('write.toast.draftResynced')) + } catch (error) { + toast.error(getErrorMessage(error, t('write.toast.draftSaveFailed'))) + } + }, + onRemoteTitle: (title) => + options.setState((previous) => + previous.title === title ? previous : { ...previous, title }, + ), + }) + const crdtRef = useRef(crdt) + crdtRef.current = crdt + + const sendFields = useCallback(async () => { + if (fieldsTimerRef.current !== null) { + window.clearTimeout(fieldsTimerRef.current) + fieldsTimerRef.current = null + } + await fieldsInFlightRef.current + const acked = ackedFieldsRef.current + if (!crdtRef.current.active || !draftId || !acked) return + const next = fieldsOf(latestDataRef.current) + const patch = diffDraftFields(acked, next) + if (!patch) return + const request = updateDraftFields(draftId, patch).then( + () => { + ackedFieldsRef.current = next + }, + (error: unknown) => { + if (fieldsTimerRef.current === null) { + fieldsTimerRef.current = window.setTimeout(() => { + fieldsTimerRef.current = null + void sendFieldsRef.current().catch(() => undefined) + }, FIELDS_RETRY_MS) + } + throw error + }, + ) + fieldsInFlightRef.current = request.finally(() => { + fieldsInFlightRef.current = null + }) + await request + }, [draftId]) + const sendFieldsRef = useRef(sendFields) + sendFieldsRef.current = sendFields + + const flushCrdt = useCallback(async () => { + await crdtRef.current.flush() + await sendFields() + }, [sendFields]) useEffect(() => { latestDataRef.current = options.data latestFingerprintRef.current = options.fingerprint }, [options.data, options.fingerprint]) - const hasUnsavedChanges = useCallback( - () => + const hasUnsavedChanges = useCallback(() => { + if (crdtRef.current.active) { + const acked = ackedFieldsRef.current + return ( + crdtRef.current.pending() || + fieldsTimerRef.current !== null || + fieldsInFlightRef.current !== null || + (acked !== null && + diffDraftFields(acked, fieldsOf(latestDataRef.current)) !== null) + ) + } + return ( dirtyRef.current && options.hasContent && - latestFingerprintRef.current !== acceptedFingerprintRef.current, - [options.hasContent], - ) + latestFingerprintRef.current !== acceptedFingerprintRef.current + ) + }, [options.hasContent]) const accept = useCallback( ( @@ -498,6 +593,13 @@ function useDraftSession(options: { }, ) => { acceptedDraftRef.current = draft + ackedFieldsRef.current = draft + ? { + meta: draft.headRevision.meta ?? null, + typeSpecificData: (draft.headRevision.typeSpecificData ?? + null) as DraftFields['typeSpecificData'], + } + : null acceptedFingerprintRef.current = fingerprint if (!settings?.preserveLatest) { latestFingerprintRef.current = fingerprint @@ -682,6 +784,10 @@ function useDraftSession(options: { const openDraft = acceptedDraftRef.current const currentDraftId = openDraft?.id ?? draftId if (!currentDraftId || payload.branchId !== currentDraftId) return + if (crdtRef.current.active) { + void crdtRef.current.pull() + return + } if (payload.headRevisionId === openDraft?.headRevisionId) return void adoptLatestRemoteDraft(false) @@ -712,7 +818,14 @@ function useDraftSession(options: { ) const mutation = useMutation({ - mutationFn: (variables) => { + mutationFn: async (variables) => { + if (variables.draftId && crdtRef.current.active) { + await flushCrdt() + return createDraftCheckpoint(variables.draftId, 'save') + } + if (variables.draftId && crdtRef.current.locked) { + throw new Error(t('write.toast.draftSyncPending')) + } if (!variables.draftId) { return createDraft({ baseRevisionId: options.baseRevisionId, @@ -768,6 +881,7 @@ function useDraftSession(options: { }, onSuccess: async (draft, variables) => { const isFirstDraftSave = !variables.draftId + if (isFirstDraftSave) crdtRef.current.adoptEditorOnBind() acceptedDraftRef.current = draft setDraftId(draft.id) setConflict(null) @@ -814,6 +928,7 @@ function useDraftSession(options: { }, [buildSaveVariables, conflict, options.hasContent, resolvingConflict, t]) useEffect(() => { + if (crdt.active) return if (conflict || resolvingConflict || !dirtyRef.current) return if (!options.hasContent) return if (options.fingerprint === acceptedFingerprintRef.current) return @@ -821,6 +936,7 @@ function useDraftSession(options: { const timer = window.setTimeout(() => { if (autosaveTimerRef.current === timer) autosaveTimerRef.current = null if (!dirtyRef.current || mutationRef.current.isPending) return + if (crdtRef.current.locked) return mutationRef.current.mutate(buildSaveVariables()) }, 10_000) autosaveTimerRef.current = timer @@ -833,11 +949,28 @@ function useDraftSession(options: { acceptedFingerprint, buildSaveVariables, conflict, + crdt.active, options.fingerprint, options.hasContent, resolvingConflict, ]) + useEffect(() => { + if (!crdt.active || !ackedFieldsRef.current) return + if (!diffDraftFields(ackedFieldsRef.current, fieldsOf(options.data))) return + const timer = window.setTimeout(() => { + if (fieldsTimerRef.current === timer) fieldsTimerRef.current = null + void sendFields().catch((error) => + toast.error(getErrorMessage(error, t('write.toast.draftSaveFailed'))), + ) + }, 500) + fieldsTimerRef.current = timer + return () => { + window.clearTimeout(timer) + if (fieldsTimerRef.current === timer) fieldsTimerRef.current = null + } + }, [crdt.active, options.data, sendFields, t]) + useEffect(() => { const onKeyDown = (event: KeyboardEvent) => { if ((event.metaKey || event.ctrlKey) && event.key.toLowerCase() === 's') { @@ -881,10 +1014,29 @@ function useDraftSession(options: { accept, acceptedDraft: () => acceptedDraftRef.current, acceptedFingerprint: () => acceptedFingerprintRef.current, + flushCrdt, + checkpoint: async (kind: 'publish' | 'save') => { + await flushCrdt() + return createDraftCheckpoint(draftId, kind) + }, + reloadFields: async () => { + if (!draftId) return + const draft = await getDraftById(draftId) + acceptedDraftRef.current = draft + ackedFieldsRef.current = fieldsOf(draft.headRevision as DraftWriteData) + options.setState((previous) => ({ + ...fromRevision(options.kind, draft.headRevision, previous), + content: previous.content, + contentFormat: previous.contentFormat, + text: previous.text, + title: previous.title, + })) + }, clearAcceptedDraft: (id: string) => { if (acceptedDraftRef.current?.id === id) acceptedDraftRef.current = null }, conflict, + crdt, conflictDialogOpen, dismissRemoteUpdate, draftId, @@ -1435,6 +1587,9 @@ function WritePage(props: { kind: WriteKind }) { } appliedRouteDraftIdRef.current = draft.id + // Our own first save put this id in the URL; the editor is already ahead of the saved + // revision, and re-applying it would remount the editor and drop keystrokes typed since. + if (draftSession.draftId === draft.id) return const nextState = fromRevision(props.kind, draft.headRevision, state) const fingerprint = getDraftFingerprint( props.kind, @@ -1454,6 +1609,7 @@ function WritePage(props: { kind: WriteKind }) { } }, [ draftRefType, + draftSession.draftId, id, isEditing, props.kind, @@ -1587,17 +1743,19 @@ function WritePage(props: { kind: WriteKind }) { if (currentDraftId && !baseline) { throw new Error(t('write.toast.draftBaselineMissing')) } - const draft = currentDraftId - ? await updateDraft(currentDraftId, { - data, - expectedHeadRevisionId: baseline!.headRevisionId, - }) - : await createDraft({ - baseRevisionId: branchBaseRevisionId, - data, - refId: isEditing ? id : undefined, - refType: draftRefType, - }) + const draft = draftSession.crdt.active + ? await draftSession.checkpoint('publish') + : currentDraftId + ? await updateDraft(currentDraftId, { + data, + expectedHeadRevisionId: baseline!.headRevisionId, + }) + : await createDraft({ + baseRevisionId: branchBaseRevisionId, + data, + refId: isEditing ? id : undefined, + refType: draftRefType, + }) const task = await createPublishJob({ aiResources: props.kind === 'page' ? [] : aiResources, branchId: draft.id, @@ -1688,6 +1846,29 @@ function WritePage(props: { kind: WriteKind }) { } setPublishConfirmOpen(true) } + const crdtActive = draftSession.crdt.active + const setCrdtTitle = draftSession.crdt.setTitle + useEffect(() => { + if (crdtActive) setCrdtTitle(state.title) + }, [crdtActive, setCrdtTitle, state.title]) + + const workingCopy = activeDocumentBranches[0] + useEffect(() => { + if (!isEditing || routeDraftId || routeBaseRevisionId || !workingCopy) + return + if (workingCopy.headRevision.contentFormat !== 'lexical') return + const nextParams = new URLSearchParams(searchParams) + nextParams.set('draftId', workingCopy.id) + setSearchParams(nextParams, { replace: true }) + }, [ + isEditing, + routeBaseRevisionId, + routeDraftId, + searchParams, + setSearchParams, + workingCopy, + ]) + const updateField = ( key: TKey, value: WriteFormState[TKey], @@ -2025,21 +2206,26 @@ function WritePage(props: { kind: WriteKind }) { activeDocumentBranches.find( (branch) => branch.id === (draftId || routeDraftId), ) ?? routeDraftQuery.data - const versionBranchCount = isEditing - ? activeDocumentBranches.length - : activeNewDrafts.length + const versionBranchCount = crdtActive + ? 0 + : isEditing + ? activeDocumentBranches.length + : activeNewDrafts.length const editorBranchCount = isEditing ? activeDocumentBranches.length : 0 - const versionTreeTriggerTitle = - versionBranchCount > 0 + const versionTreeTriggerTitle = crdtActive + ? t('write.timeline.title') + : versionBranchCount > 0 ? t('write.versionTree.triggerWithCount', { count: versionBranchCount, }) : t('write.versionTree.title') - const publishIsDiverged = selectedBranch - ? selectedBranch.relationToPublished === 'descendant' || - selectedBranch.relationToPublished === 'diverged' - : baseRelationQuery.data?.relation === 'descendant' || - baseRelationQuery.data?.relation === 'diverged' + const publishIsDiverged = crdtActive + ? false + : selectedBranch + ? selectedBranch.relationToPublished === 'descendant' || + selectedBranch.relationToPublished === 'diverged' + : baseRelationQuery.data?.relation === 'descendant' || + baseRelationQuery.data?.relation === 'diverged' const draftKindText = getDraftKindLabel(props.kind) const isDirty = draftSession.hasUnsavedChanges() const metaStatus = computeMetaStatus({ @@ -2147,7 +2333,11 @@ function WritePage(props: { kind: WriteKind }) { type="button" variant={draftsPanelOpen ? 'active' : 'default'} > -