diff --git a/apps/admin/src/api/collab.ts b/apps/admin/src/api/collab.ts new file mode 100644 index 00000000000..c6c11fb95e6 --- /dev/null +++ b/apps/admin/src/api/collab.ts @@ -0,0 +1,119 @@ +import { deleteJson, getJson, patchJson, postJson, requestBytes } from './http' + +export interface CollabInvite { + createdAt: string + expiresAt: string + id: string + maxUses: number | null + uses: number +} + +export interface CreatedCollabInvite extends CollabInvite { + token: string +} + +export interface Collaborator { + avatar: string | null + contributedAt: string | null + createdAt: string + name: string + readerId: string + removedAt: string | null + showInByline: boolean +} + +export interface CollabPerson { + avatar: string | null + name: string +} + +export interface CollabWorkspace { + collaborators: CollabPerson[] + documentId: string + draftId: string + me: CollabPerson & { id: string } + refType: string +} + +export type AcceptedCollabInvite = + | { documentId: string; role: 'collaborator' } + | { + documentId: string + draftId: string + refId: string | null + refType: string + role: 'owner' + } + +const documentPath = (documentId: string) => `/collab/documents/${documentId}` + +export function listCollabInvites(documentId: string) { + return getJson(`${documentPath(documentId)}/invites`) +} + +export function createCollabInvite( + documentId: string, + data: { expiresInDays?: number; maxUses?: number | null } = {}, +) { + return postJson( + `${documentPath(documentId)}/invites`, + data, + ) +} + +export function revokeCollabInvite(documentId: string, inviteId: string) { + return deleteJson(`${documentPath(documentId)}/invites/${inviteId}`) +} + +export function listCollaborators(documentId: string) { + return getJson(`${documentPath(documentId)}/collaborators`) +} + +export function updateCollaborator( + documentId: string, + readerId: string, + data: { showInByline: boolean }, +) { + return patchJson( + `${documentPath(documentId)}/collaborators/${encodeURIComponent(readerId)}`, + data, + ) +} + +export function removeCollaborator(documentId: string, readerId: string) { + return deleteJson( + `${documentPath(documentId)}/collaborators/${encodeURIComponent(readerId)}`, + ) +} + +export function acceptCollabInvite(token: string) { + return postJson>( + `/collab/invites/${encodeURIComponent(token)}/accept`, + {}, + ) +} + +export function getCollabWorkspace(documentId: string) { + return getJson(documentPath(documentId)) +} + +const encodeBase64 = (bytes: Uint8Array) => + btoa(Array.from(bytes, (byte) => String.fromCharCode(byte)).join('')) + +export function getCollabCrdt(documentId: string, since?: Uint8Array) { + const query = since ? `?since=${encodeURIComponent(encodeBase64(since))}` : '' + return requestBytes(`${documentPath(documentId)}/crdt${query}`, { + method: 'GET', + }) +} + +export async function postCollabCrdt(documentId: string, bytes: Uint8Array) { + await requestBytes(`${documentPath(documentId)}/crdt`, { + body: bytes as Uint8Array, + headers: { 'content-type': 'application/octet-stream' }, + method: 'POST', + }) +} + +export const collabInviteUrlOf = (token: string) => + `${window.location.origin}${window.location.pathname}#/collab/join/${token}` diff --git a/apps/admin/src/api/drafts.ts b/apps/admin/src/api/drafts.ts index d2e07bca651..5b657c71cce 100644 --- a/apps/admin/src/api/drafts.ts +++ b/apps/admin/src/api/drafts.ts @@ -1,6 +1,9 @@ import type { Image, PaginateResult } from '~/models/base' import type { ContentRevision, + CrdtOpId, + DraftFields, + DraftHistory, DraftModel, DraftRefType, DraftShare, @@ -9,7 +12,14 @@ import type { VersionContext, } from '~/models/draft' -import { deleteJson, getJson, postJson, putJson } from './http' +import { + deleteJson, + getJson, + patchJson, + postJson, + putJson, + requestBytes, +} from './http' export type DraftSortOrder = 'asc' | 'desc' @@ -112,3 +122,54 @@ export function updateDraft(id: string, data: UpdateDraftData) { export function deleteDraft(id: string) { return deleteJson<{ success: boolean }>(`/drafts/${id}`) } + +const encodeBase64 = (bytes: Uint8Array) => + btoa(Array.from(bytes, (byte) => String.fromCharCode(byte)).join('')) + +export function getDraftCrdt(id: string, since?: Uint8Array) { + const query = since ? `?since=${encodeURIComponent(encodeBase64(since))}` : '' + return requestBytes(`/drafts/${id}/crdt${query}`, { method: 'GET' }) +} + +export async function postDraftCrdt(id: string, bytes: Uint8Array) { + await requestBytes(`/drafts/${id}/crdt`, { + body: bytes as Uint8Array, + headers: { 'content-type': 'application/octet-stream' }, + method: 'POST', + }) +} + +export function createDraftCheckpoint(id: string, kind: 'publish' | 'save') { + return postJson( + `/drafts/${id}/checkpoints`, + { kind }, + ) +} + +export function updateDraftFields( + id: string, + patch: Partial>>, +) { + return patchJson(`/drafts/${id}/fields`, patch) +} + +export function getDraftHistory(id: string) { + return getJson(`/drafts/${id}/history`) +} + +export function restoreDraftRevision(id: string, revisionId: string) { + return postJson(`/drafts/${id}/restore`, { + revisionId, + }) +} + +export function undoDraftChange( + id: string, + change: CrdtOpId, + before: CrdtOpId[], +) { + return postJson( + `/drafts/${id}/undo`, + { before, id: change }, + ) +} diff --git a/apps/admin/src/api/http.ts b/apps/admin/src/api/http.ts index ab189f88504..658972d0045 100644 --- a/apps/admin/src/api/http.ts +++ b/apps/admin/src/api/http.ts @@ -111,6 +111,38 @@ export async function requestJson( return responseData as TResponse } +export async function requestBytes( + path: string, + init: RequestInit = {}, +): Promise { + const response = await fetch(`${API_URL}${path}`, { + credentials: 'include', + ...init, + headers: buildAdminRequestHeaders(init.headers), + }) + + if (!response.ok) { + const responseData = normalizeResponseData( + camelcaseKeys(await readResponseData(response)), + ) + if (isUnauthorizedResponse(response, responseData)) { + handleUnauthorized() + } + const message = + responseData?.error?.message || + responseData?.message || + response.statusText + throw new ApiRequestError( + Array.isArray(message) ? message.join(', ') : message || 'Request failed', + responseData?.error?.code ?? responseData?.code, + response.status, + responseData?.error?.details, + ) + } + + return new Uint8Array(await response.arrayBuffer()) +} + export async function requestBlob( path: string, init: RequestInit = {}, @@ -184,7 +216,8 @@ function handleUnauthorized() { const isAuthRoute = hash.startsWith('/login') || hash.startsWith('/setup') || - hash.startsWith('/setup-api') + hash.startsWith('/setup-api') || + hash.startsWith('/collab') if (isAuthRoute) return diff --git a/apps/admin/src/features/collab/components/CoauthorEditor.tsx b/apps/admin/src/features/collab/components/CoauthorEditor.tsx new file mode 100644 index 00000000000..7c0f0672964 --- /dev/null +++ b/apps/admin/src/features/collab/components/CoauthorEditor.tsx @@ -0,0 +1,64 @@ +import type { LexicalEditor } from 'lexical' +import { lazy, Suspense, useEffect, useState } from 'react' + +import { API_URL } from '~/constants/env' +import { RemoteCarets } from '~/features/write/components/collab/RemoteCarets' +import type { CrdtDraftController } from '~/features/write/crdt/use-crdt-draft' +import { useI18n } from '~/i18n' + +const RichEditorBridge = lazy(() => + import('~/vendor/rich-editor').then((module) => ({ + default: module.RichEditorBridge, + })), +) + +const colorScheme = () => + document.documentElement.classList.contains('dark') ? 'dark' : 'light' + +const refuseSnapshot = async (): Promise => { + throw new Error('Co-authors cannot upload drawings') +} + +export function CoauthorEditor(props: { + crdt: CrdtDraftController + variant: 'article' | 'note' +}) { + const { t } = useI18n() + const [editor, setEditor] = useState(null) + const { bindEditor, locked } = props.crdt + useEffect(() => { + editor?.setEditable(!locked) + }, [editor, locked]) + + return ( + <> + + } + > + { + setEditor(next) + bindEditor(next) + }} + saveExcalidrawSnapshot={refuseSnapshot} + /> + + + + ) +} diff --git a/apps/admin/src/features/collab/components/CollabDocumentPage.tsx b/apps/admin/src/features/collab/components/CollabDocumentPage.tsx new file mode 100644 index 00000000000..9e43718fdfc --- /dev/null +++ b/apps/admin/src/features/collab/components/CollabDocumentPage.tsx @@ -0,0 +1,119 @@ +import { useQuery } from '@tanstack/react-query' +import { Loader2, UserX } from 'lucide-react' +import { useMemo, useState } from 'react' +import { useParams } from 'react-router' +import { toast } from 'sonner' + +import { getCollabCrdt, getCollabWorkspace, postCollabCrdt } from '~/api/collab' +import { ApiRequestError } from '~/api/http' +import { PeerAvatars } from '~/features/write/components/collab/PeerAvatars' +import { + type CrdtDraftRoom, + useCrdtDraft, +} from '~/features/write/crdt/use-crdt-draft' +import { useI18n } from '~/i18n' +import { subscribeWebSocket } from '~/socket/web-socket' +import { EmptyState } from '~/ui/patterns/EmptyState' + +import { CoauthorEditor } from './CoauthorEditor' + +export function CollabDocumentPage() { + const { t } = useI18n() + const { documentId = '' } = useParams() + const [title, setTitle] = useState('') + const [revoked, setRevoked] = useState(false) + const workspaceQuery = useQuery({ + enabled: Boolean(documentId), + queryFn: () => getCollabWorkspace(documentId), + queryKey: ['collab', 'workspace', documentId], + retry: false, + }) + const workspace = workspaceQuery.data + const room = useMemo( + () => + workspace && { + author: `human:${workspace.me.id}`, + http: { + pull: (since) => getCollabCrdt(documentId, since), + push: (bytes) => postCollabCrdt(documentId, bytes), + }, + subscribeSocket: subscribeWebSocket, + }, + [documentId, workspace], + ) + const crdt = useCrdtDraft({ + draftId: workspace?.draftId ?? '', + enabled: Boolean(room) && !revoked, + onRejected: async () => { + toast.error(t('collab.document.rejected')) + }, + onRemoteTitle: setTitle, + onRevoked: () => setRevoked(true), + room, + }) + + if (revoked || workspaceQuery.error) { + const forbidden = + revoked || + (workspaceQuery.error instanceof ApiRequestError && + workspaceQuery.error.status === 403) + return ( +
+ +
+ ) + } + + if (!workspace) { + return ( +
+
+ ) + } + + return ( +
+
+ + {t('collab.document.editingAs', { name: workspace.me.name })} + + {crdt.status === 'offline' ? ( + + {t('collab.document.offline')} + + ) : null} + +
+
+ { + setTitle(event.target.value) + crdt.setTitle(event.target.value) + }} + placeholder={t('write.editor.untitled')} + value={title} + /> + +
+
+ ) +} diff --git a/apps/admin/src/features/collab/components/CollabJoinPage.test.tsx b/apps/admin/src/features/collab/components/CollabJoinPage.test.tsx new file mode 100644 index 00000000000..919e05b11b2 --- /dev/null +++ b/apps/admin/src/features/collab/components/CollabJoinPage.test.tsx @@ -0,0 +1,134 @@ +import { QueryClient, QueryClientProvider } from '@tanstack/react-query' +import { act, createElement } from 'react' +import type { Root } from 'react-dom/client' +import { createRoot } from 'react-dom/client' +import { createMemoryRouter, RouterProvider } from 'react-router' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +import { acceptCollabInvite } from '~/api/collab' +import { ApiRequestError, getJson } from '~/api/http' +import { I18nProvider } from '~/i18n' +import { authClient } from '~/utils/authjs/auth' + +import { CollabJoinPage } from './CollabJoinPage' + +vi.mock('~/api/collab', () => ({ acceptCollabInvite: vi.fn() })) +vi.mock('~/api/http', async (load) => ({ + ...(await load()), + getJson: vi.fn(), +})) +vi.mock('~/utils/authjs/auth', () => ({ + authClient: { getSession: vi.fn(), signIn: { social: vi.fn() } }, +})) + +let container: HTMLDivElement +let root: Root +let router: ReturnType + +const flush = () => + act(async () => { + for (let index = 0; index < 4; index += 1) { + await new Promise((resolve) => setTimeout(resolve, 0)) + } + }) + +const render = async () => { + const client = new QueryClient({ + defaultOptions: { mutations: { retry: false }, queries: { retry: false } }, + }) + router = createMemoryRouter( + [ + { element: createElement(CollabJoinPage), path: '/collab/join/:token' }, + { element: createElement('p', null, 'document'), path: '/collab/:id' }, + { element: createElement('p', null, 'write'), path: '/posts/edit' }, + ], + { initialEntries: ['/collab/join/tok'] }, + ) + await act(async () => { + root.render( + createElement( + QueryClientProvider, + { client }, + createElement( + I18nProvider, + null, + createElement(RouterProvider, { router }), + ), + ), + ) + }) + await flush() +} + +describe('CollabJoinPage', () => { + beforeEach(() => { + container = document.createElement('div') + document.body.append(container) + root = createRoot(container) + }) + + afterEach(() => { + act(() => root.unmount()) + container.remove() + vi.clearAllMocks() + }) + + it('offers the configured sign-in providers to a signed-out reader', async () => { + vi.mocked(authClient.getSession).mockResolvedValue({ data: null } as never) + vi.mocked(getJson).mockResolvedValue(['github']) + await render() + + const button = container.querySelector('button')! + expect(button.textContent).toContain('GitHub') + await act(async () => button.click()) + + expect(authClient.signIn.social).toHaveBeenCalledWith( + expect.objectContaining({ provider: 'github' }), + ) + expect(acceptCollabInvite).not.toHaveBeenCalled() + }) + + it('accepts for a signed-in reader and opens the document', async () => { + vi.mocked(authClient.getSession).mockResolvedValue({ + data: { user: { id: 'r1' } }, + } as never) + vi.mocked(acceptCollabInvite).mockResolvedValue({ + documentId: 'doc-1', + role: 'collaborator', + }) + await render() + + expect(acceptCollabInvite).toHaveBeenCalledWith('tok') + expect(router.state.location.pathname).toBe('/collab/doc-1') + }) + + it('sends the owner to their own draft', async () => { + vi.mocked(authClient.getSession).mockResolvedValue({ + data: { user: { id: 'owner' } }, + } as never) + vi.mocked(acceptCollabInvite).mockResolvedValue({ + documentId: 'doc-1', + draftId: 'd1', + refId: 'p1', + refType: 'post', + role: 'owner', + }) + await render() + + expect(router.state.location.pathname).toBe('/posts/edit') + expect(router.state.location.search).toBe('?draftId=d1&id=p1') + }) + + it('explains a link that can no longer be used', async () => { + vi.mocked(authClient.getSession).mockResolvedValue({ + data: { user: { id: 'r1' } }, + } as never) + vi.mocked(acceptCollabInvite).mockRejectedValue( + new ApiRequestError('gone', 'COLLAB_INVITE_INVALID', 410), + ) + await render() + + expect(container.textContent).toMatch(/expired|过期/) + expect(router.state.location.pathname).toBe('/collab/join/tok') + }) +}) diff --git a/apps/admin/src/features/collab/components/CollabJoinPage.tsx b/apps/admin/src/features/collab/components/CollabJoinPage.tsx new file mode 100644 index 00000000000..cc0f459f72d --- /dev/null +++ b/apps/admin/src/features/collab/components/CollabJoinPage.tsx @@ -0,0 +1,135 @@ +import { useMutation, useQuery } from '@tanstack/react-query' +import { Loader2, Users } from 'lucide-react' +import { useEffect } from 'react' +import { useNavigate, useParams } from 'react-router' + +import { acceptCollabInvite } from '~/api/collab' +import { ApiRequestError, getJson } from '~/api/http' +import { getEditPathForDraft } from '~/features/drafts/utils/draft-edit-path' +import { useI18n } from '~/i18n' +import { + authBackdropClassName, + authBackdropStyle, + authCardClassName, + authIconTileClassName, +} from '~/lib/auth-backdrop' +import type { DraftRefType } from '~/models/draft' +import { Button } from '~/ui/primitives/button' +import { authClient } from '~/utils/authjs/auth' +import { cn } from '~/utils/cn' + +const PROVIDER_LABELS: Record = { + apple: 'Apple', + github: 'GitHub', + google: 'Google', +} + +export function CollabJoinPage() { + const { t } = useI18n() + const navigate = useNavigate() + const { token = '' } = useParams() + const session = useQuery({ + queryFn: async () => (await authClient.getSession()).data ?? null, + queryKey: ['collab', 'session'], + retry: false, + }) + const signedIn = Boolean(session.data?.user) + const providersQuery = useQuery({ + enabled: !session.isPending && !signedIn, + queryFn: () => getJson('/auth/providers'), + queryKey: ['collab', 'providers'], + }) + const accept = useMutation({ + mutationFn: () => acceptCollabInvite(token), + onSuccess: (result) => { + if (result.role === 'owner') { + navigate( + getEditPathForDraft({ + document: { + refId: result.refId, + refType: result.refType as DraftRefType, + }, + id: result.draftId, + }), + { replace: true }, + ) + return + } + navigate(`/collab/${result.documentId}`, { replace: true }) + }, + }) + + useEffect(() => { + if (signedIn && accept.isIdle) accept.mutate() + }, [accept, signedIn]) + + const signIn = (provider: string) => { + void authClient.signIn.social({ + callbackURL: window.location.href, + provider, + }) + } + + const failure = + accept.error instanceof ApiRequestError + ? accept.error.code === 'COLLAB_INVITE_INVALID' + ? t('collab.join.invalid') + : accept.error.message + : accept.error + ? t('collab.join.failed') + : null + + return ( +
+
+
+
+
+

+ {t('collab.join.title')} +

+

+ {failure ?? + (signedIn + ? t('collab.join.accepting') + : t('collab.join.signInHint'))} +

+
+ {session.isPending || accept.isPending ? ( +
+
+ ) +} diff --git a/apps/admin/src/features/drafts/components/DraftDetail.tsx b/apps/admin/src/features/drafts/components/DraftDetail.tsx index 479074dcb16..007f0eb8750 100644 --- a/apps/admin/src/features/drafts/components/DraftDetail.tsx +++ b/apps/admin/src/features/drafts/components/DraftDetail.tsx @@ -1,9 +1,10 @@ -import { useQuery } from '@tanstack/react-query' +import { useMutation, useQuery } from '@tanstack/react-query' import { ChevronLeft, GitCompare, Loader2, Pencil, Trash2 } from 'lucide-react' import { useEffect, useMemo, useState } from 'react' import { useNavigate } from 'react-router' +import { toast } from 'sonner' -import { getDraftRevisions } from '~/api/drafts' +import { getDraftRevisions, restoreDraftRevision } from '~/api/drafts' import { APP_SHELL_HEADER_HEIGHT_CLASS } from '~/constants/layout' import { useI18n } from '~/i18n' import type { DraftModel } from '~/models/draft' @@ -47,6 +48,13 @@ export function DraftDetail(props: { null, ) const versionsScopeId = `draft-versions-${props.draft.id}` + const restoresInPlace = props.draft.headRevision.contentFormat === 'lexical' + const restoreMutation = useMutation({ + mutationFn: (revisionId: string) => + restoreDraftRevision(props.draft.id, revisionId), + onError: () => toast.error(t('write.timeline.failed')), + onSuccess: (draft) => navigate(getEditPathForDraft(draft)), + }) useScopeArrowNav({ itemSelector: '[data-scope-item="row"]', @@ -164,10 +172,15 @@ export function DraftDetail(props: { item={item} key={item.id} onRestore={() => - navigate(getEditPathForRevision(props.draft, item.id)) + restoresInPlace + ? restoreMutation.mutate(item.id) + : navigate(getEditPathForRevision(props.draft, item.id)) } onSelect={() => setSelectedRevisionId(item.id)} - restorePending={false} + restorePending={ + restoreMutation.isPending && + restoreMutation.variables === item.id + } selected={selectedRevisionId === item.id} /> ))} diff --git a/apps/admin/src/features/write/components/TimelinePanel.test.tsx b/apps/admin/src/features/write/components/TimelinePanel.test.tsx new file mode 100644 index 00000000000..247685fdb8d --- /dev/null +++ b/apps/admin/src/features/write/components/TimelinePanel.test.tsx @@ -0,0 +1,196 @@ +import { QueryClient, QueryClientProvider } from '@tanstack/react-query' +import { act, createElement } from 'react' +import type { Root } from 'react-dom/client' +import { createRoot } from 'react-dom/client' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +import { listCollaborators } from '~/api/collab' +import { + getDraftHistory, + getDraftShare, + restoreDraftRevision, + undoDraftChange, +} from '~/api/drafts' +import { I18nProvider } from '~/i18n' +import type { DraftHistory } from '~/models/draft' + +import { TimelinePanel } from './TimelinePanel' + +vi.mock('~/api/drafts', () => ({ + deleteDraftShare: vi.fn(), + getDraftHistory: vi.fn(), + getDraftShare: vi.fn(), + restoreDraftRevision: vi.fn(), + setDraftShare: vi.fn(), + undoDraftChange: vi.fn(), +})) +vi.mock('~/api/collab', () => ({ + collabInviteUrlOf: (token: string) => token, + createCollabInvite: vi.fn(), + listCollabInvites: vi.fn(async () => []), + listCollaborators: vi.fn(async () => []), + removeCollaborator: vi.fn(), + revokeCollabInvite: vi.fn(), + updateCollaborator: vi.fn(), +})) +vi.mock('~/ui/feedback/confirm', () => ({ + confirmDialog: vi.fn(async () => true), +})) + +const history: DraftHistory = { + changes: [ + { + deps: [{ counter: 3, peer: '1' }], + id: { counter: 4, peer: '1' }, + length: 1, + message: 'human:owner', + timestamp: 1_792_900_000, + }, + ], + checkpoints: [ + { + createdAt: '2026-10-25T00:00:00.000Z', + frontiers: [], + id: 'rev-1', + kind: 'save', + parentRevisionId: null, + title: 'Saved', + }, + ], +} as unknown as DraftHistory + +let container: HTMLDivElement +let root: Root +const calls: string[] = [] +const onBeforeChange = vi.fn(async () => { + calls.push('flush') +}) +const onChanged = vi.fn(async (change: { restored: boolean }) => { + calls.push(change.restored ? 'reload-fields' : 'pull') +}) + +const render = async (documentId: string | null = null) => { + const client = new QueryClient({ + defaultOptions: { queries: { retry: false } }, + }) + await act(async () => { + root.render( + createElement( + QueryClientProvider, + { client }, + createElement( + I18nProvider, + null, + createElement(TimelinePanel, { + documentId, + draftId: 'd1', + onBeforeChange, + onChanged, + onClose: () => undefined, + publishedRevisionId: null, + }), + ), + ), + ) + }) + await act(async () => { + await Promise.resolve() + }) +} + +const click = async (label: string) => { + const button = await vi.waitFor(() => { + const found = container.querySelector( + `button[aria-label="${label}"]`, + ) + expect(found).not.toBeNull() + return found! + }) + await act(async () => { + button.click() + }) + await vi.waitFor(() => expect(onChanged).toHaveBeenCalled()) +} + +beforeEach(() => { + calls.length = 0 + onChanged.mockClear() + container = document.createElement('div') + document.body.append(container) + root = createRoot(container) + vi.mocked(getDraftHistory).mockResolvedValue(history) + vi.mocked(getDraftShare).mockResolvedValue(null as never) + vi.mocked(restoreDraftRevision).mockImplementation(async () => { + calls.push('restore') + return {} as never + }) + vi.mocked(undoDraftChange).mockImplementation(async () => { + calls.push('undo') + return {} as never + }) +}) + +afterEach(() => { + act(() => root.unmount()) + document.body.innerHTML = '' +}) + +describe('TimelinePanel', () => { + it('sends unsent edits before restoring and reloads fields after', async () => { + await render() + + await click('恢复到此版本') + + expect(calls).toEqual(['flush', 'restore', 'reload-fields']) + }) + + it('sends unsent edits before undoing a step', async () => { + await render() + + await click('撤销此步') + + expect(calls).toEqual(['flush', 'undo', 'pull']) + }) + + it('names co-author steps, including people since removed', async () => { + vi.mocked(getDraftHistory).mockResolvedValue({ + changes: [ + { + deps: [], + id: { counter: 9, peer: '3' }, + length: 1, + message: 'human:r2', + timestamp: 1_792_900_200, + }, + { + deps: [], + id: { counter: 5, peer: '2' }, + length: 1, + message: 'human:r1', + timestamp: 1_792_900_100, + }, + ], + checkpoints: [], + } as unknown as DraftHistory) + vi.mocked(listCollaborators).mockResolvedValue([ + { + avatar: null, + contributedAt: '2026-10-25T00:00:00.000Z', + createdAt: '2026-10-25T00:00:00.000Z', + name: 'Bob', + readerId: 'r1', + removedAt: '2026-10-25T01:00:00.000Z', + showInByline: true, + }, + ]) + + await render('doc-1') + + await vi.waitFor(() => + expect(container.textContent).toContain('Bob 的编辑'), + ) + expect(container.textContent).toContain('共同作者 的编辑') + expect(container.textContent).not.toContain('r2 的编辑') + expect(container.querySelector('[data-collaborator]')).toBeNull() + }) +}) diff --git a/apps/admin/src/features/write/components/TimelinePanel.tsx b/apps/admin/src/features/write/components/TimelinePanel.tsx new file mode 100644 index 00000000000..a751ec10d5f --- /dev/null +++ b/apps/admin/src/features/write/components/TimelinePanel.tsx @@ -0,0 +1,315 @@ +import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' +import { History, Loader2, RotateCcw, Undo2 } from 'lucide-react' +import { useEffect, useMemo } from 'react' +import { toast } from 'sonner' + +import { listCollaborators } from '~/api/collab' +import type { SetDraftShareData } from '~/api/drafts' +import { + deleteDraftShare, + getDraftHistory, + getDraftShare, + restoreDraftRevision, + setDraftShare, + undoDraftChange, +} from '~/api/drafts' +import { useI18n } from '~/i18n' +import type { TranslationKey } from '~/i18n/types' +import type { CheckpointKind, DraftShare } from '~/models/draft' +import { adminQueryKeys } from '~/query/keys' +import { subscribeDraftUpdate } from '~/socket/draft-update-signal' +import { confirmDialog } from '~/ui/feedback/confirm' +import { AsidePanel } from '~/ui/layout/content-layout' +import { EmptyState } from '~/ui/patterns/EmptyState' +import { Button } from '~/ui/primitives/button' +import { Scroll } from '~/ui/primitives/scroll' +import { cn } from '~/utils/cn' + +import { CollaboratorsSection } from './collab/CollaboratorsSection' +import { + buildTimeline, + type StepKind, + type TimelineCheckpoint, + type TimelineRow, + type TimelineStep, +} from './timeline/build-timeline' +import { ShareBar } from './version-tree/ShareBar' + +const ROW_LIMIT = 200 + +const checkpointLabels: Record = { + import: 'write.timeline.checkpoint.import', + legacy: 'write.timeline.checkpoint.legacy', + 'legacy-save': 'write.timeline.checkpoint.legacySave', + publish: 'write.timeline.checkpoint.publish', + restore: 'write.timeline.checkpoint.restore', + save: 'write.timeline.checkpoint.save', + undo: 'write.timeline.checkpoint.undo', +} + +const stepLabels: Record = { + agent: 'write.timeline.step.agent', + human: 'write.timeline.step.human', + other: 'write.timeline.step.other', + restore: 'write.timeline.step.restore', + undo: 'write.timeline.step.undo', +} + +export interface TimelinePanelProps { + documentId: string | null + draftId: string + onBeforeChange: () => Promise + onChanged: (change: { restored: boolean }) => Promise | void + onClose: () => void + publishedRevisionId: string | null +} + +export function TimelinePanel(props: TimelinePanelProps) { + const { format, t } = useI18n() + const queryClient = useQueryClient() + const { draftId, documentId } = props + const historyKey = adminQueryKeys.drafts.timeline(draftId) + const historyQuery = useQuery({ + enabled: Boolean(draftId), + queryFn: () => getDraftHistory(draftId), + queryKey: historyKey, + }) + + useEffect( + () => + subscribeDraftUpdate((payload) => { + if (payload.branchId !== draftId) return + void queryClient.invalidateQueries({ + queryKey: adminQueryKeys.drafts.timeline(draftId), + }) + }), + [draftId, queryClient], + ) + + const collaboratorsQuery = useQuery({ + enabled: Boolean(documentId), + queryFn: () => listCollaborators(documentId!), + queryKey: adminQueryKeys.collab.collaborators(documentId ?? ''), + }) + const authorNames = useMemo( + () => + new Map( + (collaboratorsQuery.data ?? []).map((person) => [ + person.readerId, + person.name, + ]), + ), + [collaboratorsQuery.data], + ) + + const rows = useMemo( + () => + historyQuery.data + ? buildTimeline(historyQuery.data, props.publishedRevisionId).slice( + 0, + ROW_LIMIT, + ) + : [], + [historyQuery.data, props.publishedRevisionId], + ) + + const afterChange = async (restored: boolean) => { + await props.onChanged({ restored }) + await queryClient.invalidateQueries({ queryKey: historyKey }) + } + const restoreMutation = useMutation({ + mutationFn: async (revisionId: string) => { + await props.onBeforeChange() + return restoreDraftRevision(draftId, revisionId) + }, + onError: () => toast.error(t('write.timeline.failed')), + onSuccess: async () => { + await afterChange(true) + toast.success(t('write.timeline.restored')) + }, + }) + const undoMutation = useMutation({ + mutationFn: async (step: TimelineStep) => { + await props.onBeforeChange() + return undoDraftChange(draftId, step.change, step.before) + }, + onError: () => toast.error(t('write.timeline.failed')), + onSuccess: async () => { + await afterChange(false) + toast.success(t('write.timeline.undone')) + }, + }) + const busy = restoreMutation.isPending || undoMutation.isPending + + const restore = async (row: TimelineCheckpoint) => { + const confirmed = await confirmDialog({ + confirmText: t('write.timeline.restore'), + description: t('write.timeline.restoreDescription'), + title: t('write.timeline.restoreTitle'), + }) + if (confirmed) restoreMutation.mutate(row.id) + } + + const shareKey = adminQueryKeys.drafts.share(documentId ?? '') + const shareQuery = useQuery({ + enabled: Boolean(documentId), + queryFn: () => getDraftShare(documentId!), + queryKey: shareKey, + }) + const writeShare = (share: DraftShare | null) => { + queryClient.setQueryData(shareKey, share) + } + const setShareMutation = useMutation({ + mutationFn: (data: SetDraftShareData) => setDraftShare(documentId!, data), + onSuccess: writeShare, + }) + const revokeShareMutation = useMutation({ + mutationFn: () => deleteDraftShare(documentId!), + onSuccess: () => { + writeShare(null) + toast.success(t('write.share.revoked')) + }, + }) + const followDraft = () => setShareMutation.mutate({ draftId, mode: 'follow' }) + + return ( + + {documentId ? : null} + {documentId ? ( + revokeShareMutation.mutate()} + pending={setShareMutation.isPending || revokeShareMutation.isPending} + share={shareQuery.data ?? null} + /> + ) : null} + {historyQuery.isLoading ? ( +
+
+ ) : rows.length === 0 ? ( +
+ +
+ ) : ( + +
    + {rows.map((row) => ( + void restore(row as TimelineCheckpoint)} + onUndo={() => undoMutation.mutate(row as TimelineStep)} + row={row} + time={format.relativeTime(row.time)} + /> + ))} +
+ + )} + + ) +} + +const rowKey = (row: TimelineRow) => + row.type === 'checkpoint' + ? `checkpoint:${row.id}` + : `step:${row.change.peer}:${row.change.counter}` + +function TimelineItem(props: { + authorName: string | null + busy: boolean + onRestore: () => void + onUndo: () => void + row: TimelineRow + time: string +}) { + const { t } = useI18n() + const { row } = props + + if (row.type === 'checkpoint') { + return ( +
  • + +
    +
    + + {t(checkpointLabels[row.kind ?? 'legacy'])} + + {row.published ? ( + + {t('write.timeline.published')} + + ) : null} +
    +
    + {row.title || t('write.editor.untitled')} · {props.time} +
    +
    + +
    • + ) + } + + const summary = + row.kind === 'agent' ? row.message.replace(/^agent:?\s*/, '') : '' + return ( +
  • +
    +
    + {props.authorName + ? t('write.timeline.step.coauthor', { + count: row.count, + name: props.authorName, + }) + : t(stepLabels[row.kind], { count: row.count, summary })} +
    +
    {props.time}
    +
    + {row.undoable ? ( + + ) : null} +
    • + ) +} diff --git a/apps/admin/src/features/write/components/WriteRouteViewsContent.tsx b/apps/admin/src/features/write/components/WriteRouteViewsContent.tsx index 583e594e8fd..bf31fc4fe5e 100644 --- a/apps/admin/src/features/write/components/WriteRouteViewsContent.tsx +++ b/apps/admin/src/features/write/components/WriteRouteViewsContent.tsx @@ -2,7 +2,7 @@ import { projectAgentDiffNodesToFactualState } from '@haklex/rich-ext-ai-agent/s import { analyzeMxMarkdown, mxLexicalToMarkdown } from '@mx-space/editor' import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' import { load } from 'js-yaml' -import type { SerializedEditorState } from 'lexical' +import type { LexicalEditor, SerializedEditorState } from 'lexical' import type { LucideIcon } from 'lucide-react' import { ArrowLeftRight, @@ -65,17 +65,19 @@ import type { DraftWriteData } from '~/api/drafts' import { compareRevisions, createDraft, + createDraftCheckpoint, deleteDraft, getDraftById, getDraftContext, getNewDrafts, getRevision, updateDraft, + updateDraftFields, } from '~/api/drafts' import { uploadFile, uploadFileWithProgress } from '~/api/files' import { ApiRequestError } from '~/api/http' import { getNoteById } from '~/api/notes' -import { getOption } from '~/api/options' +import { getOption, getOwner } from '~/api/options' import { getPageById } from '~/api/pages' import { getPostById, getPosts } from '~/api/posts' import type { PublishAiResourceRequest, PublishTask } from '~/api/publish-jobs' @@ -130,8 +132,17 @@ import { import { PublishConfirmationDialog } from '~/features/write/components/PublishConfirmationDialog' import { openPublishProcessDock } from '~/features/write/components/PublishProcessDock' import { SkillPicker } from '~/features/write/components/SkillPicker' +import { TimelinePanel } from '~/features/write/components/TimelinePanel' import { TtsGenerationEntry } from '~/features/write/components/tts/TtsGenerationEntry' import { VersionTreePanel } from '~/features/write/components/VersionTreePanel' +import type { PresenceUser } from '~/features/write/crdt/collab-presence' +import { diffDraftFields } from '~/features/write/crdt/draft-fields' +import { createReadyGate } from '~/features/write/crdt/ready-gate' +import { resyncRejectedDraft } from '~/features/write/crdt/resync-rejected' +import { + type CrdtDraftController, + useCrdtDraft, +} from '~/features/write/crdt/use-crdt-draft' import { MetaPresetSection } from '~/features/write/meta-presets' import type { DraftMergeConflict } from '~/features/write/utils/merge-draft-conflict' import { mergeDraftConflict } from '~/features/write/utils/merge-draft-conflict' @@ -144,6 +155,7 @@ import type { TranslationKey } from '~/i18n/types' import type { Amap, AMapSearch } from '~/models/amap' import type { Image as ImageModel } from '~/models/base' import type { + DraftFields, DraftModel, RevisionSnapshot, VersionTreeNode, @@ -189,6 +201,9 @@ import { buildMetaTools, } from '~/vendor/rich-editor/utils/meta-tools' +import { PeerAvatars } from './collab/PeerAvatars' +import { RemoteCarets } from './collab/RemoteCarets' + type WriteKind = 'note' | 'page' | 'post' type ContentFormat = 'lexical' | 'markdown' type NoteCoordinates = { latitude: number; longitude: number } @@ -445,6 +460,15 @@ function WritePageRoute(props: { kind: WriteKind }) { return } +const FIELDS_RETRY_MS = 2000 +const CRDT_READY_TIMEOUT_MS = 10_000 + +const fieldsOf = (data: DraftWriteData): DraftFields => ({ + meta: data.meta ?? null, + typeSpecificData: (data.typeSpecificData ?? + null) as DraftFields['typeSpecificData'], +}) + function useDraftSession(options: { baseRevisionId: string | null data: DraftWriteData @@ -473,19 +497,109 @@ function useDraftSession(options: { const [conflict, setConflict] = useState(null) const [conflictDialogOpen, setConflictDialogOpen] = useState(false) const [resolvingConflict, setResolvingConflict] = useState(false) + const ackedFieldsRef = useRef(null) + const fieldsTimerRef = useRef(null) + const fieldsInFlightRef = useRef | null>(null) + const ownerQuery = useQuery({ + queryFn: getOwner, + queryKey: ['shell', 'owner'], + retry: false, + }) + const owner = ownerQuery.data + const presenceUser = useMemo( + () => ({ + avatar: owner?.avatar || owner?.image || null, + id: owner?.id ?? 'owner', + name: owner?.name || owner?.username || '', + }), + [owner], + ) + const crdt = useCrdtDraft({ + draftId, + enabled: options.state.contentFormat === 'lexical', + onRejected: async () => { + if (!draftId) return + try { + const draft = await resyncRejectedDraft({ + checkpoint: () => createDraftCheckpoint(draftId, 'save'), + data: latestDataRef.current, + update: (data, expectedHeadRevisionId) => + updateDraft(draftId, { data, expectedHeadRevisionId }), + }) + acceptedDraftRef.current = draft + toast.info(t('write.toast.draftResynced')) + } catch (error) { + toast.error(getErrorMessage(error, t('write.toast.draftSaveFailed'))) + } + }, + onRemoteTitle: (title) => + options.setState((previous) => + previous.title === title ? previous : { ...previous, title }, + ), + }) + const crdtRef = useRef(crdt) + crdtRef.current = crdt + + const sendFields = useCallback(async () => { + if (fieldsTimerRef.current !== null) { + window.clearTimeout(fieldsTimerRef.current) + fieldsTimerRef.current = null + } + await fieldsInFlightRef.current + const acked = ackedFieldsRef.current + if (!crdtRef.current.active || !draftId || !acked) return + const next = fieldsOf(latestDataRef.current) + const patch = diffDraftFields(acked, next) + if (!patch) return + const request = updateDraftFields(draftId, patch).then( + () => { + ackedFieldsRef.current = next + }, + (error: unknown) => { + if (fieldsTimerRef.current === null) { + fieldsTimerRef.current = window.setTimeout(() => { + fieldsTimerRef.current = null + void sendFieldsRef.current().catch(() => undefined) + }, FIELDS_RETRY_MS) + } + throw error + }, + ) + fieldsInFlightRef.current = request.finally(() => { + fieldsInFlightRef.current = null + }) + await request + }, [draftId]) + const sendFieldsRef = useRef(sendFields) + sendFieldsRef.current = sendFields + + const flushCrdt = useCallback(async () => { + await crdtRef.current.flush() + await sendFields() + }, [sendFields]) useEffect(() => { latestDataRef.current = options.data latestFingerprintRef.current = options.fingerprint }, [options.data, options.fingerprint]) - const hasUnsavedChanges = useCallback( - () => + const hasUnsavedChanges = useCallback(() => { + if (crdtRef.current.active) { + const acked = ackedFieldsRef.current + return ( + crdtRef.current.pending() || + fieldsTimerRef.current !== null || + fieldsInFlightRef.current !== null || + (acked !== null && + diffDraftFields(acked, fieldsOf(latestDataRef.current)) !== null) + ) + } + return ( dirtyRef.current && options.hasContent && - latestFingerprintRef.current !== acceptedFingerprintRef.current, - [options.hasContent], - ) + latestFingerprintRef.current !== acceptedFingerprintRef.current + ) + }, [options.hasContent]) const accept = useCallback( ( @@ -498,6 +612,13 @@ function useDraftSession(options: { }, ) => { acceptedDraftRef.current = draft + ackedFieldsRef.current = draft + ? { + meta: draft.headRevision.meta ?? null, + typeSpecificData: (draft.headRevision.typeSpecificData ?? + null) as DraftFields['typeSpecificData'], + } + : null acceptedFingerprintRef.current = fingerprint if (!settings?.preserveLatest) { latestFingerprintRef.current = fingerprint @@ -682,6 +803,10 @@ function useDraftSession(options: { const openDraft = acceptedDraftRef.current const currentDraftId = openDraft?.id ?? draftId if (!currentDraftId || payload.branchId !== currentDraftId) return + if (crdtRef.current.active) { + if (!crdtRef.current.live()) void crdtRef.current.pull() + return + } if (payload.headRevisionId === openDraft?.headRevisionId) return void adoptLatestRemoteDraft(false) @@ -712,7 +837,14 @@ function useDraftSession(options: { ) const mutation = useMutation({ - mutationFn: (variables) => { + mutationFn: async (variables) => { + if (variables.draftId && crdtRef.current.active) { + await flushCrdt() + return createDraftCheckpoint(variables.draftId, 'save') + } + if (variables.draftId && crdtRef.current.locked) { + throw new Error(t('write.toast.draftSyncPending')) + } if (!variables.draftId) { return createDraft({ baseRevisionId: options.baseRevisionId, @@ -768,6 +900,7 @@ function useDraftSession(options: { }, onSuccess: async (draft, variables) => { const isFirstDraftSave = !variables.draftId + if (isFirstDraftSave) crdtRef.current.adoptEditorOnBind() acceptedDraftRef.current = draft setDraftId(draft.id) setConflict(null) @@ -795,6 +928,43 @@ function useDraftSession(options: { const mutationRef = useRef(mutation) mutationRef.current = mutation + const [crdtGate] = useState(() => + createReadyGate(() => crdtRef.current.active), + ) + useEffect(() => { + if (crdt.active) crdtGate.release() + }, [crdt.active, crdtGate]) + + const ensureCrdt = useCallback( + () => + crdtGate.once(async () => { + if (crdtRef.current.active) return true + if (options.state.contentFormat !== 'lexical') return false + if (!draftId && !mutationRef.current.isPending) { + if (!options.hasContent) { + toast.error(t('write.toast.contentEmptyForDraft')) + return false + } + try { + await mutationRef.current.mutateAsync(buildSaveVariables(true)) + } catch { + return false + } + } + const ready = await crdtGate.wait(CRDT_READY_TIMEOUT_MS) + if (!ready) toast.error(t('write.toast.draftSyncPending')) + return ready + }), + [ + buildSaveVariables, + crdtGate, + draftId, + options.hasContent, + options.state.contentFormat, + t, + ], + ) + const saveNow = useCallback(() => { if (conflict || resolvingConflict) { toast.error(t('write.toast.draftConflictNeedsResolution')) @@ -821,6 +991,7 @@ function useDraftSession(options: { const timer = window.setTimeout(() => { if (autosaveTimerRef.current === timer) autosaveTimerRef.current = null if (!dirtyRef.current || mutationRef.current.isPending) return + if (crdtRef.current.locked) return mutationRef.current.mutate(buildSaveVariables()) }, 10_000) autosaveTimerRef.current = timer @@ -838,6 +1009,22 @@ function useDraftSession(options: { resolvingConflict, ]) + useEffect(() => { + if (!crdt.active || !ackedFieldsRef.current) return + if (!diffDraftFields(ackedFieldsRef.current, fieldsOf(options.data))) return + const timer = window.setTimeout(() => { + if (fieldsTimerRef.current === timer) fieldsTimerRef.current = null + void sendFields().catch((error) => + toast.error(getErrorMessage(error, t('write.toast.draftSaveFailed'))), + ) + }, 500) + fieldsTimerRef.current = timer + return () => { + window.clearTimeout(timer) + if (fieldsTimerRef.current === timer) fieldsTimerRef.current = null + } + }, [crdt.active, options.data, sendFields, t]) + useEffect(() => { const onKeyDown = (event: KeyboardEvent) => { if ((event.metaKey || event.ctrlKey) && event.key.toLowerCase() === 's') { @@ -881,10 +1068,31 @@ function useDraftSession(options: { accept, acceptedDraft: () => acceptedDraftRef.current, acceptedFingerprint: () => acceptedFingerprintRef.current, + flushCrdt, + ensureCrdt, + presenceUser, + checkpoint: async (kind: 'publish' | 'save') => { + await flushCrdt() + return createDraftCheckpoint(draftId, kind) + }, + reloadFields: async () => { + if (!draftId) return + const draft = await getDraftById(draftId) + acceptedDraftRef.current = draft + ackedFieldsRef.current = fieldsOf(draft.headRevision as DraftWriteData) + options.setState((previous) => ({ + ...fromRevision(options.kind, draft.headRevision, previous), + content: previous.content, + contentFormat: previous.contentFormat, + text: previous.text, + title: previous.title, + })) + }, clearAcceptedDraft: (id: string) => { if (acceptedDraftRef.current?.id === id) acceptedDraftRef.current = null }, conflict, + crdt, conflictDialogOpen, dismissRemoteUpdate, draftId, @@ -1435,6 +1643,9 @@ function WritePage(props: { kind: WriteKind }) { } appliedRouteDraftIdRef.current = draft.id + // Our own first save put this id in the URL; the editor is already ahead of the saved + // revision, and re-applying it would remount the editor and drop keystrokes typed since. + if (draftSession.draftId === draft.id) return const nextState = fromRevision(props.kind, draft.headRevision, state) const fingerprint = getDraftFingerprint( props.kind, @@ -1454,6 +1665,7 @@ function WritePage(props: { kind: WriteKind }) { } }, [ draftRefType, + draftSession.draftId, id, isEditing, props.kind, @@ -1587,17 +1799,19 @@ function WritePage(props: { kind: WriteKind }) { if (currentDraftId && !baseline) { throw new Error(t('write.toast.draftBaselineMissing')) } - const draft = currentDraftId - ? await updateDraft(currentDraftId, { - data, - expectedHeadRevisionId: baseline!.headRevisionId, - }) - : await createDraft({ - baseRevisionId: branchBaseRevisionId, - data, - refId: isEditing ? id : undefined, - refType: draftRefType, - }) + const draft = draftSession.crdt.active + ? await draftSession.checkpoint('publish') + : currentDraftId + ? await updateDraft(currentDraftId, { + data, + expectedHeadRevisionId: baseline!.headRevisionId, + }) + : await createDraft({ + baseRevisionId: branchBaseRevisionId, + data, + refId: isEditing ? id : undefined, + refType: draftRefType, + }) const task = await createPublishJob({ aiResources: props.kind === 'page' ? [] : aiResources, branchId: draft.id, @@ -1688,6 +1902,35 @@ function WritePage(props: { kind: WriteKind }) { } setPublishConfirmOpen(true) } + const crdtActive = draftSession.crdt.active + const peerAvatars = crdtActive ? ( + + ) : null + const setCrdtTitle = draftSession.crdt.setTitle + useEffect(() => { + if (crdtActive) setCrdtTitle(state.title) + }, [crdtActive, setCrdtTitle, state.title]) + + const workingCopy = activeDocumentBranches[0] + useEffect(() => { + if (!isEditing || routeDraftId || routeBaseRevisionId || !workingCopy) + return + if (workingCopy.headRevision.contentFormat !== 'lexical') return + const nextParams = new URLSearchParams(searchParams) + nextParams.set('draftId', workingCopy.id) + setSearchParams(nextParams, { replace: true }) + }, [ + isEditing, + routeBaseRevisionId, + routeDraftId, + searchParams, + setSearchParams, + workingCopy, + ]) + const updateField = ( key: TKey, value: WriteFormState[TKey], @@ -2025,21 +2268,26 @@ function WritePage(props: { kind: WriteKind }) { activeDocumentBranches.find( (branch) => branch.id === (draftId || routeDraftId), ) ?? routeDraftQuery.data - const versionBranchCount = isEditing - ? activeDocumentBranches.length - : activeNewDrafts.length + const versionBranchCount = crdtActive + ? 0 + : isEditing + ? activeDocumentBranches.length + : activeNewDrafts.length const editorBranchCount = isEditing ? activeDocumentBranches.length : 0 - const versionTreeTriggerTitle = - versionBranchCount > 0 + const versionTreeTriggerTitle = crdtActive + ? t('write.timeline.title') + : versionBranchCount > 0 ? t('write.versionTree.triggerWithCount', { count: versionBranchCount, }) : t('write.versionTree.title') - const publishIsDiverged = selectedBranch - ? selectedBranch.relationToPublished === 'descendant' || - selectedBranch.relationToPublished === 'diverged' - : baseRelationQuery.data?.relation === 'descendant' || - baseRelationQuery.data?.relation === 'diverged' + const publishIsDiverged = crdtActive + ? false + : selectedBranch + ? selectedBranch.relationToPublished === 'descendant' || + selectedBranch.relationToPublished === 'diverged' + : baseRelationQuery.data?.relation === 'descendant' || + baseRelationQuery.data?.relation === 'diverged' const draftKindText = getDraftKindLabel(props.kind) const isDirty = draftSession.hasUnsavedChanges() const metaStatus = computeMetaStatus({ @@ -2123,6 +2371,7 @@ function WritePage(props: { kind: WriteKind }) { {props.kind === 'page' ? (
    + {peerAvatars} {state.contentFormat === 'markdown' ? ( setPageParseDialogOpen(true)} @@ -2147,7 +2396,11 @@ function WritePage(props: { kind: WriteKind }) { type="button" variant={draftsPanelOpen ? 'active' : 'default'} > - toggleAsidePanel('meta')} @@ -2182,6 +2435,7 @@ function WritePage(props: { kind: WriteKind }) {
    ) : (
    + {peerAvatars} toggleAsidePanel('drafts')} @@ -2189,7 +2443,11 @@ function WritePage(props: { kind: WriteKind }) { type="button" variant={draftsPanelOpen ? 'active' : 'default'} > - updateField('content', content) } + crdt={draftSession.crdt} + ensureDocument={draftSession.ensureCrdt} onMetaFieldsUpdate={applyAgentMetaUpdates} onPinSelection={() => setAsidePanel('agent')} onTextChange={(text) => updateField('text', text)} @@ -2384,37 +2644,53 @@ function WritePage(props: { kind: WriteKind }) { )} - draft.id === (draftId || routeDraftId), - )?.documentId ?? - null - } - drafts={isEditing ? activeDocumentBranches : activeNewDrafts} - nodes={ - isEditing - ? (versionContextQuery.data?.versionTree ?? []) - : standaloneDraftTree - } - onClose={closeDraftsPanel} - onCompare={setReviewingDraft} - onDelete={confirmAndDeleteDraft} - onContinue={(draft) => void continueDraft(draft)} - onHistory={(draft) => navigate(`/drafts/${draft.id}`)} - onPublish={(draft) => void publishDraftFromTree(draft)} - onViewOnline={() => void viewCurrentArticle()} - /> + {crdtActive ? ( + { + await draftSession.crdt.pull() + if (restored) await draftSession.reloadFields() + }} + onClose={closeDraftsPanel} + publishedRevisionId={ + versionContextQuery.data?.document.publishedRevisionId ?? null + } + /> + ) : ( + draft.id === (draftId || routeDraftId), + )?.documentId ?? + null + } + drafts={isEditing ? activeDocumentBranches : activeNewDrafts} + nodes={ + isEditing + ? (versionContextQuery.data?.versionTree ?? []) + : standaloneDraftTree + } + onClose={closeDraftsPanel} + onCompare={setReviewingDraft} + onDelete={confirmAndDeleteDraft} + onContinue={(draft) => void continueDraft(draft)} + onHistory={(draft) => navigate(`/drafts/${draft.id}`)} + onPublish={(draft) => void publishDraftFromTree(draft)} + onViewOnline={() => void viewCurrentArticle()} + /> + )} {props.kind === 'page' ? ( @@ -2426,7 +2702,7 @@ function WritePage(props: { kind: WriteKind }) { /> ) : ( Promise kind: WriteKind onContentChange: (content: string) => void onTextChange: (text: string) => void @@ -5154,13 +5432,24 @@ function RichWriteSurface(props: { ) const agent = useWriteAgent({ agentVisible: Boolean(props.agentVisible), + document: props.crdt, documentId: props.refId, documentKind: props.kind, + ensureDocument: props.ensureDocument, }) const latestCallbacks = useRef({ onContentChange: props.onContentChange, onTextChange: props.onTextChange, }) + const bindCrdtEditor = props.crdt?.bindEditor + const lexicalEditorRef = useRef(null) + const [lexicalEditor, setLexicalEditor] = useState(null) + const crdtLocked = Boolean(props.crdt?.locked) + const crdtLockedRef = useRef(crdtLocked) + crdtLockedRef.current = crdtLocked + useEffect(() => { + lexicalEditorRef.current?.setEditable(!crdtLocked) + }, [crdtLocked]) latestCallbacks.current = { onContentChange: props.onContentChange, @@ -5202,7 +5491,9 @@ function RichWriteSurface(props: { }, initialValue: parseSerializedEditorState(props.content), systemMessages: [ - ...buildDocumentBashSystemMessages(), + ...buildDocumentBashSystemMessages({ + shared: Boolean(props.crdt?.active), + }), ...dynamicCatalogMessages, ...(props.metaFieldsSchema ? buildMetaSystemMessages(props.metaFieldsSchema) @@ -5228,7 +5519,13 @@ function RichWriteSurface(props: { onChange: (value) => { latestCallbacks.current.onContentChange(JSON.stringify(value)) }, - onEditorReady: agent.onEditorReady, + onEditorReady: (editor) => { + lexicalEditorRef.current = editor + setLexicalEditor(editor) + if (crdtLockedRef.current) editor?.setEditable(false) + agent.onEditorReady(editor) + bindCrdtEditor?.(editor) + }, onPinSelection: props.onPinSelection, onTextChange: (text) => { latestCallbacks.current.onTextChange(text) @@ -5248,6 +5545,13 @@ function RichWriteSurface(props: { > + {props.crdt?.active ? ( + + ) : null} diff --git a/apps/admin/src/features/write/components/agent/use-write-agent.ts b/apps/admin/src/features/write/components/agent/use-write-agent.ts index 41ab7cdb789..f85ac21201a 100644 --- a/apps/admin/src/features/write/components/agent/use-write-agent.ts +++ b/apps/admin/src/features/write/components/agent/use-write-agent.ts @@ -15,6 +15,8 @@ import { toast } from 'sonner' import type { ProviderModelsResponse } from '~/api/ai' import { getModels } from '~/api/ai' +import { agentCommitMessage } from '~/features/write/crdt/agent-commit-message' +import type { AgentTurn } from '~/features/write/crdt/crdt-draft-session' import type { AgentSessionMeta } from '~/hooks/use-agent-session-manager' import { useAgentSessionManager } from '~/hooks/use-agent-session-manager' import { useLocalStorageState } from '~/hooks/use-local-storage-state' @@ -87,10 +89,17 @@ function deriveDocumentSessionId( return `${documentKind}:${documentId}` } +export interface AgentDocument { + active: boolean + startAgentTurn: (message: string) => AgentTurn | null +} + export function useWriteAgent(opts: { agentVisible: boolean + document?: AgentDocument documentId?: string documentKind: 'note' | 'page' | 'post' + ensureDocument?: () => Promise }): WriteAgentController { const { t } = useI18n() @@ -139,6 +148,11 @@ export function useWriteAgent(opts: { }, [providerGroups, selectedModel, selectModel]) const agentLoopRef = useRef(null) + const documentRef = useRef(opts.document) + documentRef.current = opts.document + const ensureDocumentRef = useRef(opts.ensureDocument) + ensureDocumentRef.current = opts.ensureDocument + const turnRef = useRef(null) const lexicalEditorRef = useRef(null) const dynamicTools = useMemo( () => buildDynamicTools(() => lexicalEditorRef.current), @@ -147,6 +161,12 @@ export function useWriteAgent(opts: { const documentBash = useMemo( () => createDocumentBashWorkspace({ + applyDocument: (target, focusBlockId) => { + if (!turnRef.current) throw new Error('No agent turn is open') + turnRef.current.apply(target, focusBlockId) + }, + ownsWrites: () => turnRef.current !== null, + onSeed: () => turnRef.current?.markView(), getBaseState: () => { const editor = lexicalEditorRef.current if (!editor) return null @@ -244,6 +264,13 @@ export function useWriteAgent(opts: { toast.error(t('write.agent.selectModelFirst')) return } + const ensureDocument = ensureDocumentRef.current + if (ensureDocument && !documentRef.current?.active) { + void ensureDocument().then((ready) => { + if (ready && documentRef.current?.active) sendMessage(message) + }) + return + } if (!agentLoopRef.current) { toast.error(t('write.agent.notReady')) return @@ -268,6 +295,11 @@ export function useWriteAgent(opts: { const startRun = (loop: AgentLoopHandle, message: string) => { documentBash.beginRun() + const shared = documentRef.current + const turn = shared?.active + ? shared.startAgentTurn(agentCommitMessage(message)) + : null + turnRef.current = turn const promise = loop.run(message).catch((error: unknown) => { if ((error as Error)?.name === 'AbortError') return const detail = error instanceof Error ? error.message : String(error) @@ -277,6 +309,10 @@ export function useWriteAgent(opts: { activeRunRef.current = promise void promise.then(() => { if (activeRunRef.current === promise) activeRunRef.current = null + if (turn) { + turn.end() + if (turnRef.current === turn) turnRef.current = null + } }) } diff --git a/apps/admin/src/features/write/components/collab/CollaboratorsSection.test.tsx b/apps/admin/src/features/write/components/collab/CollaboratorsSection.test.tsx new file mode 100644 index 00000000000..86e8c9dcefe --- /dev/null +++ b/apps/admin/src/features/write/components/collab/CollaboratorsSection.test.tsx @@ -0,0 +1,163 @@ +import { QueryClient, QueryClientProvider } from '@tanstack/react-query' +import { act, createElement } from 'react' +import type { Root } from 'react-dom/client' +import { createRoot } from 'react-dom/client' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +import { + createCollabInvite, + listCollabInvites, + listCollaborators, + removeCollaborator, + updateCollaborator, +} from '~/api/collab' +import { I18nProvider } from '~/i18n' + +import { CollaboratorsSection } from './CollaboratorsSection' + +vi.mock('~/api/collab', () => ({ + collabInviteUrlOf: (token: string) => + `https://admin.test/#/collab/join/${token}`, + createCollabInvite: vi.fn(), + listCollabInvites: vi.fn(), + listCollaborators: vi.fn(), + removeCollaborator: vi.fn(), + revokeCollabInvite: vi.fn(), + updateCollaborator: vi.fn(), +})) + +const socketHandlers = new Map void>() +vi.mock('~/socket/SocketBridge', () => ({ + subscribeAdminSocket: (listener: (socket: unknown) => void) => { + listener({ + on: (event: string, handler: (payload: unknown) => void) => { + socketHandlers.set(event, handler) + return () => socketHandlers.delete(event) + }, + }) + return () => undefined + }, +})) + +let container: HTMLDivElement +let root: Root + +const render = async () => { + const client = new QueryClient({ + defaultOptions: { queries: { retry: false } }, + }) + await act(async () => { + root.render( + createElement( + QueryClientProvider, + { client }, + createElement( + I18nProvider, + null, + createElement(CollaboratorsSection, { documentId: 'doc-1' }), + ), + ), + ) + }) + await act(async () => { + await new Promise((resolve) => setTimeout(resolve, 0)) + }) +} + +const click = async (element: Element | null) => { + await act(async () => { + ;(element as HTMLElement).click() + await new Promise((resolve) => setTimeout(resolve, 0)) + }) +} + +describe('CollaboratorsSection', () => { + beforeEach(() => { + container = document.createElement('div') + document.body.append(container) + root = createRoot(container) + vi.mocked(listCollabInvites).mockResolvedValue([]) + vi.mocked(listCollaborators).mockResolvedValue([ + { + avatar: null, + contributedAt: null, + createdAt: '2026-10-25T00:00:00.000Z', + name: 'Bob', + readerId: 'r1', + removedAt: null, + showInByline: true, + }, + ]) + Object.defineProperty(navigator, 'clipboard', { + configurable: true, + value: { writeText: vi.fn(async () => undefined) }, + }) + }) + + afterEach(() => { + act(() => root.unmount()) + container.remove() + vi.clearAllMocks() + }) + + it('copies a fresh invite link to the clipboard', async () => { + vi.mocked(createCollabInvite).mockResolvedValue({ + createdAt: '2026-10-25T00:00:00.000Z', + expiresAt: '2026-11-01T00:00:00.000Z', + id: 'inv-1', + maxUses: null, + token: 'tok', + uses: 0, + }) + await render() + + const invite = [...container.querySelectorAll('button')].find((button) => + button.textContent?.match(/invite|邀请/i), + ) + await click(invite ?? null) + + expect(createCollabInvite).toHaveBeenCalledWith('doc-1') + expect(navigator.clipboard.writeText).toHaveBeenCalledWith( + 'https://admin.test/#/collab/join/tok', + ) + }) + + it('turns a co-author off in the byline and removes them', async () => { + vi.mocked(updateCollaborator).mockResolvedValue(undefined) + vi.mocked(removeCollaborator).mockResolvedValue(undefined) + await render() + + const row = container.querySelector('[data-collaborator="r1"]')! + expect(row.textContent).toContain('Bob') + await click(row.querySelector('input[type="checkbox"]')) + await click(row.querySelector('button')) + + expect(updateCollaborator).toHaveBeenCalledWith('doc-1', 'r1', { + showInByline: false, + }) + expect(removeCollaborator).toHaveBeenCalledWith('doc-1', 'r1') + }) + + it('warns that co-authors can see the edit history', async () => { + await render() + + expect(container.querySelector('[data-history-notice]')).not.toBeNull() + }) + + it('refreshes the list when a reader joins from another device', async () => { + await render() + expect(listCollaborators).toHaveBeenCalledTimes(1) + + await act(async () => { + socketHandlers.get('collab.collaborators.changed')?.({ + documentId: 'other', + }) + socketHandlers.get('collab.collaborators.changed')?.({ + documentId: 'doc-1', + }) + await new Promise((resolve) => setTimeout(resolve, 0)) + }) + + expect(listCollaborators).toHaveBeenCalledTimes(2) + }) +}) diff --git a/apps/admin/src/features/write/components/collab/CollaboratorsSection.tsx b/apps/admin/src/features/write/components/collab/CollaboratorsSection.tsx new file mode 100644 index 00000000000..3f5b73b7302 --- /dev/null +++ b/apps/admin/src/features/write/components/collab/CollaboratorsSection.tsx @@ -0,0 +1,197 @@ +import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' +import { Copy, Link2, Loader2, Trash2, UserMinus, Users } from 'lucide-react' +import { useEffect } from 'react' +import { toast } from 'sonner' + +import { + collabInviteUrlOf, + createCollabInvite, + listCollabInvites, + listCollaborators, + removeCollaborator, + revokeCollabInvite, + updateCollaborator, +} from '~/api/collab' +import { useI18n } from '~/i18n' +import { adminQueryKeys } from '~/query/keys' +import { subscribeAdminSocket } from '~/socket/SocketBridge' +import { Button } from '~/ui/primitives/button' + +export function CollaboratorsSection(props: { documentId: string }) { + const { format, t } = useI18n() + const queryClient = useQueryClient() + const { documentId } = props + const invitesKey = adminQueryKeys.collab.invites(documentId) + const collaboratorsKey = adminQueryKeys.collab.collaborators(documentId) + const invitesQuery = useQuery({ + queryFn: () => listCollabInvites(documentId), + queryKey: invitesKey, + }) + const collaboratorsQuery = useQuery({ + queryFn: () => listCollaborators(documentId), + queryKey: collaboratorsKey, + }) + const refresh = () => + Promise.all([ + queryClient.invalidateQueries({ queryKey: invitesKey }), + queryClient.invalidateQueries({ queryKey: collaboratorsKey }), + ]) + + useEffect(() => { + let off: (() => void) | undefined + const unsubscribe = subscribeAdminSocket((socket) => { + off?.() + off = socket?.on( + 'collab.collaborators.changed', + (payload: { documentId?: string }) => { + if (payload?.documentId !== documentId) return + void queryClient.invalidateQueries({ + queryKey: adminQueryKeys.collab.invites(documentId), + }) + void queryClient.invalidateQueries({ + queryKey: adminQueryKeys.collab.collaborators(documentId), + }) + }, + ) + }) + return () => { + off?.() + unsubscribe() + } + }, [documentId, queryClient]) + + const createInvite = useMutation({ + mutationFn: () => createCollabInvite(documentId), + onError: () => toast.error(t('write.collaborators.failed')), + onSuccess: async (invite) => { + await navigator.clipboard + .writeText(collabInviteUrlOf(invite.token)) + .catch(() => undefined) + toast.success(t('write.collaborators.linkCopied')) + await refresh() + }, + }) + const revokeInvite = useMutation({ + mutationFn: (inviteId: string) => revokeCollabInvite(documentId, inviteId), + onSuccess: refresh, + }) + const toggleByline = useMutation({ + mutationFn: (input: { readerId: string; showInByline: boolean }) => + updateCollaborator(documentId, input.readerId, { + showInByline: input.showInByline, + }), + onSuccess: refresh, + }) + const remove = useMutation({ + mutationFn: (readerId: string) => removeCollaborator(documentId, readerId), + onSuccess: async () => { + toast.success(t('write.collaborators.removed')) + await refresh() + }, + }) + + const invites = invitesQuery.data ?? [] + const collaborators = (collaboratorsQuery.data ?? []).filter( + (person) => !person.removedAt, + ) + + return ( +
    +
    +
    +

    + {t('write.collaborators.historyNotice')} +

    + {invites.map((invite) => ( +
    +
    + ))} + {collaborators.map((person) => ( +
    + + {person.avatar ? ( + + ) : ( + (person.name || '?').slice(0, 1).toUpperCase() + )} + + + {person.name} + + + +
    + ))} +
    + ) +} diff --git a/apps/admin/src/features/write/components/collab/PeerAvatars.test.tsx b/apps/admin/src/features/write/components/collab/PeerAvatars.test.tsx new file mode 100644 index 00000000000..f9bdeda3418 --- /dev/null +++ b/apps/admin/src/features/write/components/collab/PeerAvatars.test.tsx @@ -0,0 +1,104 @@ +import { act, createElement } from 'react' +import type { Root } from 'react-dom/client' +import { createRoot } from 'react-dom/client' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' + +import { I18nProvider } from '~/i18n' + +import type { RemotePresence } from '../../crdt/collab-presence' +import { PeerAvatars } from './PeerAvatars' + +const sourceOf = (peers: RemotePresence[]) => ({ + get: () => peers, + subscribe: () => () => undefined, +}) + +let container: HTMLDivElement +let root: Root + +const peer = (key: string, name: string, id = 'owner'): RemotePresence => ({ + anchor: null, + color: '#0090ff', + focus: null, + key, + user: { avatar: null, id, name }, +}) + +const render = (peers: RemotePresence[]) => + act(() => { + root.render( + createElement( + I18nProvider, + null, + createElement(PeerAvatars, { peers: sourceOf(peers), selfId: 'owner' }), + ), + ) + }) + +beforeEach(() => { + container = document.createElement('div') + document.body.append(container) + root = createRoot(container) +}) + +afterEach(() => { + act(() => root.unmount()) + document.body.innerHTML = '' +}) + +describe('PeerAvatars', () => { + it('renders nothing when nobody else is here', () => { + render([]) + + expect(container.textContent).toBe('') + }) + + it('shows up to four members and counts the rest', () => { + render([ + peer('c1', 'Ann', 'r1'), + peer('c2', 'Bob', 'r2'), + peer('c3', 'Cid', 'r3'), + peer('c4', 'Dee', 'r4'), + peer('c5', 'Eve', 'r5'), + peer('c6', 'Fay', 'r6'), + ]) + + expect(container.querySelectorAll('[data-peer]')).toHaveLength(4) + expect(container.textContent).toContain('+2') + }) + + it('labels the owner’s own other tabs', () => { + render([peer('c1', 'Owner')]) + + const avatar = container.querySelector('[data-peer]')! + expect(avatar.getAttribute('title')).toContain('Owner') + expect(avatar.getAttribute('title')).toContain('另一个标签页') + }) + + it('shows one avatar per person even with several tabs open', () => { + render([ + peer('c1', 'Ann', 'r1'), + peer('c2', 'Ann', 'r1'), + peer('c3', 'Bob', 'r2'), + ]) + + expect(container.querySelectorAll('[data-peer]')).toHaveLength(2) + }) + + it('falls back to the initial when the avatar image fails to load', () => { + render([ + { + ...peer('c1', 'Jerome', 'r1'), + user: { avatar: 'https://broken.test/a.png', id: 'r1', name: 'Jerome' }, + }, + ]) + + const image = container.querySelector('[data-peer] img')! + act(() => { + image.dispatchEvent(new Event('error')) + }) + + expect(container.querySelector('[data-peer] img')).toBeNull() + expect(container.querySelector('[data-peer]')!.textContent).toBe('J') + }) +}) diff --git a/apps/admin/src/features/write/components/collab/PeerAvatars.tsx b/apps/admin/src/features/write/components/collab/PeerAvatars.tsx new file mode 100644 index 00000000000..da9e3cdbd0a --- /dev/null +++ b/apps/admin/src/features/write/components/collab/PeerAvatars.tsx @@ -0,0 +1,80 @@ +import { type CSSProperties, useState } from 'react' + +import { useI18n } from '~/i18n' + +import type { RemotePresence } from '../../crdt/collab-presence' +import { type PeersSource, usePeers } from '../../crdt/peers-store' + +const VISIBLE = 4 + +const onePerPerson = (peers: RemotePresence[]) => { + const seen = new Set() + return peers.filter((peer) => { + if (seen.has(peer.user.id)) return false + seen.add(peer.user.id) + return true + }) +} + +export function PeerAvatars(props: { peers: PeersSource; selfId: string }) { + const { t } = useI18n() + const people = onePerPerson(usePeers(props.peers)) + if (!people.length) return null + const visible = people.slice(0, VISIBLE) + const hidden = people.length - visible.length + + return ( +
    + {visible.map((peer, index) => { + const name = peer.user.name || '?' + const title = + peer.user.id === props.selfId + ? t('write.collab.peerOtherTab', { name }) + : name + return ( + + + + ) + })} + {hidden > 0 ? ( + + ) : null} +
    + ) +} + +function PeerFace(props: { avatar: string | null; name: string }) { + const [failed, setFailed] = useState(null) + if (!props.avatar || failed === props.avatar) { + return <>{props.name.slice(0, 1).toUpperCase()} + } + return ( + setFailed(props.avatar)} + src={props.avatar} + /> + ) +} diff --git a/apps/admin/src/features/write/components/collab/RemoteCarets.test.tsx b/apps/admin/src/features/write/components/collab/RemoteCarets.test.tsx new file mode 100644 index 00000000000..d21048ab576 --- /dev/null +++ b/apps/admin/src/features/write/components/collab/RemoteCarets.test.tsx @@ -0,0 +1,100 @@ +import { + $createParagraphNode, + $createTextNode, + $getRoot, + createEditor, +} from 'lexical' +import { act, createElement } from 'react' +import type { Root } from 'react-dom/client' +import { createRoot } from 'react-dom/client' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' + +import type { RemotePresence } from '../../crdt/collab-presence' +import { RemoteCarets } from './RemoteCarets' + +const sourceOf = (peers: RemotePresence[]) => ({ + get: () => peers, + subscribe: () => () => undefined, +}) + +let host: HTMLDivElement +let root: Root + +const peer = (focus: RemotePresence['focus']): RemotePresence => ({ + anchor: focus, + color: '#e5484d', + focus, + key: 'c1', + user: { avatar: null, id: 'owner', name: 'Ann' }, +}) + +beforeEach(() => { + host = document.createElement('div') + document.body.append(host) + root = createRoot(host) +}) + +afterEach(() => { + act(() => root.unmount()) + document.body.innerHTML = '' +}) + +const mountEditor = () => { + const frame = document.createElement('div') + const editable = document.createElement('div') + editable.contentEditable = 'true' + frame.append(editable) + document.body.append(frame) + const editor = createEditor({ + onError: (error) => { + throw error + }, + }) + editor.setRootElement(editable) + let key = '' + editor.update( + () => { + const text = $createTextNode('hello') + $getRoot().append($createParagraphNode().append(text)) + key = text.getKey() + }, + { discrete: true }, + ) + return { editor, frame, key } +} + +describe('RemoteCarets', () => { + it('draws a named caret for a member whose position resolves', () => { + const { editor, frame, key } = mountEditor() + + act(() => { + root.render( + createElement(RemoteCarets, { + editor, + peers: sourceOf([peer({ id: '1@1', offset: 2 })]), + resolveCaret: () => ({ key, offset: 2 }), + }), + ) + }) + + const caret = frame.querySelector('[data-remote-caret]') + expect(caret).not.toBeNull() + expect(caret!.textContent).toBe('Ann') + }) + + it('draws nothing for a member without a resolvable caret', () => { + const { editor, frame } = mountEditor() + + act(() => { + root.render( + createElement(RemoteCarets, { + editor, + peers: sourceOf([peer({ id: '9@9', offset: 0 }), peer(null)]), + resolveCaret: () => null, + }), + ) + }) + + expect(frame.querySelector('[data-remote-caret]')).toBeNull() + }) +}) diff --git a/apps/admin/src/features/write/components/collab/RemoteCarets.tsx b/apps/admin/src/features/write/components/collab/RemoteCarets.tsx new file mode 100644 index 00000000000..c432adf457f --- /dev/null +++ b/apps/admin/src/features/write/components/collab/RemoteCarets.tsx @@ -0,0 +1,174 @@ +import type { LexicalEditor, NodeKey } from 'lexical' +import { useEffect, useState } from 'react' +import { createPortal } from 'react-dom' + +import type { CaretPoint, RemotePresence } from '../../crdt/collab-presence' +import { type PeersSource, usePeers } from '../../crdt/peers-store' + +type Resolve = (point: CaretPoint) => { key: NodeKey; offset: number } | null + +interface Box { + height: number + left: number + top: number + width: number +} + +interface DrawnPeer { + caret: Box + color: string + key: string + name: string + selection: Box[] +} + +const domPoint = ( + editor: LexicalEditor, + point: { key: NodeKey; offset: number }, +): { node: Node; offset: number } | null => { + const element = editor.getElementByKey(point.key) + if (!element) return null + const walker = document.createTreeWalker(element, NodeFilter.SHOW_TEXT) + const text = walker.nextNode() as Text | null + if (!text) return { node: element, offset: 0 } + return { node: text, offset: Math.min(point.offset, text.length) } +} + +const relative = (rect: DOMRect, origin: DOMRect, frame: HTMLElement): Box => ({ + height: rect.height, + left: rect.left - origin.left + frame.scrollLeft, + top: rect.top - origin.top + frame.scrollTop, + width: rect.width, +}) + +const measure = ( + editor: LexicalEditor, + frame: HTMLElement, + peer: RemotePresence, + resolve: Resolve, +): DrawnPeer | null => { + const focusPoint = peer.focus && resolve(peer.focus) + const focus = focusPoint && domPoint(editor, focusPoint) + if (!focus) return null + const origin = frame.getBoundingClientRect() + const range = document.createRange() + range.setStart(focus.node, focus.offset) + range.collapse(true) + const caretRect = + range.getClientRects()[0] ?? + (focus.node instanceof Element + ? focus.node.getBoundingClientRect() + : range.getBoundingClientRect()) + + const selection: Box[] = [] + const anchorPoint = peer.anchor && resolve(peer.anchor) + const anchor = anchorPoint && domPoint(editor, anchorPoint) + if (anchor) { + const span = document.createRange() + span.setStart(anchor.node, anchor.offset) + span.setEnd(focus.node, focus.offset) + if (span.collapsed) { + span.setStart(focus.node, focus.offset) + span.setEnd(anchor.node, anchor.offset) + } + for (const rect of span.getClientRects()) { + if (rect.width > 0) selection.push(relative(rect, origin, frame)) + } + } + + return { + caret: relative(caretRect, origin, frame), + color: peer.color, + key: peer.key, + name: peer.user.name, + selection, + } +} + +export function RemoteCarets(props: { + editor: LexicalEditor | null + peers: PeersSource + resolveCaret: Resolve +}) { + const { editor, resolveCaret } = props + const peers = usePeers(props.peers) + const [frame, setFrame] = useState(null) + const [drawn, setDrawn] = useState([]) + + useEffect(() => { + if (!editor) return + return editor.registerRootListener((rootElement) => { + const parent = rootElement?.parentElement ?? null + if (parent && getComputedStyle(parent).position === 'static') { + parent.style.position = 'relative' + } + setFrame(parent) + }) + }, [editor]) + + useEffect(() => { + if (!editor || !frame) { + setDrawn([]) + return + } + const update = () => + setDrawn( + peers + .map((peer) => measure(editor, frame, peer, resolveCaret)) + .filter((peer): peer is DrawnPeer => peer !== null), + ) + update() + let raf = 0 + const schedule = () => { + cancelAnimationFrame(raf) + raf = requestAnimationFrame(update) + } + const offUpdate = editor.registerUpdateListener(schedule) + const resize = new ResizeObserver(schedule) + resize.observe(frame) + return () => { + cancelAnimationFrame(raf) + offUpdate() + resize.disconnect() + } + }, [editor, frame, peers, resolveCaret]) + + if (!frame || !drawn.length) return null + + return createPortal( + <> + {drawn.map((peer) => ( +
    + {peer.selection.map((box, index) => ( +
    + ))} +
    +
    + + {peer.name} + +
    +
    + ))} + , + frame, + ) +} diff --git a/apps/admin/src/features/write/components/timeline/build-timeline.test.ts b/apps/admin/src/features/write/components/timeline/build-timeline.test.ts new file mode 100644 index 00000000000..3b01564a0eb --- /dev/null +++ b/apps/admin/src/features/write/components/timeline/build-timeline.test.ts @@ -0,0 +1,116 @@ +import { describe, expect, it } from 'vitest' + +import type { DraftHistory } from '~/models/draft' + +import { buildTimeline } from './build-timeline' + +const change = ( + peer: `${number}`, + counter: number, + timestamp: number, + message: string, + deps: { counter: number; peer: `${number}` }[] = [], +) => ({ deps, id: { counter, peer }, lamport: counter, message, timestamp }) + +const at = (seconds: number) => new Date(seconds * 1000).toISOString() + +describe('buildTimeline', () => { + it('merges checkpoints and grouped steps newest first', () => { + const history: DraftHistory = { + changes: [ + change('2', 9, 400, 'agent:shorten', [{ counter: 5, peer: '1' }]), + change('2', 8, 390, 'agent:shorten', [{ counter: 4, peer: '1' }]), + change('1', 5, 300, 'human:owner', [{ counter: 4, peer: '1' }]), + change('1', 4, 280, 'human:owner', [{ counter: 3, peer: '1' }]), + change('1', 3, 100, 'import 7'), + ], + checkpoints: [ + { + contentFormat: 'lexical', + createdAt: at(350), + frontiers: [], + id: 'c2', + kind: 'save', + parentRevisionId: 'c1', + title: 'T', + }, + { + contentFormat: 'lexical', + createdAt: at(100), + frontiers: [], + id: 'c1', + kind: 'import', + parentRevisionId: 'p', + title: 'T', + }, + { + contentFormat: 'lexical', + createdAt: at(50), + frontiers: null, + id: 'p', + kind: null, + parentRevisionId: null, + title: 'T', + }, + ], + } + + const rows = buildTimeline(history, 'p') + + expect( + rows.map((row) => + row.type === 'step' ? `${row.kind}:${row.count}` : row.id, + ), + ).toEqual(['agent:2', 'c2', 'human:2', 'c1', 'other:1', 'p']) + const agent = rows[0] + expect(agent).toMatchObject({ + before: [{ counter: 4, peer: '1' }], + change: { counter: 9, peer: '2' }, + undoable: true, + }) + expect(rows.at(-1)).toMatchObject({ published: true }) + }) + + it('splits human edits separated by an idle gap', () => { + const rows = buildTimeline( + { + changes: [ + change('1', 2, 1000, 'human:owner'), + change('1', 1, 100, 'human:owner'), + ], + checkpoints: [], + }, + null, + ) + + expect(rows).toHaveLength(2) + }) + + it('does not offer undo for server bookkeeping steps', () => { + const rows = buildTimeline( + { changes: [change('1', 1, 1, 'restore c1')], checkpoints: [] }, + null, + ) + + expect(rows[0]).toMatchObject({ kind: 'restore', undoable: false }) + }) + + it('keeps each co-author in their own steps and names who wrote them', () => { + const history: DraftHistory = { + changes: [ + change('3', 2, 210, 'human:r1'), + change('1', 6, 205, 'human:owner'), + change('3', 1, 200, 'human:r1'), + ], + checkpoints: [], + } + + const steps = buildTimeline(history, null).filter( + (row) => row.type === 'step', + ) + + expect( + steps.map((row) => (row.type === 'step' ? row.author : undefined)), + ).toEqual(['r1', null, 'r1']) + }) +}) diff --git a/apps/admin/src/features/write/components/timeline/build-timeline.ts b/apps/admin/src/features/write/components/timeline/build-timeline.ts new file mode 100644 index 00000000000..3ad0b786de4 --- /dev/null +++ b/apps/admin/src/features/write/components/timeline/build-timeline.ts @@ -0,0 +1,103 @@ +import type { + CheckpointKind, + CrdtOpId, + DraftChange, + DraftHistory, +} from '~/models/draft' + +export type StepKind = 'agent' | 'human' | 'other' | 'restore' | 'undo' + +export interface TimelineStep { + author: string | null + before: CrdtOpId[] + change: CrdtOpId + count: number + kind: StepKind + message: string + time: number + type: 'step' + undoable: boolean +} + +export interface TimelineCheckpoint { + id: string + kind: CheckpointKind | null + published: boolean + time: number + title: string + type: 'checkpoint' +} + +export type TimelineRow = TimelineCheckpoint | TimelineStep + +const HUMAN_IDLE_GAP_SECONDS = 60 + +const kindOf = (message = ''): StepKind => { + if (message.startsWith('human')) return 'human' + if (message.startsWith('agent')) return 'agent' + if (message.startsWith('restore')) return 'restore' + if (message.startsWith('undo')) return 'undo' + return 'other' +} + +const authorOf = (message: string) => { + const match = /^human:(.+)$/.exec(message) + return match && match[1] !== 'owner' ? match[1] : null +} + +const groupSteps = (changes: DraftChange[]): TimelineStep[] => { + const steps: TimelineStep[] = [] + let oldestInGroup = 0 + let groupPeer = '' + for (const entry of changes) { + const kind = kindOf(entry.message) + const message = entry.message ?? '' + const last = steps.at(-1) + const joins = + last?.kind === kind && + last.message === message && + ((kind === 'human' && + oldestInGroup - entry.timestamp <= HUMAN_IDLE_GAP_SECONDS) || + (kind === 'agent' && groupPeer === entry.id.peer)) + if (joins && last) { + last.count += 1 + last.before = entry.deps + } else { + groupPeer = entry.id.peer + steps.push({ + author: authorOf(message), + before: entry.deps, + change: entry.id, + count: 1, + kind, + message, + time: entry.timestamp * 1000, + type: 'step', + undoable: kind === 'human' || kind === 'agent', + }) + } + oldestInGroup = entry.timestamp + } + return steps +} + +export function buildTimeline( + history: DraftHistory, + publishedRevisionId: string | null, +): TimelineRow[] { + const checkpoints: TimelineCheckpoint[] = history.checkpoints.map( + (checkpoint) => ({ + id: checkpoint.id, + kind: checkpoint.kind ?? null, + published: checkpoint.id === publishedRevisionId, + time: Date.parse(checkpoint.createdAt), + title: checkpoint.title, + type: 'checkpoint', + }), + ) + return [...checkpoints, ...groupSteps(history.changes)].sort( + (left, right) => + right.time - left.time || + (left.type === right.type ? 0 : left.type === 'checkpoint' ? -1 : 1), + ) +} diff --git a/apps/admin/src/features/write/crdt/agent-commit-message.test.ts b/apps/admin/src/features/write/crdt/agent-commit-message.test.ts new file mode 100644 index 00000000000..fea2ff5bf74 --- /dev/null +++ b/apps/admin/src/features/write/crdt/agent-commit-message.test.ts @@ -0,0 +1,17 @@ +import { describe, expect, it } from 'vitest' + +import { agentCommitMessage } from './agent-commit-message' + +describe('agentCommitMessage', () => { + it('labels a turn with its request, whitespace folded', () => { + expect(agentCommitMessage(' tighten the\nintro ')).toBe( + 'agent:tighten the intro', + ) + }) + + it('caps the summary length', () => { + expect(agentCommitMessage('a'.repeat(200)).length).toBe( + 'agent:'.length + 60, + ) + }) +}) diff --git a/apps/admin/src/features/write/crdt/agent-commit-message.ts b/apps/admin/src/features/write/crdt/agent-commit-message.ts new file mode 100644 index 00000000000..eb09a65a7a6 --- /dev/null +++ b/apps/admin/src/features/write/crdt/agent-commit-message.ts @@ -0,0 +1,4 @@ +const SUMMARY_LIMIT = 60 + +export const agentCommitMessage = (request: string) => + `agent:${request.replaceAll(/\s+/g, ' ').trim().slice(0, SUMMARY_LIMIT)}` diff --git a/apps/admin/src/features/write/crdt/collab-channel.test.ts b/apps/admin/src/features/write/crdt/collab-channel.test.ts new file mode 100644 index 00000000000..5a1191a35ca --- /dev/null +++ b/apps/admin/src/features/write/crdt/collab-channel.test.ts @@ -0,0 +1,366 @@ +import { describe, expect, it, vi } from 'vitest' + +import { ApiRequestError } from '~/api/http' + +import { type CollabSocket, createCollabChannel } from './collab-channel' + +const b64 = (bytes: number[]) => btoa(String.fromCharCode(...bytes)) + +const fakeSocket = (initial: CollabSocket['state'] = 'open') => { + const handlers = new Map void>>() + const requests: { event: string; payload: any }[] = [] + let reply: (event: string, payload: any) => Promise = async (event) => + event === 'collab.join' + ? { + data: { + connectionId: 'conn-x', + documentId: 'doc-1', + update: b64([1, 2]), + }, + ok: true, + } + : { ok: true } + const socket = { + emit: (event: string, payload: unknown) => { + for (const handler of handlers.get(event) ?? []) handler(payload) + }, + on: (event: string, handler: (payload: any) => void) => { + let set = handlers.get(event) + if (!set) { + set = new Set() + handlers.set(event, set) + } + set.add(handler) + return () => set.delete(handler) + }, + replyWith: (next: typeof reply) => { + reply = next + }, + request: vi.fn(async (event: string, payload: any) => { + requests.push({ event, payload }) + return reply(event, payload) + }), + requests, + state: initial, + } + return socket +} + +const httpTransport = () => ({ + pull: vi.fn(async () => new Uint8Array([7])), + push: vi.fn(async () => undefined), +}) + +const setup = (socket: ReturnType | null) => { + let listener: ((socket: CollabSocket | null) => void) | undefined + const http = httpTransport() + const onRemote = vi.fn() + const onReconnect = vi.fn() + const onJoined = vi.fn() + const channel = createCollabChannel({ + draftId: 'd1', + http, + onJoined, + onReconnect, + onRemote, + subscribeSocket: (next) => { + listener = next + next(socket) + return () => { + listener = undefined + } + }, + }) + return { + channel, + http, + onJoined, + onReconnect, + onRemote, + swapSocket: (next: CollabSocket | null) => listener?.(next), + } +} + +describe('createCollabChannel', () => { + it('joins over the socket and pushes updates as frames', async () => { + const socket = fakeSocket() + const { channel, http } = setup(socket) + + expect(await channel.pull()).toEqual(new Uint8Array([1, 2])) + await channel.push(new Uint8Array([3])) + + expect(socket.requests).toEqual([ + { event: 'collab.join', payload: { draftId: 'd1' } }, + { event: 'collab.update', payload: { draftId: 'd1', update: b64([3]) } }, + ]) + expect(http.pull).not.toHaveBeenCalled() + expect(http.push).not.toHaveBeenCalled() + expect(channel.live()).toBe(true) + }) + + it('reports the connection id the server gave it on join', async () => { + const { channel, onJoined } = setup(fakeSocket()) + + await channel.pull() + + expect(onJoined).toHaveBeenCalledWith('conn-x') + }) + + it('routes presence and membership frames for the joined document', async () => { + const socket = fakeSocket() + const presence = vi.fn() + const joined = vi.fn() + const left = vi.fn() + const channel = createCollabChannel({ + draftId: 'd1', + http: httpTransport(), + onPeerJoined: joined, + onPeerLeft: left, + onPresence: presence, + onRemote: () => undefined, + subscribeSocket: (next) => { + next(socket) + return () => undefined + }, + }) + await channel.pull() + + socket.emit('collab.presence', { + caret: null, + connectionId: 'c2', + documentId: 'doc-1', + entry: null, + user: { avatar: null, id: 'r1', name: 'Bob' }, + }) + socket.emit('collab.peer.joined', { + connectionId: 'c2', + documentId: 'doc-1', + }) + socket.emit('collab.peer.left', { connectionId: 'c2', documentId: 'doc-1' }) + channel.sendPresence({ caret: { anchor: null, focus: null } }) + + expect(presence).toHaveBeenCalledWith({ + caret: null, + connectionId: 'c2', + entry: null, + user: { avatar: null, id: 'r1', name: 'Bob' }, + }) + expect(joined).toHaveBeenCalledWith('c2') + expect(left).toHaveBeenCalledWith('c2') + expect(socket.requests.at(-1)).toEqual({ + event: 'collab.presence', + payload: { caret: { anchor: null, focus: null }, draftId: 'd1' }, + }) + }) + + it('sends the version it already has when re-joining', async () => { + const socket = fakeSocket() + const { channel } = setup(socket) + + await channel.pull(new Uint8Array([5, 6])) + + expect(socket.requests[0]).toEqual({ + event: 'collab.join', + payload: { draftId: 'd1', since: b64([5, 6]) }, + }) + }) + + it('applies relayed updates only for the joined document', async () => { + const socket = fakeSocket() + const { channel, onRemote } = setup(socket) + await channel.pull() + + socket.emit('collab.update', { documentId: 'other', update: b64([9]) }) + socket.emit('collab.update', { documentId: 'doc-1', update: b64([4]) }) + + expect(onRemote).toHaveBeenCalledTimes(1) + expect(onRemote).toHaveBeenCalledWith(new Uint8Array([4])) + }) + + it('falls back to HTTP while the socket is down', async () => { + const socket = fakeSocket('reconnecting') + const { channel, http } = setup(socket) + + expect(await channel.pull()).toEqual(new Uint8Array([7])) + await channel.push(new Uint8Array([3])) + + expect(http.pull).toHaveBeenCalledTimes(1) + expect(http.push).toHaveBeenCalledWith(new Uint8Array([3])) + expect(channel.live()).toBe(false) + }) + + it('sends oversized updates over HTTP even when the socket is live', async () => { + const socket = fakeSocket() + const { channel, http } = setup(socket) + await channel.pull() + + await channel.push(new Uint8Array(600 * 1024)) + + expect(http.push).toHaveBeenCalledTimes(1) + expect(socket.requests.map((request) => request.event)).toEqual([ + 'collab.join', + ]) + }) + + it('surfaces a rejected frame as the rejection error', async () => { + const socket = fakeSocket() + const { channel } = setup(socket) + await channel.pull() + socket.replyWith(async () => { + throw Object.assign(new Error('Request rejected'), { + code: 'DRAFT_CRDT_UPDATE_REJECTED', + }) + }) + + const error = await channel.push(new Uint8Array([1])).catch((e) => e) + + expect(error).toBeInstanceOf(ApiRequestError) + expect(error.code).toBe('DRAFT_CRDT_UPDATE_REJECTED') + }) + + it('asks to resync when the socket comes back after a drop', async () => { + const socket = fakeSocket() + const { channel, onReconnect } = setup(socket) + await channel.pull() + + socket.state = 'reconnecting' + socket.emit('$state', 'reconnecting') + expect(channel.live()).toBe(false) + socket.state = 'open' + socket.emit('$state', 'open') + + expect(onReconnect).toHaveBeenCalledTimes(1) + }) + + it('stops listening once disposed', async () => { + const socket = fakeSocket() + const { channel, onRemote } = setup(socket) + await channel.pull() + + channel.dispose() + socket.emit('collab.update', { documentId: 'doc-1', update: b64([4]) }) + + expect(onRemote).not.toHaveBeenCalled() + expect(socket.requests.at(-1)).toEqual({ + event: 'collab.leave', + payload: { draftId: 'd1' }, + }) + }) + + it('keeps updates relayed while the join is in flight', async () => { + const socket = fakeSocket() + let release: () => void = () => undefined + socket.replyWith( + (event) => + new Promise((resolve) => { + release = () => + resolve( + event === 'collab.join' + ? { + data: { + connectionId: 'c', + documentId: 'doc-1', + update: b64([1]), + }, + ok: true, + } + : { ok: true }, + ) + }), + ) + const { channel, onRemote } = setup(socket) + + const joining = channel.pull() + socket.emit('collab.update', { documentId: 'doc-1', update: b64([5]) }) + socket.emit('collab.update', { documentId: 'other', update: b64([6]) }) + release() + await joining + await Promise.resolve() + + expect(onRemote).toHaveBeenCalledTimes(1) + expect(onRemote).toHaveBeenCalledWith(new Uint8Array([5])) + }) + + it('loads over HTTP when the server cannot join the room', async () => { + const socket = fakeSocket() + socket.replyWith(async () => { + throw Object.assign(new Error('Request rejected'), { + code: 'UNSUPPORTED_EVENT', + }) + }) + const { channel, http } = setup(socket) + + expect(await channel.pull()).toEqual(new Uint8Array([7])) + expect(http.pull).toHaveBeenCalledTimes(1) + expect(channel.live()).toBe(false) + }) + + it('leaves again when disposed while a join is in flight', async () => { + const socket = fakeSocket() + let release: () => void = () => undefined + socket.replyWith((event) => + event === 'collab.join' + ? new Promise((resolve) => { + release = () => + resolve({ + data: { + connectionId: 'c', + documentId: 'doc-1', + update: b64([1]), + }, + ok: true, + }) + }) + : Promise.resolve({ ok: true }), + ) + const { channel, onJoined } = setup(socket) + + const joining = channel.pull() + channel.dispose() + release() + await joining.catch(() => undefined) + + expect(onJoined).not.toHaveBeenCalled() + expect(socket.requests.at(-1)).toEqual({ + event: 'collab.leave', + payload: { draftId: 'd1' }, + }) + }) + + it('joins once the socket opens when the first load went over HTTP', async () => { + const socket = fakeSocket('connecting') + const { channel, onReconnect } = setup(socket) + await channel.pull() + + socket.state = 'open' + socket.emit('$state', 'open') + + expect(onReconnect).toHaveBeenCalledTimes(1) + }) + + it('stops relaying and reports once the owner revokes access', async () => { + const socket = fakeSocket() + const onRevoked = vi.fn() + const onRemote = vi.fn() + const channel = createCollabChannel({ + draftId: 'd1', + http: httpTransport(), + onRemote, + onRevoked, + subscribeSocket: (next) => { + next(socket) + return () => undefined + }, + }) + await channel.pull() + + socket.emit('collab.revoked', { documentId: 'other' }) + expect(onRevoked).not.toHaveBeenCalled() + socket.emit('collab.revoked', { documentId: 'doc-1' }) + socket.emit('collab.update', { documentId: 'doc-1', update: b64([4]) }) + + expect(onRevoked).toHaveBeenCalledTimes(1) + expect(onRemote).not.toHaveBeenCalled() + expect(channel.live()).toBe(false) + }) +}) diff --git a/apps/admin/src/features/write/crdt/collab-channel.ts b/apps/admin/src/features/write/crdt/collab-channel.ts new file mode 100644 index 00000000000..d1e1ee31b2e --- /dev/null +++ b/apps/admin/src/features/write/crdt/collab-channel.ts @@ -0,0 +1,216 @@ +import type { WsClient, WsClientState } from '@mx-space/ws-client' + +import { ApiRequestError } from '~/api/http' +import { subscribeAdminSocket } from '~/socket/SocketBridge' + +import type { IncomingPresence, OutgoingPresence } from './collab-presence' +import { CRDT_REJECTED_CODE, type CrdtTransport } from './crdt-sync' + +export type CollabSocket = Pick & { + readonly state: WsClientState +} + +export interface CollabChannelOptions { + draftId: string + http: CrdtTransport + maxFrameBytes?: number + onJoined?: (connectionId: string) => void + onPeerJoined?: (connectionId: string) => void + onPeerLeft?: (connectionId: string) => void + onPresence?: (frame: IncomingPresence) => void + onReconnect?: () => void + onRemote: (update: Uint8Array) => void + onRevoked?: () => void + subscribeSocket?: ( + listener: (socket: CollabSocket | null) => void, + ) => () => void +} + +export interface CollabChannel extends CrdtTransport { + dispose: () => void + live: () => boolean + sendPresence: (frame: OutgoingPresence) => void +} + +interface DocumentFrame { + caret?: IncomingPresence['caret'] + connectionId?: string + documentId?: string + entry?: string | null + update?: string + user?: IncomingPresence['user'] +} + +const CHUNK = 0x8000 + +export const encodeBase64 = (bytes: Uint8Array) => { + let binary = '' + for (let index = 0; index < bytes.length; index += CHUNK) { + binary += String.fromCharCode(...bytes.subarray(index, index + CHUNK)) + } + return btoa(binary) +} + +export const decodeBase64 = (value: string) => { + const binary = atob(value) + const bytes = new Uint8Array(binary.length) + for (let index = 0; index < binary.length; index += 1) { + bytes[index] = binary.charCodeAt(index) + } + return bytes +} + +export function createCollabChannel( + options: CollabChannelOptions, +): CollabChannel { + const { draftId, http } = options + const maxFrameBytes = options.maxFrameBytes ?? 512 * 1024 + let socket: CollabSocket | null = null + let documentId: string | null = null + let joining: { frames: DocumentFrame[] } | null = null + let disposed = false + let unbind: Array<() => void> = [] + + const open = () => socket?.state === 'open' + const live = () => open() && documentId !== null + + const forDocument = + (handle: (frame: DocumentFrame) => void) => (frame: DocumentFrame) => { + if (!disposed && documentId && frame?.documentId === documentId) { + handle(frame) + } + } + + const bind = (next: CollabSocket | null) => { + for (const off of unbind) off() + unbind = [] + socket = next + if (!next) return + unbind = [ + next.on('collab.update', (frame: DocumentFrame) => { + if (joining) joining.frames.push(frame) + else + forDocument((relayed) => { + if (relayed.update) options.onRemote(decodeBase64(relayed.update)) + })(frame) + }), + next.on( + 'collab.presence', + forDocument((frame) => { + if (!frame.connectionId || !frame.user) return + options.onPresence?.({ + caret: frame.caret ?? null, + connectionId: frame.connectionId, + entry: frame.entry ?? null, + user: frame.user, + }) + }), + ), + next.on( + 'collab.revoked', + forDocument(() => { + documentId = null + options.onRevoked?.() + }), + ), + next.on( + 'collab.peer.joined', + forDocument((frame) => { + if (frame.connectionId) options.onPeerJoined?.(frame.connectionId) + }), + ), + next.on( + 'collab.peer.left', + forDocument((frame) => { + if (frame.connectionId) options.onPeerLeft?.(frame.connectionId) + }), + ), + next.on('$state', (state: WsClientState) => { + if (state !== 'open' || disposed || joining) return + documentId = null + options.onReconnect?.() + }), + ] + } + + const unsubscribe = (options.subscribeSocket ?? subscribeAdminSocket)( + (next) => { + const replaced = socket !== null && next !== socket && documentId + bind(next) + if (replaced && !disposed) { + documentId = null + options.onReconnect?.() + } + }, + ) + + return { + dispose: () => { + if (disposed) return + disposed = true + if (open() && documentId) { + void socket!.request('collab.leave', { draftId }).catch(() => undefined) + } + unsubscribe() + for (const off of unbind) off() + unbind = [] + }, + live, + pull: async (since) => { + if (!open()) return http.pull(since) + const attempt = { frames: [] as DocumentFrame[] } + joining = attempt + let ack: Awaited> + try { + ack = await socket!.request('collab.join', { + draftId, + ...(since ? { since: encodeBase64(since) } : {}), + }) + } catch { + if (joining === attempt) joining = null + documentId = null + return http.pull(since) + } finally { + if (joining === attempt) joining = null + } + const data = ack.data as { + connectionId: string + documentId: string + update: string + } + if (disposed) { + void socket?.request('collab.leave', { draftId }).catch(() => undefined) + return decodeBase64(data.update) + } + documentId = data.documentId + options.onJoined?.(data.connectionId) + for (const frame of attempt.frames) { + if (frame.documentId === documentId && frame.update) { + options.onRemote(decodeBase64(frame.update)) + } + } + return decodeBase64(data.update) + }, + push: async (bytes) => { + if (!live() || bytes.length > maxFrameBytes) return http.push(bytes) + try { + await socket!.request('collab.update', { + draftId, + update: encodeBase64(bytes), + }) + } catch (error) { + const code = (error as { code?: string }).code + if (code === CRDT_REJECTED_CODE) { + throw new ApiRequestError((error as Error).message, code, 400) + } + throw error + } + }, + sendPresence: (frame) => { + if (!live()) return + void socket! + .request('collab.presence', { draftId, ...frame }) + .catch(() => undefined) + }, + } +} diff --git a/apps/admin/src/features/write/crdt/collab-presence.test.ts b/apps/admin/src/features/write/crdt/collab-presence.test.ts new file mode 100644 index 00000000000..52e365ed32f --- /dev/null +++ b/apps/admin/src/features/write/crdt/collab-presence.test.ts @@ -0,0 +1,194 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +import { + AGENT_PRESENCE_COLOR, + createCollabPresence, + type OutgoingPresence, + PRESENCE_COLORS, +} from './collab-presence' + +const user = (id: string) => ({ avatar: null, id, name: `User ${id}` }) + +const member = (connectionId: string, id: string) => { + const outbox: OutgoingPresence[] = [] + const presence = createCollabPresence({ + send: (frame) => outbox.push(frame), + }) + const stamp = (frame: OutgoingPresence) => ({ + caret: frame.caret, + connectionId, + documentId: 'doc-1', + entry: frame.entry ?? null, + user: frame.entry + ? { avatar: null, id: frame.entry, name: 'AI' } + : user(id), + }) + return { connectionId, outbox, presence, stamp } +} + +const pair = () => { + const a = member('conn-a', 'owner') + const b = member('conn-b', 'r1') + const deliver = () => { + for (const frame of a.outbox.splice(0)) b.presence.apply(a.stamp(frame)) + for (const frame of b.outbox.splice(0)) a.presence.apply(b.stamp(frame)) + } + return { a, b, deliver } +} + +describe('createCollabPresence', () => { + beforeEach(() => { + vi.useFakeTimers() + }) + afterEach(() => { + vi.useRealTimers() + }) + + it('shows other members with the identity the server stamped', () => { + const { a, b, deliver } = pair() + const seen = vi.fn() + b.presence.subscribe(seen) + a.presence.start() + b.presence.start() + + a.presence.setCursor({ id: '1@2', offset: 3 }, { id: '1@2', offset: 5 }) + deliver() + + expect(b.presence.remotes()).toEqual([ + { + anchor: { id: '1@2', offset: 3 }, + color: expect.any(String), + focus: { id: '1@2', offset: 5 }, + key: 'conn-a', + user: user('owner'), + }, + ]) + expect(a.presence.remotes()).toEqual([ + expect.objectContaining({ key: 'conn-b', user: user('r1') }), + ]) + expect(seen).toHaveBeenCalled() + }) + + it('answers a newcomer with its caret and extras', () => { + const { a } = pair() + a.presence.start() + a.presence.setCursor({ id: '1@2', offset: 1 }, null) + a.presence.setExtra('agent:7', { anchor: null, focus: null }) + a.outbox.length = 0 + + a.presence.announce() + + expect(a.outbox).toEqual([ + { caret: { anchor: { id: '1@2', offset: 1 }, focus: null } }, + { caret: { anchor: null, focus: null }, entry: 'agent:7' }, + ]) + }) + + it('drops every entry of a connection that left', () => { + const { a, b, deliver } = pair() + a.presence.start() + a.presence.setExtra('agent:7', { anchor: null, focus: null }) + deliver() + expect(b.presence.remotes()).toHaveLength(2) + + b.presence.removeConnection('conn-a') + + expect(b.presence.remotes()).toEqual([]) + }) + + it('expires members it has not heard from and keeps its own state fresh', () => { + const { a, b, deliver } = pair() + a.presence.start() + b.presence.start() + deliver() + expect(b.presence.remotes()).toHaveLength(1) + + vi.advanceTimersByTime(10_000) + deliver() + vi.advanceTimersByTime(25_000) + expect(b.presence.remotes()).toHaveLength(1) + + a.presence.dispose() + vi.advanceTimersByTime(31_000) + expect(b.presence.remotes()).toEqual([]) + }) + + it('removes an entry the sender cleared', () => { + const { a, b, deliver } = pair() + a.presence.start() + a.presence.setExtra('agent:7', { + anchor: null, + focus: { id: '3@4', offset: 0 }, + }) + deliver() + expect(b.presence.remotes().map((peer) => peer.key)).toEqual([ + 'conn-a', + 'conn-a/agent:7', + ]) + + a.presence.setExtra('agent:7', null) + deliver() + + expect(b.presence.remotes().map((peer) => peer.key)).toEqual(['conn-a']) + }) + + it('lists its own agent entry locally, since the server never echoes it', () => { + const { a } = pair() + a.presence.start() + + a.presence.setExtra('agent:7', { anchor: null, focus: null }) + + expect(a.presence.remotes()).toEqual([ + expect.objectContaining({ + color: AGENT_PRESENCE_COLOR, + key: 'local/agent:7', + user: expect.objectContaining({ name: 'AI' }), + }), + ]) + }) + + it('picks a stable color per member and the agent color for agent entries', () => { + const { a, b, deliver } = pair() + a.presence.start() + a.presence.setExtra('agent:7', { anchor: null, focus: null }) + deliver() + + const [human, agent] = b.presence.remotes() + expect(PRESENCE_COLORS).toContain(human.color) + expect(agent.color).toBe(AGENT_PRESENCE_COLOR) + }) + + it('skips sending a caret that did not move', () => { + const { a } = pair() + a.presence.start() + a.outbox.length = 0 + + a.presence.setCursor({ id: '1@2', offset: 1 }, null) + a.presence.setCursor({ id: '1@2', offset: 1 }, null) + + expect(a.outbox).toHaveLength(1) + }) + + it('stays silent before it has joined a room', () => { + const { a } = pair() + + a.presence.setCursor({ id: '1@2', offset: 1 }, null) + + expect(a.outbox).toEqual([]) + }) + + it('gives everyone on screen a different color while the palette lasts', () => { + const presence = createCollabPresence({ send: () => undefined }) + for (const connectionId of ['conn-0', 'conn-8']) { + presence.apply({ + caret: { anchor: null, focus: null }, + connectionId, + entry: null, + user: user(connectionId), + }) + } + + const [first, second] = presence.remotes() + expect(first.color).not.toBe(second.color) + }) +}) diff --git a/apps/admin/src/features/write/crdt/collab-presence.ts b/apps/admin/src/features/write/crdt/collab-presence.ts new file mode 100644 index 00000000000..686d490db9c --- /dev/null +++ b/apps/admin/src/features/write/crdt/collab-presence.ts @@ -0,0 +1,208 @@ +export const PRESENCE_COLORS = [ + '#e5484d', + '#f76b15', + '#ffc53d', + '#30a46c', + '#12a594', + '#0090ff', + '#6e56cf', + '#d6409f', +] as const + +export const AGENT_PRESENCE_COLOR = '#8e4ec6' + +export type PresenceUser = { + avatar: string | null + id: string + name: string +} + +export type CaretPoint = { + id: string + offset: number +} + +export type PresenceCaret = { + anchor: CaretPoint | null + focus: CaretPoint | null +} + +export type PresenceState = PresenceCaret & { + color: string + user: PresenceUser +} + +export type RemotePresence = PresenceState & { key: string } + +export type OutgoingPresence = { + caret: PresenceCaret | null + entry?: string +} + +export type IncomingPresence = { + caret: PresenceCaret | null + connectionId: string + entry: string | null + user: PresenceUser +} + +const LOCAL_AGENT_USER: PresenceUser = { avatar: null, id: 'agent', name: 'AI' } + +const samePoint = (left: CaretPoint | null, right: CaretPoint | null) => + left === right || + (left !== null && + right !== null && + left.id === right.id && + left.offset === right.offset) + +const colorOf = ( + key: string, + entry: string | null, + taken: ReadonlySet = new Set(), +) => { + if (entry?.startsWith('agent:')) return AGENT_PRESENCE_COLOR + let hash = 0 + for (const char of key) hash = (hash * 31 + char.charCodeAt(0)) | 0 + const start = Math.abs(hash) % PRESENCE_COLORS.length + for (let step = 0; step < PRESENCE_COLORS.length; step += 1) { + const color = PRESENCE_COLORS[(start + step) % PRESENCE_COLORS.length] + if (!taken.has(color)) return color + } + return PRESENCE_COLORS[start] +} + +const pointOf = (value: unknown): CaretPoint | null => { + const point = value as CaretPoint | null + return point && typeof point.id === 'string' && Number.isFinite(point.offset) + ? { id: point.id, offset: point.offset } + : null +} + +export function createCollabPresence(options: { + refreshMs?: number + send: (frame: OutgoingPresence) => void + timeoutMs?: number +}) { + const timeoutMs = options.timeoutMs ?? 30_000 + const remote = new Map() + const heardAt = new Map() + const extras = new Map() + const listeners = new Set<(remotes: RemotePresence[]) => void>() + let self: PresenceCaret = { anchor: null, focus: null } + let started = false + let timer: ReturnType | undefined + + const remotes = (): RemotePresence[] => { + const local = [...extras].map(([entry, caret]) => ({ + ...caret, + color: colorOf(entry, entry), + key: `local/${entry}`, + user: LOCAL_AGENT_USER, + })) + const others = [...remote.values()].map( + ({ connectionId: _connectionId, ...peer }) => peer, + ) + return [...others, ...local].sort((left, right) => + left.key.localeCompare(right.key), + ) + } + + const notify = () => { + const current = remotes() + for (const listener of listeners) listener(current) + } + + const announce = () => { + if (!started) return + options.send({ caret: self }) + for (const [entry, caret] of extras) options.send({ caret, entry }) + } + + const expire = () => { + const now = Date.now() + let changed = false + for (const [key, at] of heardAt) { + if (now - at < timeoutMs) continue + heardAt.delete(key) + changed = remote.delete(key) || changed + } + if (changed) notify() + } + + return { + announce, + apply: (frame: IncomingPresence) => { + const key = frame.entry + ? `${frame.connectionId}/${frame.entry}` + : frame.connectionId + if (!frame.caret) { + heardAt.delete(key) + if (remote.delete(key)) notify() + return + } + heardAt.set(key, Date.now()) + const taken = new Set( + [...remote.values()] + .filter((peer) => peer.key !== key) + .map((peer) => peer.color), + ) + remote.set(key, { + anchor: pointOf(frame.caret.anchor), + color: remote.get(key)?.color ?? colorOf(key, frame.entry, taken), + connectionId: frame.connectionId, + focus: pointOf(frame.caret.focus), + key, + user: frame.user, + }) + notify() + }, + dispose: () => { + clearInterval(timer) + started = false + listeners.clear() + remote.clear() + heardAt.clear() + }, + remotes, + removeConnection: (connectionId: string) => { + let changed = false + for (const [key, peer] of remote) { + if (peer.connectionId !== connectionId) continue + remote.delete(key) + heardAt.delete(key) + changed = true + } + if (changed) notify() + }, + setCursor: (anchor: CaretPoint | null, focus: CaretPoint | null) => { + if (samePoint(self.anchor, anchor) && samePoint(self.focus, focus)) { + return + } + self = { anchor, focus } + if (started) options.send({ caret: self }) + }, + setExtra: (entry: string, caret: PresenceCaret | null) => { + if (caret) extras.set(entry, caret) + else extras.delete(entry) + if (started) options.send({ caret, entry }) + notify() + }, + start: () => { + started = true + announce() + clearInterval(timer) + timer = setInterval(() => { + announce() + expire() + }, options.refreshMs ?? 10_000) + }, + subscribe: (listener: (remotes: RemotePresence[]) => void) => { + listeners.add(listener) + return () => { + listeners.delete(listener) + } + }, + } +} + +export type CollabPresence = ReturnType diff --git a/apps/admin/src/features/write/crdt/crdt-draft-session.test.ts b/apps/admin/src/features/write/crdt/crdt-draft-session.test.ts new file mode 100644 index 00000000000..60e5bfa5f57 --- /dev/null +++ b/apps/admin/src/features/write/crdt/crdt-draft-session.test.ts @@ -0,0 +1,611 @@ +import { createLoroBinding } from '@haklex/rich-collab-loro' +import { + $createParagraphNode, + $createTextNode, + $getRoot, + createEditor, + type ElementNode, + type LexicalEditor, +} from 'lexical' +import { LoroDoc, VersionVector } from 'loro-crdt' +import { describe, expect, it, vi } from 'vitest' + +import { ApiRequestError } from '~/api/http' + +import { + type ChannelHooks, + createCrdtDraftSession, + type SessionChannel, +} from './crdt-draft-session' +import type { CrdtTransport } from './crdt-sync' + +const newEditor = () => + createEditor({ + onError: (error) => { + throw error + }, + }) + +const write = (editor: LexicalEditor, ...values: string[]) => + editor.update( + () => { + const root = $getRoot() + root.clear() + for (const value of values) { + root.append($createParagraphNode().append($createTextNode(value))) + } + }, + { discrete: true }, + ) + +const read = (editor: LexicalEditor) => + editor.getEditorState().read(() => + $getRoot() + .getChildren() + .map((node) => node.getTextContent()), + ) + +const seededServer = (title: string, ...values: string[]) => { + const server = new LoroDoc() + server.getText('title').insert(0, title) + const editor = newEditor() + write(editor, ...values) + createLoroBinding(editor, server, { commitMessage: 'import 1' }).dispose() + return server +} + +const transportFor = (server: LoroDoc): CrdtTransport => ({ + pull: async (since) => + since + ? server.export({ from: VersionVector.decode(since), mode: 'update' }) + : server.export({ mode: 'snapshot' }), + push: async (bytes) => { + server.import(bytes) + }, +}) + +const serverTexts = (server: LoroDoc) => { + const editor = newEditor() + createLoroBinding(editor, server.fork()).dispose() + return read(editor) +} + +describe('createCrdtDraftSession', () => { + it('fills a bound editor and the title from the server document', async () => { + const server = seededServer('Hello', 'p1') + const onRemoteTitle = vi.fn() + const session = await createCrdtDraftSession({ + onRemoteTitle, + transport: transportFor(server), + }) + const editor = newEditor() + + session.bindEditor(editor) + + expect(read(editor)).toEqual(['p1']) + expect(onRemoteTitle).toHaveBeenCalledWith('Hello') + session.dispose() + }) + + it('sends editor edits and title changes to the server', async () => { + const server = seededServer('T', 'p1') + const session = await createCrdtDraftSession({ + transport: transportFor(server), + }) + const editor = newEditor() + session.bindEditor(editor) + + write(editor, 'p1', 'p2') + session.setTitle('T2') + await session.flush() + + expect(serverTexts(server)).toEqual(['p1', 'p2']) + expect(server.getText('title').toString()).toBe('T2') + session.dispose() + }) + + it('pulls a remote edit into the editor and reports the remote title', async () => { + const server = seededServer('T', 'p1') + const onRemoteTitle = vi.fn() + const session = await createCrdtDraftSession({ + onRemoteTitle, + transport: transportFor(server), + }) + const editor = newEditor() + session.bindEditor(editor) + const other = await createCrdtDraftSession({ + transport: transportFor(server), + }) + const otherEditor = newEditor() + other.bindEditor(otherEditor) + write(otherEditor, 'p1', 'from other tab') + other.setTitle('Other title') + await other.flush() + + await session.pull() + + expect(read(editor)).toEqual(['p1', 'from other tab']) + expect(onRemoteTitle).toHaveBeenLastCalledWith('Other title') + session.dispose() + other.dispose() + }) + + it('reports a rejected update to the caller', async () => { + const server = seededServer('T', 'p1') + const transport = transportFor(server) + transport.push = async () => { + throw new ApiRequestError('no', 'DRAFT_CRDT_UPDATE_REJECTED', 400) + } + const onRejected = vi.fn() + const session = await createCrdtDraftSession({ onRejected, transport }) + const editor = newEditor() + session.bindEditor(editor) + + write(editor, 'p1', 'bad') + await session.flush().catch(() => undefined) + + expect(onRejected).toHaveBeenCalledTimes(1) + session.dispose() + }) +}) + +describe('createCrdtDraftSession adopt', () => { + it('keeps editor content typed before binding when asked to adopt it', async () => { + const server = seededServer('T', 'p1') + const session = await createCrdtDraftSession({ + transport: transportFor(server), + }) + const editor = newEditor() + write(editor, 'p1', 'typed before bind') + + session.bindEditor(editor, { adopt: true }) + await session.flush() + + expect(read(editor)).toEqual(['p1', 'typed before bind']) + expect(serverTexts(server)).toEqual(['p1', 'typed before bind']) + session.dispose() + }) +}) + +describe('createCrdtDraftSession over a live channel', () => { + const otherTabEdit = async (server: LoroDoc, ...values: string[]) => { + const other = await createCrdtDraftSession({ + transport: transportFor(server), + }) + const editor = newEditor() + other.bindEditor(editor) + const before = server.oplogVersion() + write(editor, ...values) + await other.flush() + other.dispose() + return server.export({ from: before, mode: 'update' }) + } + + it('applies relayed updates, including ones that land during the join', async () => { + const server = seededServer('T', 'p1') + const early = await otherTabEdit(server, 'p1', 'early') + const late = await otherTabEdit(server, 'p1', 'early', 'late') + const base = new LoroDoc() + base.import(server.export({ mode: 'snapshot' })) + let hooks: { + onReconnect: () => void + onRemote: (bytes: Uint8Array) => void + } + const http = transportFor(server) + const session = await createCrdtDraftSession({ + channel: (next) => { + hooks = next + return { + dispose: () => undefined, + live: () => true, + pull: async (since) => { + if (!since) next.onRemote(early) + return http.pull(since) + }, + push: http.push, + sendPresence: () => undefined, + } + }, + }) + const editor = newEditor() + session.bindEditor(editor) + + hooks!.onRemote(late) + + expect(read(editor)).toEqual(['p1', 'early', 'late']) + expect(session.live()).toBe(true) + session.dispose() + }) + + it('pulls what it missed when the channel reconnects', async () => { + const server = seededServer('T', 'p1') + let hooks: { + onReconnect: () => void + onRemote: (bytes: Uint8Array) => void + } + const http = transportFor(server) + const pull = vi.fn(http.pull) + const session = await createCrdtDraftSession({ + channel: (next) => { + hooks = next + return { + dispose: () => undefined, + live: () => true, + pull, + push: http.push, + sendPresence: () => undefined, + } + }, + }) + const editor = newEditor() + session.bindEditor(editor) + await otherTabEdit(server, 'p1', 'missed') + + hooks!.onReconnect() + await vi.waitFor(() => expect(read(editor)).toEqual(['p1', 'missed'])) + + expect(pull).toHaveBeenLastCalledWith(expect.any(Uint8Array)) + session.dispose() + }) +}) + +describe('createCrdtDraftSession presence', () => { + const owner = { avatar: null, id: 'owner', name: 'Owner' } + + const fakeRoom = (server: LoroDoc) => { + const members: { hooks: ChannelHooks; id: string }[] = [] + let next = 0 + const others = (id: string) => members.filter((member) => member.id !== id) + return (hooks: ChannelHooks): SessionChannel => { + const id = `conn-${++next}` + return { + dispose: () => { + members.splice( + members.findIndex((member) => member.id === id), + 1, + ) + for (const member of others(id)) member.hooks.onPeerLeft?.(id) + }, + live: () => true, + pull: async (since) => { + if (!members.some((member) => member.id === id)) { + members.push({ hooks, id }) + hooks.onJoined?.(id) + for (const member of others(id)) member.hooks.onPeerJoined?.(id) + } + return since + ? server.export({ + from: VersionVector.decode(since), + mode: 'update', + }) + : server.export({ mode: 'snapshot' }) + }, + push: async (bytes) => { + server.import(bytes) + for (const member of others(id)) member.hooks.onRemote(bytes) + }, + sendPresence: (frame) => { + for (const member of others(id)) { + member.hooks.onPresence?.({ + caret: frame.caret, + connectionId: id, + entry: frame.entry ?? null, + user: owner, + }) + } + }, + } + } + } + + const caretAt = (editor: LexicalEditor, block: number, offset: number) => + editor.update( + () => { + const text = $getRoot() + .getChildAtIndex(block)! + .getFirstDescendant()! + ;(text as ReturnType).select(offset, offset) + }, + { discrete: true }, + ) + + const textKey = (editor: LexicalEditor, block: number) => + editor + .getEditorState() + .read(() => + $getRoot() + .getChildAtIndex(block)! + .getFirstDescendant()! + .getKey(), + ) + + it('shows where the other member is typing, mapped into this editor', async () => { + const server = seededServer('T', 'p1', 'p2') + const room = fakeRoom(server) + const a = await createCrdtDraftSession({ + channel: room, + presence: true, + }) + const b = await createCrdtDraftSession({ + channel: room, + presence: true, + }) + const editorA = newEditor() + const editorB = newEditor() + a.bindEditor(editorA) + b.bindEditor(editorB) + + caretAt(editorA, 1, 1) + + await vi.waitFor(() => expect(b.peers()[0]?.focus).toBeTruthy()) + expect(b.peers()).toHaveLength(1) + expect(b.resolveCaret(b.peers()[0].focus!)).toEqual({ + key: textKey(editorB, 1), + offset: 1, + }) + a.dispose() + expect(b.peers()).toHaveLength(0) + b.dispose() + }) +}) + +describe('createCrdtDraftSession agent turns', () => { + const stateOf = (...values: string[]) => { + const editor = newEditor() + write(editor, ...values) + return editor.getEditorState().toJSON() + } + + it('merges an agent write at the version it read, under its own peer', async () => { + const server = seededServer('T', 'p1', 'p2') + const session = await createCrdtDraftSession({ + presence: true, + transport: transportFor(server), + }) + const editor = newEditor() + session.bindEditor(editor) + + const turn = session.startAgentTurn('agent:rewrite p1') + editor.update( + () => { + const text = $getRoot() + .getChildAtIndex(1)! + .getFirstDescendant() as ReturnType + text.setTextContent('p2 human') + }, + { discrete: true }, + ) + turn.apply(stateOf('P1 agent', 'p2')) + turn.apply(stateOf('P1 agent', 'p2', 'p3 agent')) + turn.end() + await session.flush() + + expect(read(editor)).toEqual(['P1 agent', 'p2 human', 'p3 agent']) + expect(serverTexts(server)).toEqual(['P1 agent', 'p2 human', 'p3 agent']) + const agentPeers = [...server.getAllChanges().entries()] + .filter(([, changes]) => + changes.some((change) => change.message === 'agent:rewrite p1'), + ) + .map(([peer]) => peer) + expect(agentPeers).toHaveLength(1) + expect( + server + .getAllChanges() + .get(agentPeers[0])! + .every((change) => change.message === 'agent:rewrite p1'), + ).toBe(true) + session.dispose() + }) + + it('shows the agent as a member while its turn runs', async () => { + const server = seededServer('T', 'p1') + const session = await createCrdtDraftSession({ + channel: (hooks) => ({ + dispose: () => undefined, + live: () => true, + pull: async (since) => { + if (!since) hooks.onJoined?.('conn-1') + return transportFor(server).pull(since) + }, + push: transportFor(server).push, + sendPresence: () => undefined, + }), + presence: true, + }) + session.bindEditor(newEditor()) + + const turn = session.startAgentTurn('agent:x') + expect(session.peers().map((peer) => peer.user.name)).toEqual(['AI']) + turn.end() + expect(session.peers()).toEqual([]) + session.dispose() + }) + + it('sends an agent write to the server on its own', async () => { + const server = seededServer('T', 'p1') + const session = await createCrdtDraftSession({ + transport: transportFor(server), + }) + session.bindEditor(newEditor()) + + const turn = session.startAgentTurn('agent:append') + turn.apply(stateOf('p1', 'agent line')) + turn.end() + + await vi.waitFor(() => + expect(serverTexts(server)).toEqual(['p1', 'agent line']), + ) + session.dispose() + }) + + it('diffs a write against the version the agent actually read', async () => { + const server = seededServer('T', 'p1', 'p2') + const session = await createCrdtDraftSession({ + transport: transportFor(server), + }) + const editor = newEditor() + session.bindEditor(editor) + + const turn = session.startAgentTurn('agent:x') + editor.update( + () => { + const text = $getRoot() + .getChildAtIndex(1)! + .getFirstDescendant() as ReturnType + text.setTextContent('p2 typed') + }, + { discrete: true }, + ) + turn.markView() + turn.apply(stateOf('P1', 'p2 typed')) + turn.end() + + expect(read(editor)).toEqual(['P1', 'p2 typed']) + session.dispose() + }) + + it('puts the agent caret at the end of the block it last wrote', async () => { + const server = seededServer('T', 'p1') + const session = await createCrdtDraftSession({ + channel: (hooks) => ({ + dispose: () => undefined, + live: () => true, + pull: async (since) => { + if (!since) hooks.onJoined?.('conn-1') + return transportFor(server).pull(since) + }, + push: transportFor(server).push, + sendPresence: () => undefined, + }), + presence: true, + }) + const editor = newEditor() + session.bindEditor(editor) + const target = stateOf('p1', 'agent line') as unknown as { + root: { children: Array> } + } + target.root.children[1].$ = { blockId: 'blk-new' } + + const turn = session.startAgentTurn('agent:x') + turn.apply(target as never, 'blk-new') + + const agent = session.peers().find((peer) => peer.user.name === 'AI')! + expect(agent.focus).not.toBeNull() + expect(session.resolveCaret(agent.focus!)).toEqual({ + key: editor + .getEditorState() + .read(() => + $getRoot() + .getChildAtIndex(1)! + .getFirstDescendant()! + .getKey(), + ), + offset: 'agent line'.length, + }) + turn.end() + session.dispose() + }) + + it('leaves a title the human changed before the agent read it', async () => { + const server = seededServer('Old', 'p1') + const session = await createCrdtDraftSession({ + transport: transportFor(server), + }) + session.bindEditor(newEditor()) + + const turn = session.startAgentTurn('agent:x') + session.setTitle('New') + turn.markView() + turn.apply(stateOf('p1', 'agent')) + await session.flush() + + expect(server.getText('title').toString()).toBe('New') + turn.end() + session.dispose() + }) + + it('refuses writes once the turn ended or the session closed', async () => { + const server = seededServer('T', 'p1') + const session = await createCrdtDraftSession({ + transport: transportFor(server), + }) + session.bindEditor(newEditor()) + + const ended = session.startAgentTurn('agent:a') + ended.end() + expect(() => ended.apply(stateOf('p1', 'late'))).toThrow() + + const open = session.startAgentTurn('agent:b') + session.dispose() + expect(() => open.apply(stateOf('p1', 'late'))).toThrow() + }) +}) + +describe('createCrdtDraftSession co-authors', () => { + it('signs every commit with the author it was opened as', async () => { + const server = seededServer('T', 'p1') + const session = await createCrdtDraftSession({ + author: 'human:r1', + transport: transportFor(server), + }) + const editor = newEditor() + session.bindEditor(editor) + + write(editor, 'p1 edited') + session.setTitle('T2') + await session.flush() + + const fresh = [...server.getAllChanges().values()] + .flat() + .filter((change) => change.message !== 'import 1') + .map((change) => change.message) + expect(fresh.length).toBeGreaterThan(0) + expect(new Set(fresh)).toEqual(new Set(['human:r1'])) + session.dispose() + }) + + it('reports when the owner removes this member from the room', async () => { + const server = seededServer('T', 'p1') + const onRevoked = vi.fn() + let hooks: ChannelHooks | undefined + const session = await createCrdtDraftSession({ + channel: (next) => { + hooks = next + return { + dispose: () => undefined, + live: () => true, + pull: transportFor(server).pull, + push: transportFor(server).push, + sendPresence: () => undefined, + } + }, + onRevoked, + }) + + hooks?.onRevoked?.() + + expect(onRevoked).toHaveBeenCalledTimes(1) + session.dispose() + }) + + it('joins the room when the socket opens while the first load is still running', async () => { + const server = seededServer('T', 'p1') + const pulls: string[] = [] + const session = await createCrdtDraftSession({ + channel: (hooks) => ({ + dispose: () => undefined, + live: () => pulls.length > 1, + pull: async (since) => { + pulls.push(since ? 'since' : 'first') + if (pulls.length === 1) hooks.onReconnect() + return transportFor(server).pull(since) + }, + push: transportFor(server).push, + sendPresence: () => undefined, + }), + }) + + await vi.waitFor(() => expect(pulls).toEqual(['first', 'since'])) + session.dispose() + }) +}) diff --git a/apps/admin/src/features/write/crdt/crdt-draft-session.ts b/apps/admin/src/features/write/crdt/crdt-draft-session.ts new file mode 100644 index 00000000000..dcfe563f2e7 --- /dev/null +++ b/apps/admin/src/features/write/crdt/crdt-draft-session.ts @@ -0,0 +1,305 @@ +import { + createLoroBinding, + type LoroBinding, + TREE_NAME, +} from '@haklex/rich-collab-loro' +import { editDocAt, type Frontiers } from '@mx-space/editor/loro' +import { + $getSelection, + $isRangeSelection, + BLUR_COMMAND, + COMMAND_PRIORITY_LOW, + type LexicalEditor, + type NodeKey, + type PointType, + type SerializedEditorState, +} from 'lexical' +import { LoroDoc, type LoroTreeNode } from 'loro-crdt' + +import type { ApiRequestError } from '~/api/http' + +import type { CollabChannel } from './collab-channel' +import { + type CaretPoint, + createCollabPresence, + type IncomingPresence, + type RemotePresence, +} from './collab-presence' +import { + type CrdtSync, + type CrdtSyncStatus, + type CrdtTransport, + createCrdtSync, +} from './crdt-sync' + +const HUMAN_MESSAGE = 'human:owner' +const TITLE = 'title' +const CARET_DEBOUNCE_MS = 50 +const TEXT_KEY = '#text' + +export interface AgentTurn { + apply: (target: SerializedEditorState, focusBlockId?: string) => void + end: () => void + markView: () => void +} + +const randomPeer = (): `${number}` => { + const [value] = crypto.getRandomValues(new BigUint64Array(1)) + return `${value}` as `${number}` +} + +const lastTextIn = (node: LoroTreeNode): CaretPoint | null => { + const children = node.children() ?? [] + for (let index = children.length - 1; index >= 0; index -= 1) { + const found = lastTextIn(children[index]) + if (found) return found + } + const text = node.data.get(TEXT_KEY) + return text === undefined + ? null + : { id: node.id, offset: String(text).length } +} + +const caretAtBlock = (doc: LoroDoc, blockId: string): CaretPoint | null => { + for (const node of doc.getTree(TREE_NAME).nodes()) { + const state = node.data.get('$') as { blockId?: string } | undefined + if (state?.blockId === blockId && !node.isDeleted()) { + return lastTextIn(node) ?? { id: node.id, offset: 0 } + } + } + return null +} + +export interface ChannelHooks { + onJoined?: (connectionId: string) => void + onPeerJoined?: (connectionId: string) => void + onPeerLeft?: (connectionId: string) => void + onPresence?: (frame: IncomingPresence) => void + onReconnect: () => void + onRemote: (bytes: Uint8Array) => void + onRevoked?: () => void +} + +export type SessionChannel = Pick< + CollabChannel, + 'dispose' | 'live' | 'pull' | 'push' | 'sendPresence' +> + +export interface CrdtDraftSessionOptions { + author?: string + channel?: (hooks: ChannelHooks) => SessionChannel + onRejected?: (error: ApiRequestError) => void + onRemoteTitle?: (title: string) => void + onRevoked?: () => void + onStatus?: (status: CrdtSyncStatus) => void + presence?: boolean + transport?: CrdtTransport +} + +export interface BindOptions { + adopt?: boolean +} + +export interface CrdtDraftSession { + bindEditor: (editor: LexicalEditor | null, options?: BindOptions) => void + dispose: () => void + flush: () => Promise + live: () => boolean + peers: () => RemotePresence[] + pending: () => boolean + pull: () => Promise + resolveCaret: (point: CaretPoint) => { key: NodeKey; offset: number } | null + setTitle: (title: string) => void + startAgentTurn: (message: string) => AgentTurn + subscribePeers: (listener: (peers: RemotePresence[]) => void) => () => void +} + +export async function createCrdtDraftSession( + options: CrdtDraftSessionOptions, +): Promise { + const doc = new LoroDoc() + doc.setRecordTimestamp(true) + const bindingOptions = { commitMessage: options.author ?? HUMAN_MESSAGE } + let binding: LoroBinding | null = null + let lastTitle: string | null = null + + const reportTitle = () => { + const title = doc.getText(TITLE).toString() + if (title === lastTitle) return + lastTitle = title + options.onRemoteTitle?.(title) + } + + let sync: CrdtSync | null = null + let disposed = false + const early: Uint8Array[] = [] + let reconnected = false + const presence = createCollabPresence({ + send: (frame) => channel?.sendPresence(frame), + }) + let caretTimer: ReturnType | undefined + let stopCaret: (() => void) | null = null + + const pointOf = (point: PointType): CaretPoint | null => { + const id = binding?.idOf(point.key) + return id ? { id, offset: point.offset } : null + } + + const trackCaret = (editor: LexicalEditor) => { + const report = () => { + const caret = editor.getEditorState().read(() => { + const selection = $getSelection() + if (!$isRangeSelection(selection)) return null + return { + anchor: pointOf(selection.anchor), + focus: pointOf(selection.focus), + } + }) + presence.setCursor(caret?.anchor ?? null, caret?.focus ?? null) + } + const offUpdate = editor.registerUpdateListener(() => { + clearTimeout(caretTimer) + caretTimer = setTimeout(report, CARET_DEBOUNCE_MS) + }) + const offBlur = editor.registerCommand( + BLUR_COMMAND, + () => { + clearTimeout(caretTimer) + presence.setCursor(null, null) + return false + }, + COMMAND_PRIORITY_LOW, + ) + return () => { + clearTimeout(caretTimer) + offUpdate() + offBlur() + } + } + + const channel: SessionChannel | undefined = options.channel?.({ + onJoined: () => { + if (options.presence) presence.start() + }, + onPeerJoined: () => presence.announce(), + onPeerLeft: (connectionId) => presence.removeConnection(connectionId), + onPresence: (frame) => presence.apply(frame), + onRevoked: () => options.onRevoked?.(), + onReconnect: () => { + if (sync) void sync.pull().catch(() => undefined) + else reconnected = true + }, + onRemote: (bytes) => { + if (sync) sync.receive(bytes) + else early.push(bytes) + }, + }) + const transport = channel ?? options.transport + if (!transport) throw new Error('A CRDT session needs a transport') + + try { + sync = await createCrdtSync({ + apply: (bytes) => { + if (binding) binding.import(bytes) + else doc.import(bytes) + reportTitle() + }, + doc, + onRejected: options.onRejected, + onStatus: options.onStatus, + transport, + }) + } catch (error) { + channel?.dispose() + presence.dispose() + throw error + } + const ready = sync + for (const bytes of early.splice(0)) ready.receive(bytes) + if (reconnected) void ready.pull().catch(() => undefined) + reportTitle() + + return { + bindEditor: (editor, bind) => { + stopCaret?.() + stopCaret = null + binding?.dispose() + binding = null + if (!editor) return + const kept = bind?.adopt ? editor.getEditorState() : null + binding = createLoroBinding(editor, doc, bindingOptions) + if (kept) editor.setEditorState(kept) + stopCaret = trackCaret(editor) + }, + dispose: () => { + disposed = true + stopCaret?.() + stopCaret = null + binding?.dispose() + binding = null + ready.dispose() + channel?.dispose() + presence.dispose() + }, + flush: () => ready.flush(), + live: () => channel?.live() ?? false, + peers: () => presence.remotes(), + pending: () => ready.pending(), + pull: () => ready.pull(), + resolveCaret: (point) => { + const key = binding?.keyOf(point.id as never) + return key ? { key, offset: point.offset } : null + }, + setTitle: (title) => { + const text = doc.getText(TITLE) + if (text.toString() === title) return + lastTitle = title + text.update(title) + doc.commit({ message: bindingOptions.commitMessage }) + }, + startAgentTurn: (message) => { + const peer = randomPeer() + const key = `agent:${peer}` + let title = doc.getText(TITLE).toString() + let view: Frontiers = doc.frontiers() + let ended = false + presence.setExtra(key, { anchor: null, focus: null }) + return { + apply: (target, focusBlockId) => { + if (ended || disposed) { + throw new Error( + 'The document session for this agent turn has closed', + ) + } + const update = editDocAt( + doc, + view, + { content: JSON.stringify(target), title }, + message, + { peer }, + ) + if (binding) binding.import(update) + else doc.import(update) + // Imported ops are not local updates to the sync loop, so ask it to send them. + ready.touch() + const end = doc.oplogVersion().get(peer) + if (end) view = [{ counter: end - 1, peer }] + presence.setExtra(key, { + anchor: null, + focus: focusBlockId ? caretAtBlock(doc, focusBlockId) : null, + }) + }, + end: () => { + if (ended) return + ended = true + if (!disposed) presence.setExtra(key, null) + }, + markView: () => { + view = doc.frontiers() + title = doc.getText(TITLE).toString() + }, + } + }, + subscribePeers: (listener) => presence.subscribe(listener), + } +} diff --git a/apps/admin/src/features/write/crdt/crdt-sync.test.ts b/apps/admin/src/features/write/crdt/crdt-sync.test.ts new file mode 100644 index 00000000000..c515b2df5f9 --- /dev/null +++ b/apps/admin/src/features/write/crdt/crdt-sync.test.ts @@ -0,0 +1,230 @@ +import { LoroDoc, VersionVector } from 'loro-crdt' +import { describe, expect, it, vi } from 'vitest' + +import { ApiRequestError } from '~/api/http' + +import { type CrdtTransport, createCrdtSync } from './crdt-sync' + +const serverWith = (seed: string) => { + const server = new LoroDoc() + server.getText('title').insert(0, seed) + server.commit() + return server +} + +const fakeTransport = (server: LoroDoc) => { + const pushes: Uint8Array[] = [] + const transport: CrdtTransport = { + pull: vi.fn(async (since?: Uint8Array) => + since + ? server.export({ from: VersionVector.decode(since), mode: 'update' }) + : server.export({ mode: 'snapshot' }), + ), + push: vi.fn(async (bytes: Uint8Array) => { + pushes.push(bytes) + server.import(bytes) + }), + } + return { pushes, transport } +} + +const type = (doc: LoroDoc, value: string) => { + const title = doc.getText('title') + title.insert(title.length, value) + doc.commit({ message: 'human:owner' }) +} + +describe('createCrdtSync', () => { + it('loads the server document and batches local edits into one push', async () => { + const server = serverWith('Hi') + const { pushes, transport } = fakeTransport(server) + const doc = new LoroDoc() + const sync = await createCrdtSync({ batchMs: 5, doc, transport }) + + type(doc, ' a') + type(doc, 'b') + await vi.waitFor(() => expect(pushes).toHaveLength(1)) + + expect(server.getText('title').toString()).toBe('Hi ab') + expect(sync.pending()).toBe(false) + sync.dispose() + }) + + it('keeps unsent edits after a failed push and delivers them on retry', async () => { + const server = serverWith('') + const { transport } = fakeTransport(server) + const push = transport.push + let fail = true + transport.push = vi.fn(async (bytes: Uint8Array) => { + if (fail) { + fail = false + throw new TypeError('network down') + } + await push(bytes) + }) + const statuses: string[] = [] + const doc = new LoroDoc() + const sync = await createCrdtSync({ + batchMs: 5, + doc, + onStatus: (status) => statuses.push(status), + retryMs: 5, + transport, + }) + + type(doc, 'kept') + await vi.waitFor(() => + expect(server.getText('title').toString()).toBe('kept'), + ) + + expect(statuses).toContain('offline') + expect(statuses.at(-1)).toBe('synced') + sync.dispose() + }) + + it('stops pushing and reports when the server rejects an update', async () => { + const server = serverWith('') + const { transport } = fakeTransport(server) + transport.push = vi.fn(async () => { + throw new ApiRequestError('rejected', 'DRAFT_CRDT_UPDATE_REJECTED', 400) + }) + const onRejected = vi.fn() + const doc = new LoroDoc() + const sync = await createCrdtSync({ + batchMs: 5, + doc, + onRejected, + retryMs: 5, + transport, + }) + + type(doc, 'x') + await vi.waitFor(() => expect(onRejected).toHaveBeenCalledTimes(1)) + type(doc, 'y') + await new Promise((resolve) => setTimeout(resolve, 30)) + + expect(transport.push).toHaveBeenCalledTimes(1) + sync.dispose() + }) + + it('pulls only what it lacks and applies it through the importer', async () => { + const server = serverWith('a') + const { transport } = fakeTransport(server) + const doc = new LoroDoc() + const apply = vi.fn((bytes: Uint8Array) => doc.import(bytes)) + const sync = await createCrdtSync({ apply, batchMs: 5, doc, transport }) + const peer = new LoroDoc() + peer.import(server.export({ mode: 'snapshot' })) + type(peer, 'b') + server.import(peer.export({ mode: 'update' })) + + await sync.pull() + + expect(doc.getText('title').toString()).toBe('ab') + expect(apply).toHaveBeenCalledTimes(1) + expect(sync.pending()).toBe(false) + sync.dispose() + }) + + it('flushes pending edits on demand', async () => { + const server = serverWith('') + const { transport } = fakeTransport(server) + const doc = new LoroDoc() + const sync = await createCrdtSync({ batchMs: 10_000, doc, transport }) + + type(doc, 'now') + await sync.flush() + + expect(server.getText('title').toString()).toBe('now') + sync.dispose() + }) + + it('stays pending and flushes everything while a slow push is in flight', async () => { + const server = serverWith('') + const { transport } = fakeTransport(server) + const push = transport.push + transport.push = vi.fn(async (bytes: Uint8Array) => { + await new Promise((resolve) => setTimeout(resolve, 40)) + await push(bytes) + }) + const doc = new LoroDoc() + const sync = await createCrdtSync({ batchMs: 5, doc, transport }) + + type(doc, 'a') + await new Promise((resolve) => setTimeout(resolve, 15)) + type(doc, 'b') + await new Promise((resolve) => setTimeout(resolve, 30)) + + expect(sync.pending()).toBe(true) + await sync.flush() + expect(server.getText('title').toString()).toBe('ab') + expect(sync.pending()).toBe(false) + sync.dispose() + }) + + it('does not skip in-flight ops when a pull lands mid-push', async () => { + const server = serverWith('') + const { transport } = fakeTransport(server) + const push = transport.push + let failNext = true + transport.push = vi.fn(async (bytes: Uint8Array) => { + await new Promise((resolve) => setTimeout(resolve, 20)) + if (failNext) { + failNext = false + throw new TypeError('network down') + } + await push(bytes) + }) + const doc = new LoroDoc() + const sync = await createCrdtSync({ + batchMs: 1, + doc, + retryMs: 5, + transport, + }) + + type(doc, 'lost?') + await new Promise((resolve) => setTimeout(resolve, 8)) + await sync.pull() + await vi.waitFor(() => + expect(server.getText('title').toString()).toBe('lost?'), + ) + sync.dispose() + }) + + it('refuses to flush after the server rejected an update', async () => { + const server = serverWith('') + const { transport } = fakeTransport(server) + transport.push = vi.fn(async () => { + throw new ApiRequestError('no', 'DRAFT_CRDT_UPDATE_REJECTED', 400) + }) + const doc = new LoroDoc() + const sync = await createCrdtSync({ batchMs: 5, doc, transport }) + + type(doc, 'x') + await expect(sync.flush()).rejects.toThrow() + sync.dispose() + }) + + it('applies a pushed remote update and keeps syncing local edits', async () => { + const server = serverWith('a') + const { pushes, transport } = fakeTransport(server) + const doc = new LoroDoc() + const apply = vi.fn((bytes: Uint8Array) => doc.import(bytes)) + const sync = await createCrdtSync({ apply, batchMs: 5, doc, transport }) + const peer = new LoroDoc() + peer.import(server.export({ mode: 'snapshot' })) + const from = peer.oplogVersion() + type(peer, 'b') + const relayed = peer.export({ from, mode: 'update' }) + server.import(relayed) + + sync.receive(relayed) + type(doc, 'c') + await vi.waitFor(() => expect(pushes).toHaveLength(1)) + + expect(apply).toHaveBeenCalledWith(relayed) + expect(server.getText('title').toString()).toBe('abc') + sync.dispose() + }) +}) diff --git a/apps/admin/src/features/write/crdt/crdt-sync.ts b/apps/admin/src/features/write/crdt/crdt-sync.ts new file mode 100644 index 00000000000..209dd953adf --- /dev/null +++ b/apps/admin/src/features/write/crdt/crdt-sync.ts @@ -0,0 +1,129 @@ +import type { LoroDoc, VersionVector } from 'loro-crdt' + +import { ApiRequestError } from '~/api/http' + +export type CrdtSyncStatus = 'offline' | 'synced' | 'syncing' + +export interface CrdtTransport { + pull: (since?: Uint8Array) => Promise + push: (bytes: Uint8Array) => Promise +} + +export interface CrdtSyncOptions { + apply?: (bytes: Uint8Array) => void + batchMs?: number + doc: LoroDoc + onRejected?: (error: ApiRequestError) => void + onStatus?: (status: CrdtSyncStatus) => void + retryMs?: number + transport: CrdtTransport +} + +export interface CrdtSync { + dispose: () => void + flush: () => Promise + pending: () => boolean + pull: () => Promise + receive: (bytes: Uint8Array) => void + touch: () => void +} + +export const CRDT_REJECTED_CODE = 'DRAFT_CRDT_UPDATE_REJECTED' + +export async function createCrdtSync( + options: CrdtSyncOptions, +): Promise { + const { doc, transport } = options + const batchMs = options.batchMs ?? 100 + const retryMs = options.retryMs ?? 2000 + const apply = options.apply ?? ((bytes: Uint8Array) => doc.import(bytes)) + + doc.import(await transport.pull()) + let sent: VersionVector = doc.oplogVersion() + let dirty = false + let rejected = false + let disposed = false + let timer: ReturnType | undefined + let inFlight: Promise | null = null + let failed = false + + const setStatus = (status: CrdtSyncStatus) => options.onStatus?.(status) + + const send = async (): Promise => { + timer = undefined + if (!dirty || rejected || disposed) return + const from = sent + const target = doc.oplogVersion() + dirty = false + setStatus('syncing') + try { + await transport.push(doc.export({ from, mode: 'update' })) + sent = target + failed = false + if (!dirty) setStatus('synced') + } catch (error) { + dirty = true + failed = true + if ( + error instanceof ApiRequestError && + error.code === CRDT_REJECTED_CODE + ) { + rejected = true + options.onRejected?.(error) + return + } + setStatus('offline') + if (!disposed) timer = setTimeout(run, retryMs) + } + } + + const run = (): Promise => { + if (inFlight) return inFlight.then(run) + const current = send().finally(() => { + if (inFlight === current) inFlight = null + }) + inFlight = current + return current + } + + const schedule = () => { + dirty = true + if (timer || rejected) return + timer = setTimeout(run, batchMs) + } + + const receive = (bytes: Uint8Array) => { + apply(bytes) + if (!dirty && !inFlight) sent = doc.oplogVersion() + } + + const unsubscribe = doc.subscribeLocalUpdates(schedule) + + return { + dispose: () => { + disposed = true + clearTimeout(timer) + unsubscribe() + }, + flush: async () => { + clearTimeout(timer) + timer = undefined + for (;;) { + if (rejected) throw new Error('The server rejected unsent edits') + if (inFlight) { + await inFlight + continue + } + if (!dirty) return + await run() + if (dirty && failed) throw new Error('Unsent edits remain') + } + }, + pending: () => dirty || inFlight !== null, + pull: async () => { + receive(await transport.pull(doc.oplogVersion().encode())) + }, + receive, + touch: schedule, + } +} diff --git a/apps/admin/src/features/write/crdt/draft-fields.test.ts b/apps/admin/src/features/write/crdt/draft-fields.test.ts new file mode 100644 index 00000000000..ecf96302711 --- /dev/null +++ b/apps/admin/src/features/write/crdt/draft-fields.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, it } from 'vitest' + +import { diffDraftFields } from './draft-fields' + +describe('diffDraftFields', () => { + it('returns null when nothing changed', () => { + const fields = { meta: { a: 1 }, typeSpecificData: { tags: ['x'] } } + + expect(diffDraftFields(fields, structuredClone(fields))).toBeNull() + }) + + it('sends only changed keys, per section', () => { + expect( + diffDraftFields( + { meta: { a: 1, b: 2 }, typeSpecificData: { slug: 's', tags: ['x'] } }, + { meta: { a: 1, b: 3 }, typeSpecificData: { slug: 's', tags: ['x'] } }, + ), + ).toEqual({ meta: { b: 3 } }) + }) + + it('clears a removed key with null', () => { + expect( + diffDraftFields( + { meta: null, typeSpecificData: { password: 'p', slug: 's' } }, + { meta: null, typeSpecificData: { slug: 's' } }, + ), + ).toEqual({ typeSpecificData: { password: null } }) + }) + + it('treats a missing section as empty', () => { + expect( + diffDraftFields( + { meta: null, typeSpecificData: null }, + { meta: { cover: 'c' }, typeSpecificData: null }, + ), + ).toEqual({ meta: { cover: 'c' } }) + }) +}) diff --git a/apps/admin/src/features/write/crdt/draft-fields.ts b/apps/admin/src/features/write/crdt/draft-fields.ts new file mode 100644 index 00000000000..d22e9cf214c --- /dev/null +++ b/apps/admin/src/features/write/crdt/draft-fields.ts @@ -0,0 +1,38 @@ +import type { DraftFields } from '~/models/draft' + +export type DraftFieldsPatch = Partial< + Record> +> + +const changedKeys = ( + acked: Record | null, + next: Record | null, +) => { + const patch: Record = {} + for (const key of new Set([ + ...Object.keys(acked ?? {}), + ...Object.keys(next ?? {}), + ])) { + const value = next?.[key] ?? null + if (JSON.stringify(acked?.[key] ?? null) !== JSON.stringify(value)) { + patch[key] = value + } + } + return Object.keys(patch).length ? patch : undefined +} + +export function diffDraftFields( + acked: DraftFields, + next: DraftFields, +): DraftFieldsPatch | null { + const meta = changedKeys(acked.meta, next.meta) + const typeSpecificData = changedKeys( + acked.typeSpecificData, + next.typeSpecificData, + ) + if (!meta && !typeSpecificData) return null + return { + ...(meta ? { meta } : {}), + ...(typeSpecificData ? { typeSpecificData } : {}), + } +} diff --git a/apps/admin/src/features/write/crdt/peers-store.ts b/apps/admin/src/features/write/crdt/peers-store.ts new file mode 100644 index 00000000000..0fbb1d7e208 --- /dev/null +++ b/apps/admin/src/features/write/crdt/peers-store.ts @@ -0,0 +1,35 @@ +import { useSyncExternalStore } from 'react' + +import type { RemotePresence } from './collab-presence' + +export interface PeersSource { + get: () => RemotePresence[] + subscribe: (listener: () => void) => () => void +} + +const NONE: RemotePresence[] = [] + +export function createPeersStore(): PeersSource & { + set: (peers: RemotePresence[]) => void +} { + let current = NONE + const listeners = new Set<() => void>() + return { + get: () => current, + set: (peers) => { + const next = peers.length ? peers : NONE + if (next === current) return + current = next + for (const listener of listeners) listener() + }, + subscribe: (listener) => { + listeners.add(listener) + return () => { + listeners.delete(listener) + } + }, + } +} + +export const usePeers = (source: PeersSource) => + useSyncExternalStore(source.subscribe, source.get) diff --git a/apps/admin/src/features/write/crdt/ready-gate.test.ts b/apps/admin/src/features/write/crdt/ready-gate.test.ts new file mode 100644 index 00000000000..0e4221112d3 --- /dev/null +++ b/apps/admin/src/features/write/crdt/ready-gate.test.ts @@ -0,0 +1,53 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' + +import { createReadyGate } from './ready-gate' + +afterEach(() => { + vi.useRealTimers() +}) + +describe('createReadyGate', () => { + it('resolves at once when the document turned live before anyone waited', async () => { + let live = false + const gate = createReadyGate(() => live) + live = true + gate.release() + + await expect(gate.wait(1000)).resolves.toBe(true) + }) + + it('resolves waiters when released and gives up after the timeout', async () => { + vi.useFakeTimers() + let live = false + const gate = createReadyGate(() => live) + const early = gate.wait(1000) + live = true + gate.release() + await expect(early).resolves.toBe(true) + + live = false + const late = gate.wait(1000) + vi.advanceTimersByTime(1000) + await expect(late).resolves.toBe(false) + }) + + it('shares one attempt between callers that overlap', async () => { + const gate = createReadyGate(() => false) + let release: (value: boolean) => void = () => undefined + const attempt = vi.fn( + () => + new Promise((resolve) => { + release = resolve + }), + ) + + const first = gate.once(attempt) + const second = gate.once(attempt) + release(true) + + expect(attempt).toHaveBeenCalledTimes(1) + await expect(Promise.all([first, second])).resolves.toEqual([true, true]) + void gate.once(attempt) + expect(attempt).toHaveBeenCalledTimes(2) + }) +}) diff --git a/apps/admin/src/features/write/crdt/ready-gate.ts b/apps/admin/src/features/write/crdt/ready-gate.ts new file mode 100644 index 00000000000..f4dea1dbe3d --- /dev/null +++ b/apps/admin/src/features/write/crdt/ready-gate.ts @@ -0,0 +1,34 @@ +export interface ReadyGate { + once: (attempt: () => Promise) => Promise + release: () => void + wait: (timeoutMs: number) => Promise +} + +export function createReadyGate(isReady: () => boolean): ReadyGate { + const waiters = new Set<(ready: boolean) => void>() + let inFlight: Promise | null = null + + return { + once: (attempt) => { + inFlight ??= attempt().finally(() => { + inFlight = null + }) + return inFlight + }, + release: () => { + for (const settle of waiters) settle(true) + }, + wait: (timeoutMs) => { + if (isReady()) return Promise.resolve(true) + return new Promise((resolve) => { + const settle = (ready: boolean) => { + clearTimeout(timer) + waiters.delete(settle) + resolve(ready) + } + const timer = setTimeout(() => settle(false), timeoutMs) + waiters.add(settle) + }) + }, + } +} diff --git a/apps/admin/src/features/write/crdt/resync-rejected.test.ts b/apps/admin/src/features/write/crdt/resync-rejected.test.ts new file mode 100644 index 00000000000..4e052359a6f --- /dev/null +++ b/apps/admin/src/features/write/crdt/resync-rejected.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, it, vi } from 'vitest' + +import type { DraftWriteData } from '~/api/drafts' +import type { DraftModel } from '~/models/draft' + +import { resyncRejectedDraft } from './resync-rejected' + +const draftWithHead = (headRevisionId: string) => + ({ headRevisionId, id: 'd1' }) as DraftModel + +describe('resyncRejectedDraft', () => { + it('pins the server working copy first and writes local content on top of it', async () => { + const calls: string[] = [] + const checkpoint = vi.fn(async () => { + calls.push('checkpoint') + return draftWithHead('rev-now') + }) + const update = vi.fn(async (_data: DraftWriteData, head: string) => { + calls.push(`update:${head}`) + return draftWithHead('rev-merged') + }) + const data = { content: '{}', title: 'T' } as DraftWriteData + + const result = await resyncRejectedDraft({ checkpoint, data, update }) + + expect(calls).toEqual(['checkpoint', 'update:rev-now']) + expect(update).toHaveBeenCalledWith(data, 'rev-now') + expect(result.headRevisionId).toBe('rev-merged') + }) +}) diff --git a/apps/admin/src/features/write/crdt/resync-rejected.ts b/apps/admin/src/features/write/crdt/resync-rejected.ts new file mode 100644 index 00000000000..e25aea4e9e9 --- /dev/null +++ b/apps/admin/src/features/write/crdt/resync-rejected.ts @@ -0,0 +1,13 @@ +import type { DraftWriteData } from '~/api/drafts' +import type { DraftModel } from '~/models/draft' + +// The legacy write applies `data` as a diff from its base, so the base must be the server's current +// working copy; an older head would replay every accepted edit since it a second time. +export async function resyncRejectedDraft(input: { + checkpoint: () => Promise + data: DraftWriteData + update: (data: DraftWriteData, headRevisionId: string) => Promise +}) { + const pinned = await input.checkpoint() + return input.update(input.data, pinned.headRevisionId) +} diff --git a/apps/admin/src/features/write/crdt/use-crdt-draft.test.ts b/apps/admin/src/features/write/crdt/use-crdt-draft.test.ts new file mode 100644 index 00000000000..9f327b66f1f --- /dev/null +++ b/apps/admin/src/features/write/crdt/use-crdt-draft.test.ts @@ -0,0 +1,98 @@ +import { LoroDoc } from 'loro-crdt' +import { act, createElement } from 'react' +import type { Root } from 'react-dom/client' +import { createRoot } from 'react-dom/client' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +import { type CrdtDraftController, useCrdtDraft } from './use-crdt-draft' + +const api = vi.hoisted(() => ({ + getDraftCrdt: vi.fn(), + postDraftCrdt: vi.fn(), +})) +vi.mock('~/api/drafts', () => api) +vi.mock('~/socket/SocketBridge', () => ({ + subscribeAdminSocket: (listener: (socket: null) => void) => { + listener(null) + return () => undefined + }, +})) + +let container: HTMLDivElement +let root: Root +let current: CrdtDraftController + +function Probe(props: { draftId: string }) { + current = useCrdtDraft({ + draftId: props.draftId, + enabled: true, + onRejected: async () => undefined, + onRemoteTitle: () => undefined, + }) + return null +} + +const render = (draftId: string) => + act(async () => { + root.render(createElement(Probe, { draftId })) + }) + +beforeEach(() => { + vi.useFakeTimers({ shouldAdvanceTime: true }) + container = document.createElement('div') + root = createRoot(container) + api.getDraftCrdt.mockReset() + api.postDraftCrdt.mockReset() +}) + +afterEach(() => { + act(() => root.unmount()) + vi.useRealTimers() +}) + +describe('useCrdtDraft', () => { + it('stays locked and retries when the first load fails', async () => { + api.getDraftCrdt + .mockRejectedValueOnce(new TypeError('network down')) + .mockResolvedValue(new LoroDoc().export({ mode: 'snapshot' })) + + await render('d1') + await act(async () => { + await Promise.resolve() + }) + + expect(current.active).toBe(false) + expect(current.locked).toBe(true) + + await act(async () => { + await vi.advanceTimersByTimeAsync(3000) + }) + + expect(api.getDraftCrdt).toHaveBeenCalledTimes(2) + expect(current.active).toBe(true) + expect(current.locked).toBe(false) + }) +}) + +describe('useCrdtDraft adopt', () => { + it('leaves the editor writable while a first save is being adopted', async () => { + let resolve: (bytes: Uint8Array) => void = () => undefined + api.getDraftCrdt.mockReturnValue( + new Promise((done) => { + resolve = done + }), + ) + + await render('') + act(() => current.adoptEditorOnBind()) + await render('d2') + + expect(current.active).toBe(false) + expect(current.locked).toBe(false) + + await act(async () => { + resolve(new LoroDoc().export({ mode: 'snapshot' })) + }) + expect(current.active).toBe(true) + }) +}) diff --git a/apps/admin/src/features/write/crdt/use-crdt-draft.ts b/apps/admin/src/features/write/crdt/use-crdt-draft.ts new file mode 100644 index 00000000000..12d41d7748f --- /dev/null +++ b/apps/admin/src/features/write/crdt/use-crdt-draft.ts @@ -0,0 +1,177 @@ +import type { LexicalEditor } from 'lexical' +import { useCallback, useEffect, useMemo, useRef, useState } from 'react' + +import { getDraftCrdt, postDraftCrdt } from '~/api/drafts' + +import { type CollabSocket, createCollabChannel } from './collab-channel' +import type { CaretPoint } from './collab-presence' +import { + type AgentTurn, + type CrdtDraftSession, + createCrdtDraftSession, +} from './crdt-draft-session' +import type { CrdtSyncStatus, CrdtTransport } from './crdt-sync' +import { createPeersStore, type PeersSource } from './peers-store' + +export type CrdtDraftStatus = CrdtSyncStatus | 'idle' | 'loading' + +export interface CrdtDraftController { + active: boolean + adoptEditorOnBind: () => void + bindEditor: (editor: LexicalEditor | null) => void + flush: () => Promise + live: () => boolean + locked: boolean + peers: PeersSource + pending: () => boolean + pull: () => Promise + resolveCaret: (point: CaretPoint) => { key: string; offset: number } | null + setTitle: (title: string) => void + startAgentTurn: (message: string) => AgentTurn | null + status: CrdtDraftStatus +} + +const LOAD_RETRY_MS = 2000 + +export interface CrdtDraftRoom { + author: string + http: CrdtTransport + subscribeSocket: ( + listener: (socket: CollabSocket | null) => void, + ) => () => void +} + +export function useCrdtDraft(options: { + draftId: string + enabled: boolean + onRejected: () => Promise + onRemoteTitle: (title: string) => void + onRevoked?: () => void + room?: CrdtDraftRoom +}): CrdtDraftController { + const sessionRef = useRef(null) + const editorRef = useRef(null) + const adoptRef = useRef(false) + const [adopting, setAdopting] = useState(false) + const [ready, setReady] = useState(false) + const [peers] = useState(createPeersStore) + const [status, setStatus] = useState('idle') + const [generation, setGeneration] = useState(0) + const callbacksRef = useRef(options) + callbacksRef.current = options + const enabled = options.enabled && Boolean(options.draftId) + + useEffect(() => { + setReady(false) + if (!enabled) { + setStatus('idle') + return + } + const draftId = options.draftId + let disposed = false + let session: CrdtDraftSession | null = null + let retry: ReturnType | undefined + let stopPeers: (() => void) | undefined + setStatus('loading') + const { room } = callbacksRef.current + void createCrdtDraftSession({ + author: room?.author, + channel: (hooks) => + createCollabChannel({ + ...hooks, + draftId, + http: room?.http ?? { + pull: (since) => getDraftCrdt(draftId, since), + push: (bytes) => postDraftCrdt(draftId, bytes), + }, + subscribeSocket: room?.subscribeSocket, + }), + onRejected: () => { + void callbacksRef.current + .onRejected() + .finally(() => setGeneration((value) => value + 1)) + }, + onRemoteTitle: (title) => callbacksRef.current.onRemoteTitle(title), + onRevoked: () => callbacksRef.current.onRevoked?.(), + presence: true, + onStatus: (next) => { + if (!disposed) setStatus(next) + }, + }) + .then((created) => { + if (disposed) { + created.dispose() + return + } + session = created + sessionRef.current = created + stopPeers = created.subscribePeers(peers.set) + created.bindEditor(editorRef.current, { adopt: adoptRef.current }) + if (editorRef.current) { + adoptRef.current = false + setAdopting(false) + } + setStatus('synced') + setReady(true) + }) + .catch(() => { + if (disposed) return + setStatus('offline') + retry = setTimeout( + () => setGeneration((value) => value + 1), + LOAD_RETRY_MS, + ) + }) + return () => { + disposed = true + clearTimeout(retry) + stopPeers?.() + peers.set([]) + session?.dispose() + if (sessionRef.current === session) sessionRef.current = null + } + }, [enabled, generation, options.draftId, peers]) + + const bindEditor = useCallback((editor: LexicalEditor | null) => { + editorRef.current = editor + if (!sessionRef.current) return + sessionRef.current.bindEditor(editor, { adopt: adoptRef.current }) + if (editor) { + adoptRef.current = false + setAdopting(false) + } + }, []) + + const actions = useMemo( + () => ({ + adoptEditorOnBind: () => { + adoptRef.current = true + setAdopting(true) + }, + flush: async () => { + await sessionRef.current?.flush() + }, + live: () => sessionRef.current?.live() ?? false, + pending: () => sessionRef.current?.pending() ?? false, + resolveCaret: (point: CaretPoint) => + sessionRef.current?.resolveCaret(point) ?? null, + pull: async () => { + await sessionRef.current?.pull() + }, + setTitle: (title: string) => sessionRef.current?.setTitle(title), + startAgentTurn: (message: string) => + sessionRef.current?.startAgentTurn(message) ?? null, + }), + [], + ) + + const active = enabled && ready + return { + ...actions, + active, + bindEditor, + locked: enabled && !active && !adopting, + peers, + status, + } +} diff --git a/apps/admin/src/i18n/resources/en-US.ts b/apps/admin/src/i18n/resources/en-US.ts index 1dc2045329f..ce5616d3148 100644 --- a/apps/admin/src/i18n/resources/en-US.ts +++ b/apps/admin/src/i18n/resources/en-US.ts @@ -3779,7 +3779,7 @@ export const enUS = { 'write.format.switchUnavailable.title': 'Editor switching is unavailable', 'write.format.switchUnavailable.description': 'CodeMirror edits Markdown source, while Lexical stores rich-text structure. Converting an article that already has content can lose formatting, special nodes, or source syntax, so switching is disabled. Choose the editor you need before writing in a new, empty article.', - 'write.migration.action.blocked': 'Conversion blocked ({{count}})', + 'write.migration.action.blocked': 'Conversion blocked ({count})', 'write.migration.action.checking': 'Checking conversion', 'write.migration.action.committing': 'Committing conversion', 'write.migration.action.convert': 'Convert to Lexical', @@ -4169,6 +4169,65 @@ export const enUS = { 'write.share.shareRevision': 'Share this version', 'write.share.linkedHere': 'The share link points here', 'write.share.revoke': 'Revoke share link', + 'write.timeline.title': 'History', + 'write.collab.peerOtherTab': '{name} (another tab of yours)', + 'write.collab.more': '{count} more', + 'write.collab.editing': 'People editing', + 'write.timeline.step.coauthor': "{name}'s edits ({count})", + 'write.collaborators.title': 'Co-authors', + 'write.collaborators.unknownAuthor': 'A co-author', + 'write.collaborators.historyNotice': + "Co-authors receive the draft's full edit history, including text deleted before they joined.", + 'write.collaborators.invite': 'Invite link', + 'write.collaborators.linkCopied': 'Invite link copied', + 'write.collaborators.linkExpires': 'Link · expires {time} · used {uses}', + 'write.collaborators.revokeLink': 'Revoke invite link', + 'write.collaborators.byline': 'Byline', + 'write.collaborators.remove': 'Remove {name}', + 'write.collaborators.removed': 'Co-author removed', + 'write.collaborators.failed': 'Could not create the invite link', + 'collab.join.title': 'Co-write this article', + 'collab.join.signInHint': 'Sign in to accept the invitation.', + 'collab.join.signInWith': 'Continue with {provider}', + 'collab.join.noProviders': 'No sign-in method is set up on this site.', + 'collab.join.accepting': 'Opening the document…', + 'collab.join.invalid': 'This invite link has expired or was revoked.', + 'collab.join.failed': 'Could not accept the invitation.', + 'collab.document.editingAs': 'Editing as {name}', + 'collab.document.offline': 'Offline, changes will sync later', + 'collab.document.titleLabel': 'Title', + 'collab.document.rejected': + 'Your last change was refused; the document was reloaded.', + 'collab.document.revokedTitle': 'You no longer have access', + 'collab.document.revokedDescription': + 'The author removed you from this document. Your earlier edits are kept.', + 'collab.document.unavailableTitle': 'Document unavailable', + 'collab.document.unavailableDescription': + 'Open your invite link again to sign in.', + 'write.timeline.emptyTitle': 'No history yet', + 'write.timeline.emptyDescription': + 'Every step appears here once you save or edit.', + 'write.timeline.restore': 'Restore this version', + 'write.timeline.restoreTitle': 'Restore this version?', + 'write.timeline.restoreDescription': + 'Restoring is written as a new change; later history is kept.', + 'write.timeline.restored': 'Restored', + 'write.timeline.undo': 'Undo this step', + 'write.timeline.undone': 'Step undone', + 'write.timeline.failed': 'Action failed', + 'write.timeline.published': 'Live', + 'write.timeline.checkpoint.save': 'Saved', + 'write.timeline.checkpoint.publish': 'Published', + 'write.timeline.checkpoint.restore': 'Restored', + 'write.timeline.checkpoint.undo': 'Undone', + 'write.timeline.checkpoint.import': 'Collaborative editing started', + 'write.timeline.checkpoint.legacySave': 'Saved elsewhere', + 'write.timeline.checkpoint.legacy': 'Earlier version', + 'write.timeline.step.human': 'Your edits ({count})', + 'write.timeline.step.agent': 'AI: {summary}', + 'write.timeline.step.restore': 'Restore', + 'write.timeline.step.undo': 'Undo', + 'write.timeline.step.other': 'System', 'write.share.revoked': 'Share link revoked', 'write.branch.compare': 'Compare', 'write.branch.history': 'View branch history', @@ -4352,6 +4411,10 @@ export const enUS = { 'write.toast.publishedDraftSaved': 'Changes saved as a draft and not yet applied to the online article', 'write.toast.unpublishedSaved': 'Unpublished content saved', + 'write.toast.draftResynced': + 'Sync was rejected; your content was saved again', + 'write.toast.draftSyncPending': + 'The shared document is still loading; try saving again shortly', 'write.toast.draftSaveFailed': 'Failed to save draft', 'write.toast.draftTypeMismatch': 'Draft type does not match current writing page', diff --git a/apps/admin/src/i18n/resources/zh-CN.ts b/apps/admin/src/i18n/resources/zh-CN.ts index 1088fce2755..23a8efebe50 100644 --- a/apps/admin/src/i18n/resources/zh-CN.ts +++ b/apps/admin/src/i18n/resources/zh-CN.ts @@ -3586,7 +3586,7 @@ export const zhCN = { 'write.format.switchUnavailable.title': '暂时无法切换编辑器', 'write.format.switchUnavailable.description': 'CodeMirror 直接编辑 Markdown 源码，Lexical 则保存富文本结构。正文已有内容时，在两者之间转换可能丢失格式、特殊节点或原始写法，因此当前文章不支持切换。请在新建文章且正文为空时选择所需编辑器，再开始写作。', - 'write.migration.action.blocked': '转换受阻（{{count}}）', + 'write.migration.action.blocked': '转换受阻（{count}）', 'write.migration.action.checking': '正在检查转换', 'write.migration.action.committing': '正在提交转换', 'write.migration.action.convert': '转换为 Lexical', @@ -3953,6 +3953,62 @@ export const zhCN = { 'write.share.shareRevision': '分享这一版', 'write.share.linkedHere': '分享链接指向这里', 'write.share.revoke': '撤销分享', + 'write.timeline.title': '历史', + 'write.collab.peerOtherTab': '{name}（另一个标签页）', + 'write.collab.more': '另有 {count} 人', + 'write.collab.editing': '正在编辑的成员', + 'write.timeline.step.coauthor': '{name} 的编辑（{count} 处）', + 'write.collaborators.title': '共同作者', + 'write.collaborators.unknownAuthor': '共同作者', + 'write.collaborators.historyNotice': + '共同作者会拿到草稿的完整编辑历史，包括加入前已删除的文字。', + 'write.collaborators.invite': '邀请链接', + 'write.collaborators.linkCopied': '邀请链接已复制', + 'write.collaborators.linkExpires': '链接 · {time}过期 · 已用 {uses} 次', + 'write.collaborators.revokeLink': '撤销邀请链接', + 'write.collaborators.byline': '署名', + 'write.collaborators.remove': '移除 {name}', + 'write.collaborators.removed': '已移除共同作者', + 'write.collaborators.failed': '无法创建邀请链接', + 'collab.join.title': '一起写这篇文章', + 'collab.join.signInHint': '登录后即可接受邀请。', + 'collab.join.signInWith': '使用 {provider} 登录', + 'collab.join.noProviders': '本站未配置登录方式。', + 'collab.join.accepting': '正在打开文档…', + 'collab.join.invalid': '邀请链接已过期或已被撤销。', + 'collab.join.failed': '无法接受邀请。', + 'collab.document.editingAs': '以 {name} 身份编辑', + 'collab.document.offline': '离线中，恢复后自动同步', + 'collab.document.titleLabel': '标题', + 'collab.document.rejected': '上一次修改被拒绝，文档已重新载入。', + 'collab.document.revokedTitle': '你已无权编辑', + 'collab.document.revokedDescription': + '作者已将你移出此文档，你此前的修改仍会保留。', + 'collab.document.unavailableTitle': '文档不可用', + 'collab.document.unavailableDescription': '请重新打开邀请链接登录。', + 'write.timeline.emptyTitle': '暂无历史', + 'write.timeline.emptyDescription': '保存或编辑后，这里会出现每一步记录。', + 'write.timeline.restore': '恢复到此版本', + 'write.timeline.restoreTitle': '恢复到此版本？', + 'write.timeline.restoreDescription': + '恢复会作为一次新的修改写入，之后的历史仍然保留。', + 'write.timeline.restored': '已恢复', + 'write.timeline.undo': '撤销此步', + 'write.timeline.undone': '已撤销此步', + 'write.timeline.failed': '操作失败', + 'write.timeline.published': '线上', + 'write.timeline.checkpoint.save': '保存', + 'write.timeline.checkpoint.publish': '发布', + 'write.timeline.checkpoint.restore': '恢复', + 'write.timeline.checkpoint.undo': '撤销', + 'write.timeline.checkpoint.import': '开始协作编辑', + 'write.timeline.checkpoint.legacySave': '外部保存', + 'write.timeline.checkpoint.legacy': '早期版本', + 'write.timeline.step.human': '你的编辑（{count} 处）', + 'write.timeline.step.agent': 'AI：{summary}', + 'write.timeline.step.restore': '恢复', + 'write.timeline.step.undo': '撤销', + 'write.timeline.step.other': '系统', 'write.share.revoked': '分享链接已撤销', 'write.branch.compare': '比较', 'write.branch.history': '查看分支历史', @@ -4125,6 +4181,8 @@ export const zhCN = { 'write.toast.draftSaved': '草稿已保存', 'write.toast.publishedDraftSaved': '修改已保存为草稿，尚未更新线上文章', 'write.toast.unpublishedSaved': '未发布内容已保存', + 'write.toast.draftResynced': '同步被拒，已用本地内容重新保存', + 'write.toast.draftSyncPending': '协同文档仍在加载，稍后再存', 'write.toast.draftSaveFailed': '保存草稿失败', 'write.toast.draftTypeMismatch': '草稿类型与当前写作页面不匹配', 'write.toast.saved': '已保存', diff --git a/apps/admin/src/models/draft.ts b/apps/admin/src/models/draft.ts index 8a5f9c18f6c..0a5ebdf4f8f 100644 --- a/apps/admin/src/models/draft.ts +++ b/apps/admin/src/models/draft.ts @@ -20,13 +20,52 @@ export interface RevisionSnapshot { typeSpecificData: TypeSpecificData | null } +export interface CrdtOpId { + counter: number + peer: `${number}` +} + +export type CheckpointKind = + 'import' | 'legacy-save' | 'publish' | 'restore' | 'save' | 'undo' + export interface ContentRevision extends RevisionSnapshot { createdAt: string documentId: string + frontiers?: CrdtOpId[] | null id: string + kind?: CheckpointKind | null parentRevisionId: string | null } +export type DraftCheckpoint = Pick< + ContentRevision, + | 'contentFormat' + | 'createdAt' + | 'frontiers' + | 'id' + | 'kind' + | 'parentRevisionId' + | 'title' +> + +export interface DraftChange { + deps: CrdtOpId[] + id: CrdtOpId + lamport: number + message?: string + timestamp: number +} + +export interface DraftHistory { + changes: DraftChange[] + checkpoints: DraftCheckpoint[] +} + +export interface DraftFields { + meta: Record | null + typeSpecificData: Record | null +} + export interface ContentDocument { createdAt: string id: string diff --git a/apps/admin/src/query/keys.ts b/apps/admin/src/query/keys.ts index 8123bbf485e..d062a82fb00 100644 --- a/apps/admin/src/query/keys.ts +++ b/apps/admin/src/query/keys.ts @@ -243,6 +243,11 @@ export const adminQueryKeys = { ['pages', 'list', params] as const, root: ['pages'] as const, }, + collab: { + collaborators: (documentId: string) => + ['collab', 'collaborators', documentId] as const, + invites: (documentId: string) => ['collab', 'invites', documentId] as const, + }, drafts: { byRef: (params: { id: string; refType: string }) => ['drafts', 'by-ref', params.refType, params.id] as const, @@ -259,6 +264,7 @@ export const adminQueryKeys = { ['drafts', 'recovery-version', params.id, params.version] as const, root: ['drafts'] as const, share: (documentId: string) => ['drafts', 'share', documentId] as const, + timeline: (id: string) => ['drafts', 'timeline', id] as const, }, says: { list: (params: { page: number; size: number }) => diff --git a/apps/admin/src/socket/SocketBridge.tsx b/apps/admin/src/socket/SocketBridge.tsx index 584a2626e4c..5b60292ad30 100644 --- a/apps/admin/src/socket/SocketBridge.tsx +++ b/apps/admin/src/socket/SocketBridge.tsx @@ -50,7 +50,7 @@ function setAdminSocket(socket: WsClient | null) { for (const listener of socketChangeListeners) listener(socket) } -function toWsOrigin(url: string): string { +export function toWsOrigin(url: string): string { if (url.startsWith('https://')) return `wss://${url.slice('https://'.length)}` if (url.startsWith('http://')) return `ws://${url.slice('http://'.length)}` return url diff --git a/apps/admin/src/socket/web-socket.ts b/apps/admin/src/socket/web-socket.ts new file mode 100644 index 00000000000..a6caa777097 --- /dev/null +++ b/apps/admin/src/socket/web-socket.ts @@ -0,0 +1,24 @@ +import type { WsClient } from '@mx-space/ws-client' +import { createWsClient } from '@mx-space/ws-client' + +import { GATEWAY_URL } from '../constants/env' +import { toWsOrigin } from './SocketBridge' + +let client: WsClient | null = null +const listeners = new Set<(socket: WsClient | null) => void>() + +export function subscribeWebSocket( + listener: (socket: WsClient | null) => void, +): () => void { + if (!client && GATEWAY_URL) { + client = createWsClient({ url: `${toWsOrigin(GATEWAY_URL)}/ws/web` }) + } + listeners.add(listener) + listener(client) + return () => { + listeners.delete(listener) + if (listeners.size || !client) return + client.close() + client = null + } +} diff --git a/apps/admin/src/vendor/rich-editor/utils/document-bash/create-document-bash-tool.test.ts b/apps/admin/src/vendor/rich-editor/utils/document-bash/create-document-bash-tool.test.ts index b0920d24fd5..739701b9f75 100644 --- a/apps/admin/src/vendor/rich-editor/utils/document-bash/create-document-bash-tool.test.ts +++ b/apps/admin/src/vendor/rich-editor/utils/document-bash/create-document-bash-tool.test.ts @@ -123,6 +123,87 @@ describe('createDocumentBashWorkspace', () => { expect(grepped.operations).toBeUndefined() }) + it('applies an edit straight to the document when it owns the writes', async () => { + const applied: { focus?: string; target: SerializedEditorState }[] = [] + const workspace = createDocumentBashWorkspace({ + applyDocument: (target, focus) => { + applied.push({ focus, target }) + }, + getEditorState: () => state([para('a', 'Hello world')]), + }) + workspace.beginRun() + + const edited = await workspace.tool.execute({ + command: "sed -i 's/Hello world/Hello earth/' /doc.xml", + }) + const appended = await workspace.tool.execute({ + command: "sed -i 's||

    Second

    \\n|' /doc.xml", + }) + + expect(edited.ok && edited.operations).toBeFalsy() + expect(appended.ok && appended.operations).toBeFalsy() + expect(applied).toHaveLength(2) + expect(JSON.stringify(applied[0].target)).toContain('Hello earth') + expect(applied[0].focus).toBe('a') + const blocks = applied[1].target.root.children as SerializedLexicalNode[] + const minted = (blocks[1] as { $?: { blockId?: string } }).$?.blockId + expect(minted).toBeTruthy() + expect(applied[1].focus).toBe(minted) + }) + + it('keeps untouched blocks exactly as the document holds them when applying', async () => { + const styled = { + ...para('a', 'Centred red'), + format: 'center', + indent: 1, + children: [ + { + type: 'text', + text: 'Centred red', + version: 1, + detail: 0, + format: 1, + mode: 'normal', + style: 'color: red;', + }, + ], + } as SerializedLexicalNode + let target: SerializedEditorState | null = null + const workspace = createDocumentBashWorkspace({ + applyDocument: (next) => { + target = next + }, + getEditorState: () => state([styled, para('b', 'Plain')]), + }) + workspace.beginRun() + + await workspace.tool.execute({ + command: "sed -i 's/Plain/Plainer/' /doc.xml", + }) + + const blocks = target!.root.children as SerializedLexicalNode[] + expect(blocks[0]).toEqual(styled) + expect(JSON.stringify(blocks[1])).toContain('Plainer') + }) + + it('reports its own copy of the document while it owns the writes', async () => { + let live = state([para('a', 'Hello world')]) + const workspace = createDocumentBashWorkspace({ + applyDocument: () => undefined, + getEditorState: () => live, + }) + workspace.beginRun() + await workspace.tool.execute({ command: 'cat /doc.xml' }) + + live = state([para('a', 'Typed by someone else')]) + + const doc = workspace + .snapshotFiles() + .find((file) => file.path === '/doc.xml') + expect(doc?.content).toContain('Hello world') + expect(doc?.content).not.toContain('someone else') + }) + it('reports the files a command read, with their current content', async () => { const workspace = createWorkspace() workspace.beginRun() diff --git a/apps/admin/src/vendor/rich-editor/utils/document-bash/create-document-bash-tool.ts b/apps/admin/src/vendor/rich-editor/utils/document-bash/create-document-bash-tool.ts index 3118e5d1b37..a82c3c0c282 100644 --- a/apps/admin/src/vendor/rich-editor/utils/document-bash/create-document-bash-tool.ts +++ b/apps/admin/src/vendor/rich-editor/utils/document-bash/create-document-bash-tool.ts @@ -1,4 +1,8 @@ -import type { AgentToolConfig, AgentToolResult } from '@haklex/rich-agent-core' +import type { + AgentOperation, + AgentToolConfig, + AgentToolResult, +} from '@haklex/rich-agent-core' import type { AgentFileSnapshot } from '@mx-space/ai' import { deserializeMxLitexmlToLexical, @@ -81,6 +85,9 @@ export type DocumentBashWorkspace = { } export type CreateDocumentBashWorkspaceOptions = { + applyDocument?: (target: SerializedEditorState, focusBlockId?: string) => void + ownsWrites?: () => boolean + onSeed?: () => void getEditorState: () => SerializedEditorState | null getBaseState?: () => SerializedEditorState | null getSelectionXml?: () => string @@ -143,6 +150,39 @@ class TrackingFs extends InMemoryFs { } } +function keepUntouchedBlocks( + view: SerializedEditorState, + next: SerializedEditorState, + operations: AgentOperation[], +): SerializedEditorState { + const changed = new Set( + operations.flatMap((operation) => + operation.op === 'replace' ? [operation.blockId] : [], + ), + ) + const original = new Map() + for (const block of view.root.children) { + const id = (block as NodeRecord).$?.blockId + if (id) original.set(id, block) + } + return { + ...next, + root: { + ...next.root, + children: next.root.children.map((block) => { + const id = (block as NodeRecord).$?.blockId + return id && !changed.has(id) ? (original.get(id) ?? block) : block + }), + }, + } +} + +function focusOf(operation: AgentOperation | undefined): string | undefined { + if (!operation) return undefined + if (operation.op !== 'insert') return operation.blockId + return (operation.node as NodeRecord).$?.blockId +} + function extractText(node: SerializedLexicalNode): string { const record = node as NodeRecord if (typeof record.text === 'string') return record.text @@ -203,6 +243,7 @@ export function createDocumentBashWorkspace( bash: Bash emittedOperations: boolean fs: TrackingFs + ownsWrites: boolean lastEmittedXml: string } | null = null let lastTouched = new Set() @@ -220,9 +261,14 @@ export function createDocumentBashWorkspace( }) const snapshotFiles = (): AgentFileSnapshot[] => { - const editorState = options.getEditorState() + // While the run owns its writes, other people's edits stay out of the agent's view until + // its next turn, so the files it is told about must be its own copy. + const owned = session?.ownsWrites ? session : null + const editorState = owned ? owned.view : options.getEditorState() if (!editorState) return [] - const files = seedFiles(editorState) + const files = owned + ? { ...seedFiles(editorState), [DOC_PATH]: owned.lastEmittedXml } + : seedFiles(editorState) return TRACKED_PATHS.map((path) => ATTACHED_PATHS.has(path) ? { attach: true, content: files[path], path } @@ -242,6 +288,7 @@ export function createDocumentBashWorkspace( const base = options.getBaseState?.() ?? editorState if (!editorState || !base) return null const files = seedFiles(editorState) + options.onSeed?.() const fs = new TrackingFs(files) const bash = new Bash({ commands: ALLOWED_COMMANDS, @@ -253,6 +300,7 @@ export function createDocumentBashWorkspace( session = { base, bash, + ownsWrites: options.ownsWrites?.() ?? Boolean(options.applyDocument), view: editorState, emittedOperations: false, fs, @@ -304,6 +352,27 @@ export function createDocumentBashWorkspace( } await current.bash.writeFile(OUTLINE_PATH, buildOutline(diff.nextState)) + if (current.ownsWrites && options.applyDocument) { + const applied = diffDocumentStates(current.view, diff.nextState) + if (!applied.ok) throw new Error(applied.message) + if (applied.operations.length === 0) { + return { + note: `${DOC_PATH} changed textually but parses to the same document; no edit was recorded.`, + } + } + // LiteXML drops inline style, alignment and indent, so only blocks the agent changed may + // come from the parsed file; every other block keeps the document's own JSON. + const target = keepUntouchedBlocks( + current.view, + diff.nextState, + applied.operations, + ) + options.applyDocument(target, focusOf(applied.operations.at(-1))) + current.base = target + current.view = target + return {} + } + if (unchangedFromView && !current.emittedOperations) { return { note: `${DOC_PATH} changed textually but parses to the same document; no edit was recorded.`, diff --git a/apps/admin/src/vendor/rich-editor/utils/document-bash/prompt.test.ts b/apps/admin/src/vendor/rich-editor/utils/document-bash/prompt.test.ts new file mode 100644 index 00000000000..89248e6cf60 --- /dev/null +++ b/apps/admin/src/vendor/rich-editor/utils/document-bash/prompt.test.ts @@ -0,0 +1,22 @@ +import { describe, expect, it } from 'vitest' + +import { buildDocumentBashSystemMessages } from './prompt' + +const text = (shared: boolean) => + buildDocumentBashSystemMessages({ shared }) + .map((message) => ('content' in message ? message.content : '')) + .join('\n') + +describe('buildDocumentBashSystemMessages', () => { + it('describes reviewable proposals for a private draft', () => { + expect(text(false)).toContain('/review.xml') + }) + + it('describes direct, shared edits for a collaborative document', () => { + const prompt = text(true) + expect(prompt).not.toContain('/review.xml') + expect(prompt).not.toContain('propos') + expect(prompt).toContain('immediately') + expect(prompt).toMatch(/other people/i) + }) +}) diff --git a/apps/admin/src/vendor/rich-editor/utils/document-bash/prompt.ts b/apps/admin/src/vendor/rich-editor/utils/document-bash/prompt.ts index 0d903ae18d6..da551b895c6 100644 --- a/apps/admin/src/vendor/rich-editor/utils/document-bash/prompt.ts +++ b/apps/admin/src/vendor/rich-editor/utils/document-bash/prompt.ts @@ -1,13 +1,25 @@ import type { ChatMessage } from '@haklex/rich-agent-core' -export function buildDocumentBashSystemMessages(): ChatMessage[] { +const REVIEW_WORKSPACE = [ + '- /doc.xml is the article as you left it: the accepted text plus your proposals the user has not reviewed yet (pretty LiteXML). Reads and writes apply immediately.', + '- What you write is proposed to the user as one set of changes against the accepted text; it replaces your earlier unreviewed proposals. Removing a proposal from /doc.xml withdraws it.', + '- /review.xml lists which of your proposals are still unreviewed (read-only, empty when none). When the user accepts one it simply leaves /review.xml; when they reject one it also disappears from /doc.xml.', +] + +const SHARED_WORKSPACE = [ + '- /doc.xml is the article as you last read or wrote it (pretty LiteXML). Every write lands in the shared document immediately; there is no review step.', + '- Other people may be editing the same document at the same time. Your writes are merged with theirs at the version you read, so only change what you mean to change and keep other blocks as they are.', + '- The user can undo your whole turn from the history panel.', +] + +export function buildDocumentBashSystemMessages( + options: { shared?: boolean } = {}, +): ChatMessage[] { const content = [ 'You edit a rich-text document through a virtual bash workspace. This is not a host machine.', '', 'Workspace:', - '- /doc.xml is the article as you left it: the accepted text plus your proposals the user has not reviewed yet (pretty LiteXML). Reads and writes apply immediately.', - '- What you write is proposed to the user as one set of changes against the accepted text; it replaces your earlier unreviewed proposals. Removing a proposal from /doc.xml withdraws it.', - '- /review.xml lists which of your proposals are still unreviewed (read-only, empty when none). When the user accepts one it simply leaves /review.xml; when they reject one it also disappears from /doc.xml.', + ...(options.shared ? SHARED_WORKSPACE : REVIEW_WORKSPACE), '- /.meta/outline lists block ids and a short preview. cat it when you need to locate a target.', "- /selection.xml holds the user's current selection in the editor (empty when nothing is selected). It arrives with the user's message when they select something.", '- The document is not pasted into the conversation. Read what you need with cat, grep or sed -n. If a file you already read changes outside your edits, a with its diff arrives at the start of the next turn.', diff --git a/apps/admin/src/views/(public)/collab/[documentId]/page.sync.tsx b/apps/admin/src/views/(public)/collab/[documentId]/page.sync.tsx new file mode 100644 index 00000000000..e59d8bb311d --- /dev/null +++ b/apps/admin/src/views/(public)/collab/[documentId]/page.sync.tsx @@ -0,0 +1,5 @@ +import { defineMetadata } from '~/lib/route-meta' + +export const metadata = defineMetadata({}) + +export { CollabDocumentPage as default } from '~/features/collab/components/CollabDocumentPage' diff --git a/apps/admin/src/views/(public)/collab/join/[token]/page.sync.tsx b/apps/admin/src/views/(public)/collab/join/[token]/page.sync.tsx new file mode 100644 index 00000000000..28d228cff50 --- /dev/null +++ b/apps/admin/src/views/(public)/collab/join/[token]/page.sync.tsx @@ -0,0 +1,5 @@ +import { defineMetadata } from '~/lib/route-meta' + +export const metadata = defineMetadata({}) + +export { CollabJoinPage as default } from '~/features/collab/components/CollabJoinPage' diff --git a/apps/admin/vite.config.mts b/apps/admin/vite.config.mts index 1219652f46f..81baf9456e4 100644 --- a/apps/admin/vite.config.mts +++ b/apps/admin/vite.config.mts @@ -48,6 +48,9 @@ export default ({ mode }) => { resolve: { tsconfigPaths: true, alias: { + // The bundler build imports its wasm as an ES module, which the dev server cannot serve. + // Vitest keeps the Node build so externalized deps share one wasm instance with app code. + ...(process.env.VITEST ? {} : { 'loro-crdt': 'loro-crdt/base64' }), path: 'path-browserify', os: 'os-browserify', 'node-fetch': 'isomorphic-fetch', diff --git a/apps/core/package.json b/apps/core/package.json index 1dc4a5e5abe..737355c7dd4 100644 --- a/apps/core/package.json +++ b/apps/core/package.json @@ -69,6 +69,7 @@ "@earendil-works/pi-ai": "0.99.1", "@fastify/multipart": "10.1.2", "@fastify/static": "10.1.5", + "@haklex/rich-collab-loro": "0.55.0", "@haklex/rich-headless": "0.55.0", "@innei/message-engine": "0.5.0", "@innei/next-async": "0.4.0", @@ -125,6 +126,7 @@ "keyv": "5.6.0", "lexical": "^0.52.0", "linkedom": "0.18.13", + "loro-crdt": "^1.16.4", "lru-cache": "11.5.3", "marked": "18.0.14", "mime-types": "^3.0.2", diff --git a/apps/core/src/common/adapters/fastify.adapter.ts b/apps/core/src/common/adapters/fastify.adapter.ts index 3572bfe36a2..030261bdc22 100644 --- a/apps/core/src/common/adapters/fastify.adapter.ts +++ b/apps/core/src/common/adapters/fastify.adapter.ts @@ -71,6 +71,14 @@ app.register(FastifyMultipart, { }, }) +app + .getInstance() + .addContentTypeParser( + 'application/octet-stream', + { parseAs: 'buffer' }, + (_request, body, done) => done(null, body), + ) + const DEFAULT_RAW_BODY_LIMIT = 1024 * 1024 export function isRawBodyRoute(request: FastifyRequest): boolean { diff --git a/apps/core/src/common/errors/app-error-code.ts b/apps/core/src/common/errors/app-error-code.ts index 74feae8b98e..140bdb66a4c 100644 --- a/apps/core/src/common/errors/app-error-code.ts +++ b/apps/core/src/common/errors/app-error-code.ts @@ -114,6 +114,10 @@ export enum AppErrorCode { DRAFT_HEAD_CONFLICT = 'DRAFT_HEAD_CONFLICT', DRAFT_SHARE_NOT_FOUND = 'DRAFT_SHARE_NOT_FOUND', PUBLISHED_REVISION_CHANGED = 'PUBLISHED_REVISION_CHANGED', + DRAFT_CRDT_UPDATE_REJECTED = 'DRAFT_CRDT_UPDATE_REJECTED', + DRAFT_CRDT_FORMAT_UNSUPPORTED = 'DRAFT_CRDT_FORMAT_UNSUPPORTED', + COLLAB_INVITE_INVALID = 'COLLAB_INVITE_INVALID', + COLLAB_FORBIDDEN = 'COLLAB_FORBIDDEN', // document / helper DOCUMENT_NOT_FOUND = 'DOCUMENT_NOT_FOUND', diff --git a/apps/core/src/common/errors/app-error-definitions.ts b/apps/core/src/common/errors/app-error-definitions.ts index d1f439877e5..392082f0890 100644 --- a/apps/core/src/common/errors/app-error-definitions.ts +++ b/apps/core/src/common/errors/app-error-definitions.ts @@ -445,6 +445,25 @@ export const APP_ERROR_DEFINITIONS = { expectedPublishedRevisionId: p.expectedPublishedRevisionId, }), }, + [AppErrorCode.DRAFT_CRDT_UPDATE_REJECTED]: { + status: 400, + message: 'The collaborative update was rejected', + details: (p) => ({ branchId: p.branchId, reason: p.reason }), + }, + [AppErrorCode.DRAFT_CRDT_FORMAT_UNSUPPORTED]: { + status: 400, + message: 'This draft only accepts Lexical content', + details: (p) => ({ branchId: p.branchId, contentFormat: p.contentFormat }), + }, + [AppErrorCode.COLLAB_INVITE_INVALID]: { + status: 410, + message: 'This invite link can no longer be used', + details: (p) => ({ reason: p.reason }), + }, + [AppErrorCode.COLLAB_FORBIDDEN]: { + status: 403, + message: 'You are not a collaborator of this document', + }, // document / helper [AppErrorCode.DOCUMENT_NOT_FOUND]: { diff --git a/apps/core/src/common/errors/app-error-payload.ts b/apps/core/src/common/errors/app-error-payload.ts index d843b567fb0..ffa7e281a3d 100644 --- a/apps/core/src/common/errors/app-error-payload.ts +++ b/apps/core/src/common/errors/app-error-payload.ts @@ -128,6 +128,18 @@ export type AppErrorPayloadMap = { documentId: string expectedPublishedRevisionId: string | null } + [AppErrorCode.DRAFT_CRDT_UPDATE_REJECTED]: { + branchId: string + reason: string + } + [AppErrorCode.DRAFT_CRDT_FORMAT_UNSUPPORTED]: { + branchId: string + contentFormat: string + } + [AppErrorCode.COLLAB_INVITE_INVALID]: { + reason: 'exhausted' | 'expired' | 'not_found' | 'revoked' + } + [AppErrorCode.COLLAB_FORBIDDEN]: undefined // document / helper [AppErrorCode.DOCUMENT_NOT_FOUND]: WithId diff --git a/apps/core/src/database/migrations/0043_loro_content_storage.sql b/apps/core/src/database/migrations/0043_loro_content_storage.sql new file mode 100644 index 00000000000..b58fc7708be --- /dev/null +++ b/apps/core/src/database/migrations/0043_loro_content_storage.sql @@ -0,0 +1,24 @@ +-- migration-lint:allow=no-bare-create-index reason=index targets the brand-new empty content_crdt_updates table; CONCURRENTLY cannot run inside the migration transaction +CREATE TABLE "content_crdt_docs" ( + "document_id" text PRIMARY KEY NOT NULL, + "snapshot" bytea NOT NULL, + "snapshot_version" bytea NOT NULL, + "synced_revision_id" text, + "fields" jsonb, + "created_at" timestamp with time zone DEFAULT now() NOT NULL, + "updated_at" timestamp with time zone +); +--> statement-breakpoint +CREATE TABLE "content_crdt_updates" ( + "id" text PRIMARY KEY NOT NULL, + "document_id" text NOT NULL, + "bytes" bytea NOT NULL, + "created_at" timestamp with time zone DEFAULT now() NOT NULL +); +--> statement-breakpoint +ALTER TABLE "content_revisions" ADD COLUMN "frontiers" jsonb;--> statement-breakpoint +ALTER TABLE "content_revisions" ADD COLUMN "kind" text;--> statement-breakpoint +ALTER TABLE "content_crdt_docs" ADD CONSTRAINT "content_crdt_docs_document_id_content_documents_id_fk" FOREIGN KEY ("document_id") REFERENCES "public"."content_documents"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "content_crdt_docs" ADD CONSTRAINT "content_crdt_docs_synced_revision_id_content_revisions_id_fk" FOREIGN KEY ("synced_revision_id") REFERENCES "public"."content_revisions"("id") ON DELETE set null ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "content_crdt_updates" ADD CONSTRAINT "content_crdt_updates_document_id_content_documents_id_fk" FOREIGN KEY ("document_id") REFERENCES "public"."content_documents"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +CREATE INDEX "content_crdt_updates_document_idx" ON "content_crdt_updates" USING btree ("document_id"); diff --git a/apps/core/src/database/migrations/0044_content_collaborators.sql b/apps/core/src/database/migrations/0044_content_collaborators.sql new file mode 100644 index 00000000000..eae190c80ad --- /dev/null +++ b/apps/core/src/database/migrations/0044_content_collaborators.sql @@ -0,0 +1,27 @@ +CREATE TABLE "content_collaborators" ( + "id" text PRIMARY KEY NOT NULL, + "document_id" text NOT NULL, + "reader_id" text NOT NULL, + "show_in_byline" boolean DEFAULT true NOT NULL, + "contributed_at" timestamp with time zone, + "created_at" timestamp with time zone DEFAULT now() NOT NULL, + CONSTRAINT "content_collaborators_document_reader_uniq" UNIQUE("document_id","reader_id") +); +--> statement-breakpoint +CREATE TABLE "content_collab_invites" ( + "id" text PRIMARY KEY NOT NULL, + "document_id" text NOT NULL, + "token_hash" text NOT NULL, + "expires_at" timestamp with time zone NOT NULL, + "max_uses" integer, + "uses" integer DEFAULT 0 NOT NULL, + "revoked_at" timestamp with time zone, + "created_at" timestamp with time zone DEFAULT now() NOT NULL, + CONSTRAINT "content_collab_invites_token_hash_uniq" UNIQUE("token_hash") +); +--> statement-breakpoint +ALTER TABLE "content_collaborators" ADD CONSTRAINT "content_collaborators_document_id_content_documents_id_fk" FOREIGN KEY ("document_id") REFERENCES "public"."content_documents"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "content_collaborators" ADD CONSTRAINT "content_collaborators_reader_id_readers_id_fk" FOREIGN KEY ("reader_id") REFERENCES "public"."readers"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "content_collab_invites" ADD CONSTRAINT "content_collab_invites_document_id_content_documents_id_fk" FOREIGN KEY ("document_id") REFERENCES "public"."content_documents"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +-- migration-lint:allow=no-bare-create-index reason=index targets the brand-new empty content_collab_invites table; CONCURRENTLY cannot run inside the migration transaction +CREATE INDEX "content_collab_invites_document_idx" ON "content_collab_invites" USING btree ("document_id"); diff --git a/apps/core/src/database/migrations/0045_collaborator_removed_at.sql b/apps/core/src/database/migrations/0045_collaborator_removed_at.sql new file mode 100644 index 00000000000..3f1d6291d64 --- /dev/null +++ b/apps/core/src/database/migrations/0045_collaborator_removed_at.sql @@ -0,0 +1 @@ +ALTER TABLE "content_collaborators" ADD COLUMN "removed_at" timestamp with time zone; diff --git a/apps/core/src/database/migrations/meta/_journal.json b/apps/core/src/database/migrations/meta/_journal.json index 5890dfe687e..57fc3e962b5 100644 --- a/apps/core/src/database/migrations/meta/_journal.json +++ b/apps/core/src/database/migrations/meta/_journal.json @@ -302,6 +302,27 @@ "when": 1792900000000, "tag": "0042_agent_transcript", "breakpoints": true + }, + { + "idx": 43, + "version": "7", + "when": 1792950000000, + "tag": "0043_loro_content_storage", + "breakpoints": true + }, + { + "idx": 44, + "version": "7", + "when": 1793000000000, + "tag": "0044_content_collaborators", + "breakpoints": true + }, + { + "idx": 45, + "version": "7", + "when": 1793050000000, + "tag": "0045_collaborator_removed_at", + "breakpoints": true } ] } diff --git a/apps/core/src/modules/ai/ai-writer/ai-slug-backfill.service.ts b/apps/core/src/modules/ai/ai-writer/ai-slug-backfill.service.ts index 17ac2f667f0..574cc68e3ee 100644 --- a/apps/core/src/modules/ai/ai-writer/ai-slug-backfill.service.ts +++ b/apps/core/src/modules/ai/ai-writer/ai-slug-backfill.service.ts @@ -143,9 +143,9 @@ export class AiSlugBackfillService implements OnModuleInit { continue } - const updated = await this.noteService.updateById(note.id, { + const updated = await this.noteService.patchById(note.id, { slug, - } as any) + }) if (!updated) { skipped++ await context.appendLog( diff --git a/apps/core/src/modules/content-migration/content-migration-commit.service.ts b/apps/core/src/modules/content-migration/content-migration-commit.service.ts index 87200f4f692..06b930f1bb3 100644 --- a/apps/core/src/modules/content-migration/content-migration-commit.service.ts +++ b/apps/core/src/modules/content-migration/content-migration-commit.service.ts @@ -310,7 +310,15 @@ export class ContentMigrationCommitService { ) .innerJoin( contentRevisions, - eq(contentRevisions.id, drafts.headRevisionId), + and( + eq( + contentRevisions.id, + parseEntityId( + branchPrecondition.headRevisionId ?? input.branchId, + ), + ), + eq(contentRevisions.documentId, drafts.documentId), + ), ) .where(eq(drafts.id, parseEntityId(input.branchId))) .limit(1) @@ -329,10 +337,11 @@ export class ContentMigrationCommitService { draft.revision.content ? sha256(draft.revision.content) : analyzeMigrationMarkdown(draft.revision.text, input).sourceHash - if ( - draft.branch.headRevisionId !== branchPrecondition.headRevisionId || - draftHash !== branchPrecondition.hash - ) { + // A checkpoint is immutable, so it may be published after the working copy moved on. + const headMoved = + !draft.revision.kind && + draft.branch.headRevisionId !== branchPrecondition.headRevisionId + if (headMoved || draftHash !== branchPrecondition.hash) { throw new ConflictException('Draft changed after the migration check') } } diff --git a/apps/core/src/modules/content-migration/content-migration.schema.ts b/apps/core/src/modules/content-migration/content-migration.schema.ts index 97d7b6115c7..b8a264c5315 100644 --- a/apps/core/src/modules/content-migration/content-migration.schema.ts +++ b/apps/core/src/modules/content-migration/content-migration.schema.ts @@ -26,6 +26,7 @@ export const MarkdownToLexicalDryRunSchema = z.object({ refType: z.enum(DraftRefType), refId: zEntityId, branchId: zEntityId.optional(), + revisionId: zEntityId.optional(), sourceText: z.string().max(2_000_000), profile: z.literal('yohaku-v1'), }) diff --git a/apps/core/src/modules/content-migration/content-migration.service.ts b/apps/core/src/modules/content-migration/content-migration.service.ts index 279ab8e87ab..b1b38046131 100644 --- a/apps/core/src/modules/content-migration/content-migration.service.ts +++ b/apps/core/src/modules/content-migration/content-migration.service.ts @@ -154,18 +154,23 @@ export class ContentMigrationService { }) if (draft) { - const draftResult = analyzeMxMarkdown(draft.headRevision.text, { + const revision = dto.revisionId + ? await this.draftService.findRevisionById(dto.revisionId) + : draft.headRevision + if (!revision || revision.documentId !== draft.documentId) { + throw new BadRequestException('Revision does not belong to this draft') + } + const draftResult = analyzeMxMarkdown(revision.text, { profile: 'yohaku-v1', }) preconditions.push({ kind: 'branch', id: String(draft.id), hash: - draft.headRevision.contentFormat === ContentFormat.Lexical && - draft.headRevision.content - ? sha256(draft.headRevision.content) + revision.contentFormat === ContentFormat.Lexical && revision.content + ? sha256(revision.content) : draftResult.sourceHash, - headRevisionId: draft.headRevisionId, + headRevisionId: dto.revisionId ?? draft.headRevisionId, }) } diff --git a/apps/core/src/modules/draft/collab-room.service.ts b/apps/core/src/modules/draft/collab-room.service.ts new file mode 100644 index 00000000000..8f08384e3d0 --- /dev/null +++ b/apps/core/src/modules/draft/collab-room.service.ts @@ -0,0 +1,350 @@ +import type { WsAckPayload } from '@mx-space/ws-client/protocol' +import { + Inject, + Injectable, + Logger, + type OnModuleDestroy, + type OnModuleInit, +} from '@nestjs/common' +import { z } from 'zod' + +import { AppException } from '~/common/errors/exception.types' + +import { DraftService } from './draft.service' +import { type DocumentChange, DraftCrdtService } from './draft-crdt.service' + +export const CollabWsEvents = { + collaboratorsChanged: 'collab.collaborators.changed', + join: 'collab.join', + leave: 'collab.leave', + peerJoined: 'collab.peer.joined', + peerLeft: 'collab.peer.left', + presence: 'collab.presence', + revoked: 'collab.revoked', + update: 'collab.update', +} as const + +export const COLLAB_RELAY = Symbol('COLLAB_RELAY') +export const COLLAB_ACCESS = Symbol('COLLAB_ACCESS') + +export interface CollabAccess { + canJoin: (member: CollabMember, documentId: string) => Promise + recordContribution: (documentId: string, readerId: string) => Promise +} + +export interface CollabRelay { + onRevoke: ( + listener: (documentId: string, readerId: string) => void, + ) => () => void + publish: ( + event: string, + payload: unknown, + room: string, + exclude?: string, + ) => void + revoke: (documentId: string, readerId: string) => void +} + +export interface PresenceUser { + avatar: string | null + id: string + name: string +} + +export type CollabIdentity = + | { kind: 'owner'; user: PresenceUser } + | { kind: 'reader'; readerId: string; user: PresenceUser } + +export interface CollabMember { + alive: () => boolean + connectionId: string + identity: CollabIdentity + join: (room: string) => Promise + leave: (room: string) => Promise + send: (event: string, payload: unknown) => void +} + +interface Joined { + contributed: boolean + documentId: string + member: CollabMember +} + +interface Bucket { + at: number + tokens: number +} + +const MAX_UPDATE_BYTES = 4 * 1024 * 1024 +const MAX_CARET_CHARS = 4096 +const BUCKET_SIZE = 120 +const BUCKET_REFILL_PER_MS = 60 / 1000 +const REJECTED = 'DRAFT_CRDT_UPDATE_REJECTED' + +const base64 = z.string().max(Math.ceil((MAX_UPDATE_BYTES * 4) / 3) + 8) +const draftId = z.string().min(1).max(64) +const joinSchema = z.object({ draftId, since: base64.optional() }) +const updateSchema = z.object({ draftId, update: z.string() }) +const presenceSchema = z.object({ + caret: z.unknown(), + draftId, + entry: z + .string() + .regex(/^agent:\w{1,40}$/) + .optional(), +}) +const leaveSchema = z.object({ draftId }) + +const roomOf = (documentId: string) => `collab:${documentId}` +const encode = (bytes: Uint8Array) => Buffer.from(bytes).toString('base64') +const decode = (value: string) => new Uint8Array(Buffer.from(value, 'base64')) + +const invalid: WsAckPayload = { code: 'VALIDATION_FAILED', ok: false } +const rateLimited: WsAckPayload = { code: 'RATE_LIMITED', ok: false } +const notJoined: WsAckPayload = { code: 'NOT_JOINED', ok: false } + +const authorOf = (identity: CollabIdentity) => + identity.kind === 'reader' ? `human:${identity.readerId}` : undefined + +@Injectable() +export class CollabRoomService implements OnModuleInit, OnModuleDestroy { + private readonly logger = new Logger(CollabRoomService.name) + private readonly joined = new Map>() + private readonly buckets = new Map() + private stopListening: Array<() => void> = [] + + constructor( + private readonly drafts: DraftService, + private readonly crdt: DraftCrdtService, + @Inject(COLLAB_RELAY) private readonly relay: CollabRelay, + @Inject(COLLAB_ACCESS) private readonly access: CollabAccess, + ) {} + + onModuleInit() { + this.stopListening = [ + this.crdt.onDocumentChanged((change) => this.relayChange(change)), + this.relay.onRevoke((documentId, readerId) => + this.dropReader(documentId, readerId), + ), + ] + } + + onModuleDestroy() { + for (const stop of this.stopListening.splice(0)) stop() + } + + revoke(documentId: string, readerId: string) { + this.dropReader(documentId, readerId) + this.relay.revoke(documentId, readerId) + } + + async join(member: CollabMember, raw: unknown): Promise { + const parsed = joinSchema.safeParse(raw) + if (!parsed.success) return invalid + const { draftId, since } = parsed.data + let room: string | null = null + let entry: Joined | null = null + const backOut = async (ack: WsAckPayload) => { + this.forget(member.connectionId, draftId, entry) + if (room) await member.leave(room).catch(() => undefined) + return ack + } + try { + const documentId = await this.drafts.documentIdOf(draftId) + if (!(await this.access.canJoin(member, documentId))) { + return { code: 'FORBIDDEN', ok: false } + } + // Registered before joining so a removal that lands mid-join finds and drops it. + entry = { contributed: false, documentId, member } + this.membership(member.connectionId).set(draftId, entry) + room = roomOf(documentId) + // Join before reading: a change stored after the read is then relayed to this member, + // and one stored before it is in the snapshot. Loro dedupes anything seen twice. + await member.join(room) + if (!member.alive()) return backOut({ code: 'NOT_CONNECTED', ok: false }) + const { update } = await this.drafts.exportCrdtFor( + draftId, + since ? decode(since) : undefined, + ) + if (!member.alive()) return backOut({ code: 'NOT_CONNECTED', ok: false }) + const stillJoined = + this.joined.get(member.connectionId)?.get(draftId) === entry + if (!stillJoined || !(await this.access.canJoin(member, documentId))) { + return backOut({ code: 'FORBIDDEN', ok: false }) + } + this.relay.publish( + CollabWsEvents.peerJoined, + { connectionId: member.connectionId, documentId }, + room, + member.connectionId, + ) + return { + data: { + connectionId: member.connectionId, + documentId, + update: encode(update), + }, + ok: true, + } + } catch (error) { + return backOut(this.failure(error)) + } + } + + async update(member: CollabMember, raw: unknown): Promise { + const parsed = updateSchema.safeParse(raw) + if (!parsed.success) return invalid + const bytes = decode(parsed.data.update) + if (bytes.length > MAX_UPDATE_BYTES) return { code: REJECTED, ok: false } + const joined = this.joined + .get(member.connectionId) + ?.get(parsed.data.draftId) + if (!joined) return notJoined + if (!this.take(member)) return rateLimited + try { + await this.drafts.importCrdt( + parsed.data.draftId, + bytes, + member.connectionId, + authorOf(member.identity), + ) + const { identity } = member + if (identity.kind === 'reader' && !joined.contributed) { + joined.contributed = true + await this.access.recordContribution( + joined.documentId, + identity.readerId, + ) + } + return { ok: true } + } catch (error) { + return this.failure(error) + } + } + + async presence(member: CollabMember, raw: unknown): Promise { + const parsed = presenceSchema.safeParse(raw) + if (!parsed.success) return invalid + const { caret, draftId, entry } = parsed.data + const caretChars = JSON.stringify(caret ?? null).length + if (caretChars > MAX_CARET_CHARS) return invalid + const joined = this.joined.get(member.connectionId)?.get(draftId) + if (!joined) return notJoined + if (entry && member.identity.kind !== 'owner') { + return { code: 'FORBIDDEN', ok: false } + } + if (!this.take(member)) return rateLimited + this.relay.publish( + CollabWsEvents.presence, + { + caret: caret ?? null, + connectionId: member.connectionId, + documentId: joined.documentId, + entry: entry ?? null, + user: entry + ? { avatar: null, id: entry, name: 'AI' } + : member.identity.user, + }, + roomOf(joined.documentId), + member.connectionId, + ) + return { ok: true } + } + + async leave(member: CollabMember, raw: unknown): Promise { + const parsed = leaveSchema.safeParse(raw) + if (!parsed.success) return invalid + const rooms = this.joined.get(member.connectionId) + const joined = rooms?.get(parsed.data.draftId) + if (!rooms || !joined) return { ok: true } + this.forget(member.connectionId, parsed.data.draftId) + await member.leave(roomOf(joined.documentId)) + this.announceLeft(member.connectionId, joined.documentId) + return { ok: true } + } + + disconnect(connectionId: string) { + const rooms = this.joined.get(connectionId) + this.buckets.delete(connectionId) + if (!rooms) return + this.joined.delete(connectionId) + for (const documentId of new Set( + [...rooms.values()].map((joined) => joined.documentId), + )) { + this.announceLeft(connectionId, documentId) + } + } + + private dropReader(documentId: string, readerId: string) { + for (const [connectionId, rooms] of this.joined) { + for (const [draftId, joined] of rooms) { + const { identity } = joined.member + if (joined.documentId !== documentId) continue + if (identity.kind !== 'reader' || identity.readerId !== readerId) { + continue + } + this.forget(connectionId, draftId) + void joined.member.leave(roomOf(documentId)).catch(() => undefined) + joined.member.send(CollabWsEvents.revoked, { documentId }) + this.announceLeft(connectionId, documentId) + } + } + } + + private forget(connectionId: string, draftId: string, only?: Joined | null) { + const rooms = this.joined.get(connectionId) + if (only !== undefined && rooms?.get(draftId) !== only) return + rooms?.delete(draftId) + if (rooms && !rooms.size) this.joined.delete(connectionId) + } + + private take(member: CollabMember) { + if (member.identity.kind === 'owner') return true + const now = Date.now() + const bucket = this.buckets.get(member.connectionId) ?? { + at: now, + tokens: BUCKET_SIZE, + } + bucket.tokens = Math.min( + BUCKET_SIZE, + bucket.tokens + (now - bucket.at) * BUCKET_REFILL_PER_MS, + ) + bucket.at = now + this.buckets.set(member.connectionId, bucket) + if (bucket.tokens < 1) return false + bucket.tokens -= 1 + return true + } + + private relayChange(change: DocumentChange) { + this.relay.publish( + CollabWsEvents.update, + { documentId: change.documentId, update: encode(change.update) }, + roomOf(change.documentId), + change.origin, + ) + } + + private announceLeft(connectionId: string, documentId: string) { + this.relay.publish( + CollabWsEvents.peerLeft, + { connectionId, documentId }, + roomOf(documentId), + ) + } + + private membership(connectionId: string) { + let rooms = this.joined.get(connectionId) + if (!rooms) { + rooms = new Map() + this.joined.set(connectionId, rooms) + } + return rooms + } + + private failure(error: unknown): WsAckPayload { + if (error instanceof AppException) return { code: error.code, ok: false } + this.logger.warn(`collab request failed: ${(error as Error).message}`) + return { code: 'INTERNAL_ERROR', ok: false } + } +} diff --git a/apps/core/src/modules/draft/collab-room.ws.ts b/apps/core/src/modules/draft/collab-room.ws.ts new file mode 100644 index 00000000000..dbbebee5453 --- /dev/null +++ b/apps/core/src/modules/draft/collab-room.ws.ts @@ -0,0 +1,154 @@ +import { + Injectable, + type OnModuleDestroy, + type OnModuleInit, + type Provider, +} from '@nestjs/common' + +import { + type AdminWsContext, + AdminWsHandlerRegistry, +} from '~/processors/gateway/admin/admin-ws-handler.registry' +import { + type WebWsContext, + WebWsHandlerRegistry, +} from '~/processors/gateway/web/web-ws-handler.registry' +import { WsBusService } from '~/processors/gateway/ws/ws-bus.service' + +import { CollabRepository } from './collab.repository' +import { CollabService } from './collab.service' +import { + COLLAB_ACCESS, + COLLAB_RELAY, + type CollabAccess, + type CollabIdentity, + type CollabMember, + type CollabRelay, + CollabRoomService, + CollabWsEvents, +} from './collab-room.service' + +const REVOKE_EVENT = 'collab.revoke' + +export const collabRelayProvider: Provider = { + inject: [WsBusService], + provide: COLLAB_RELAY, + useFactory: (bus: WsBusService): CollabRelay => ({ + onRevoke: (listener) => + bus.register('collab', (frame) => { + const payload = frame.payload as + { documentId?: string; readerId?: string } | undefined + if (frame.event !== REVOKE_EVENT) return + if (payload?.documentId && payload.readerId) { + listener(payload.documentId, payload.readerId) + } + }), + publish: (event, payload, room, exclude) => { + for (const ns of ['admin', 'web'] as const) { + bus.publish({ + event, + exclude: exclude ? [exclude] : undefined, + ns, + payload, + rooms: [room], + }) + } + }, + revoke: (documentId, readerId) => + bus.publish({ + event: REVOKE_EVENT, + ns: 'collab', + payload: { documentId, readerId }, + }), + }), +} + +export const collabAccessProvider: Provider = { + inject: [CollabRepository], + provide: COLLAB_ACCESS, + useFactory: (repository: CollabRepository): CollabAccess => ({ + canJoin: async (member, documentId) => + member.identity.kind === 'owner' || + repository.isCollaborator(documentId, member.identity.readerId), + recordContribution: (documentId, readerId) => + repository.markContributed(documentId, readerId), + }), +} + +const memberOf = ( + ctx: AdminWsContext | WebWsContext, + identity: CollabIdentity, +): CollabMember => ({ + alive: ctx.alive, + connectionId: ctx.connectionId, + identity, + join: ctx.join, + leave: ctx.leave, + send: ctx.send, +}) + +const unauthenticated = { code: 'NOT_AUTHENTICATED', ok: false } as const + +type RoomHandler = ( + room: CollabRoomService, + member: CollabMember, + payload: unknown, +) => ReturnType + +const ROOM_HANDLERS: Array<[string, RoomHandler]> = [ + [CollabWsEvents.join, (room, member, payload) => room.join(member, payload)], + [ + CollabWsEvents.update, + (room, member, payload) => room.update(member, payload), + ], + [ + CollabWsEvents.presence, + (room, member, payload) => room.presence(member, payload), + ], + [ + CollabWsEvents.leave, + (room, member, payload) => room.leave(member, payload), + ], +] + +@Injectable() +export class CollabRoomWsHandler implements OnModuleInit, OnModuleDestroy { + private readonly disposers: Array<() => void> = [] + + constructor( + private readonly adminRegistry: AdminWsHandlerRegistry, + private readonly webRegistry: WebWsHandlerRegistry, + private readonly rooms: CollabRoomService, + private readonly collab: CollabService, + ) {} + + onModuleInit() { + for (const [event, handle] of ROOM_HANDLERS) { + this.disposers.push( + this.adminRegistry.register(event, async (ctx, payload) => { + const identity = await this.collab.ownerIdentity() + if (!identity) return unauthenticated + return handle(this.rooms, memberOf(ctx, identity), payload) + }), + this.webRegistry.register(event, async (ctx, payload) => { + const identity = ctx.readerId + ? await this.collab.identityOf(ctx.readerId) + : null + if (!identity) return unauthenticated + return handle(this.rooms, memberOf(ctx, identity), payload) + }), + ) + } + for (const registry of [this.adminRegistry, this.webRegistry]) { + this.disposers.push( + registry.onDisconnect((connectionId) => + this.rooms.disconnect(connectionId), + ), + ) + } + } + + onModuleDestroy() { + for (const dispose of this.disposers.splice(0)) dispose() + } +} diff --git a/apps/core/src/modules/draft/collab.controller.ts b/apps/core/src/modules/draft/collab.controller.ts new file mode 100644 index 00000000000..475549d1ff2 --- /dev/null +++ b/apps/core/src/modules/draft/collab.controller.ts @@ -0,0 +1,171 @@ +import { + Body, + Delete, + Get, + HttpCode, + Param, + Patch, + Post, + Query, + Req, + Res, +} from '@nestjs/common' +import type { FastifyReply, FastifyRequest } from 'fastify' + +import { ApiController } from '~/common/decorators/api-controller.decorator' +import { Auth } from '~/common/decorators/auth.decorator' +import { CurrentReaderId } from '~/common/decorators/current-user.decorator' +import { WithFastifyRouteOptions } from '~/common/decorators/fastify-route-options.decorator' +import { HTTPDecorators } from '~/common/decorators/http.decorator' +import { ReaderAuth } from '~/common/decorators/reader-auth.decorator' +import { AppErrorCode, createAppException } from '~/common/errors' + +import { + type CollabCollaboratorParamsDto, + CollabCollaboratorParamsSchema, + type CollabCrdtQueryDto, + CollabCrdtQuerySchema, + type CollabDocumentParamsDto, + CollabDocumentParamsSchema, + type CollabInviteParamsDto, + CollabInviteParamsSchema, + type CollabTokenParamsDto, + CollabTokenParamsSchema, + type CreateCollabInviteDto, + CreateCollabInviteSchema, + type UpdateCollaboratorDto, + UpdateCollaboratorSchema, +} from './collab.schema' +import { CollabService } from './collab.service' + +const CRDT_UPDATE_BODY_LIMIT = 4 * 1024 * 1024 + +@ApiController('collab') +export class CollabController { + constructor(private readonly collab: CollabService) {} + + @Get('/documents/:documentId/invites') + @Auth() + listInvites( + @Param({ schema: CollabDocumentParamsSchema }) + params: CollabDocumentParamsDto, + ) { + return this.collab.listInvites(params.documentId) + } + + @Post('/documents/:documentId/invites') + @Auth() + createInvite( + @Param({ schema: CollabDocumentParamsSchema }) + params: CollabDocumentParamsDto, + @Body({ schema: CreateCollabInviteSchema }) body: CreateCollabInviteDto, + ) { + return this.collab.createInvite(params.documentId, body) + } + + @Delete('/documents/:documentId/invites/:inviteId') + @Auth() + @HttpCode(204) + async revokeInvite( + @Param({ schema: CollabInviteParamsSchema }) params: CollabInviteParamsDto, + ) { + await this.collab.revokeInvite(params.documentId, params.inviteId) + } + + @Get('/documents/:documentId/collaborators') + @Auth() + listCollaborators( + @Param({ schema: CollabDocumentParamsSchema }) + params: CollabDocumentParamsDto, + ) { + return this.collab.listCollaborators(params.documentId) + } + + @Patch('/documents/:documentId/collaborators/:readerId') + @Auth() + @HttpCode(204) + async updateCollaborator( + @Param({ schema: CollabCollaboratorParamsSchema }) + params: CollabCollaboratorParamsDto, + @Body({ schema: UpdateCollaboratorSchema }) body: UpdateCollaboratorDto, + ) { + await this.collab.setShowInByline( + params.documentId, + params.readerId, + body.showInByline, + ) + } + + @Delete('/documents/:documentId/collaborators/:readerId') + @Auth() + @HttpCode(204) + async removeCollaborator( + @Param({ schema: CollabCollaboratorParamsSchema }) + params: CollabCollaboratorParamsDto, + ) { + await this.collab.removeCollaborator(params.documentId, params.readerId) + } + + @Post('/invites/:token/accept') + @ReaderAuth() + accept( + @Param({ schema: CollabTokenParamsSchema }) params: CollabTokenParamsDto, + @CurrentReaderId() readerId: string, + ) { + return this.collab.accept(params.token, readerId) + } + + @Get('/documents/:documentId') + @ReaderAuth() + workspace( + @Param({ schema: CollabDocumentParamsSchema }) + params: CollabDocumentParamsDto, + @CurrentReaderId() readerId: string, + ) { + return this.collab.workspaceFor(params.documentId, readerId) + } + + @Get('/documents/:documentId/crdt') + @ReaderAuth() + @HTTPDecorators.RawResponse + async exportCrdt( + @Param({ schema: CollabDocumentParamsSchema }) + params: CollabDocumentParamsDto, + @Query({ schema: CollabCrdtQuerySchema }) query: CollabCrdtQueryDto, + @CurrentReaderId() readerId: string, + @Res({ passthrough: true }) reply: FastifyReply, + ) { + const bytes = await this.collab.exportCrdt( + params.documentId, + readerId, + query.since + ? new Uint8Array(Buffer.from(query.since, 'base64')) + : undefined, + ) + reply.type('application/octet-stream') + return Buffer.from(bytes) + } + + @Post('/documents/:documentId/crdt') + @ReaderAuth() + @HttpCode(204) + @WithFastifyRouteOptions({ bodyLimit: CRDT_UPDATE_BODY_LIMIT }) + async importCrdt( + @Param({ schema: CollabDocumentParamsSchema }) + params: CollabDocumentParamsDto, + @CurrentReaderId() readerId: string, + @Req() request: FastifyRequest, + ) { + if (!Buffer.isBuffer(request.body)) { + throw createAppException(AppErrorCode.DRAFT_CRDT_UPDATE_REJECTED, { + branchId: params.documentId, + reason: 'expected an application/octet-stream body', + }) + } + await this.collab.importCrdt( + params.documentId, + readerId, + new Uint8Array(request.body), + ) + } +} diff --git a/apps/core/src/modules/draft/collab.repository.ts b/apps/core/src/modules/draft/collab.repository.ts new file mode 100644 index 00000000000..b990fb60b6d --- /dev/null +++ b/apps/core/src/modules/draft/collab.repository.ts @@ -0,0 +1,372 @@ +import { Inject, Injectable } from '@nestjs/common' +import { and, asc, eq, gt, inArray, isNotNull, isNull, sql } from 'drizzle-orm' + +import { PG_DB_TOKEN } from '~/constants/system.constant' +import { + contentCollabInvites, + contentCollaborators, + contentDocuments, + readers, +} from '~/database/schema' +import { + BaseRepository, + toEntityId, +} from '~/processors/database/base.repository' +import type { AppDatabase } from '~/processors/database/postgres.provider' +import { type EntityId, parseEntityId } from '~/shared/id/entity-id' +import { SnowflakeService } from '~/shared/id/snowflake.service' + +export interface CollabInviteRow { + createdAt: Date + documentId: EntityId + expiresAt: Date + id: EntityId + maxUses: number | null + tokenHash: string + uses: number +} + +export interface CollaboratorRow { + avatar: string | null + contributedAt: Date | null + createdAt: Date + name: string + readerId: string + removedAt: Date | null + showInByline: boolean +} + +export interface CoAuthor { + avatar: string | null + name: string +} + +export type AcceptInviteResult = + | { documentId: EntityId; ok: true } + | { ok: false; reason: 'exhausted' | 'expired' | 'not_found' | 'revoked' } + +const mapInvite = ( + row: typeof contentCollabInvites.$inferSelect, +): CollabInviteRow => ({ + createdAt: row.createdAt, + documentId: toEntityId(row.documentId)!, + expiresAt: row.expiresAt, + id: toEntityId(row.id)!, + maxUses: row.maxUses, + tokenHash: row.tokenHash, + uses: row.uses, +}) + +const usableInvite = ( + invite: typeof contentCollabInvites.$inferSelect | undefined, +): AcceptInviteResult => { + if (!invite) return { ok: false, reason: 'not_found' } + if (invite.revokedAt) return { ok: false, reason: 'revoked' } + if (invite.expiresAt.getTime() <= Date.now()) { + return { ok: false, reason: 'expired' } + } + return { documentId: toEntityId(invite.documentId)!, ok: true } +} + +const displayName = sql`coalesce(${readers.name}, ${readers.displayUsername}, ${readers.username}, '')` + +@Injectable() +export class CollabRepository extends BaseRepository { + constructor( + @Inject(PG_DB_TOKEN) db: AppDatabase, + private readonly snowflake: SnowflakeService, + ) { + super(db) + } + + async createInvite(input: { + documentId: EntityId | string + expiresAt: Date + maxUses: number | null + tokenHash: string + }): Promise { + const [row] = await this.db + .insert(contentCollabInvites) + .values({ + documentId: parseEntityId(input.documentId), + expiresAt: input.expiresAt, + id: this.snowflake.nextId(), + maxUses: input.maxUses, + tokenHash: input.tokenHash, + }) + .returning() + return mapInvite(row) + } + + async listInvites(documentId: EntityId | string): Promise { + const rows = await this.db + .select() + .from(contentCollabInvites) + .where( + and( + eq(contentCollabInvites.documentId, parseEntityId(documentId)), + isNull(contentCollabInvites.revokedAt), + gt(contentCollabInvites.expiresAt, new Date()), + ), + ) + .orderBy(asc(contentCollabInvites.createdAt)) + return rows + .filter((row) => row.maxUses === null || row.uses < row.maxUses) + .map(mapInvite) + } + + async revokeInvite( + documentId: EntityId | string, + inviteId: EntityId | string, + ): Promise { + const rows = await this.db + .update(contentCollabInvites) + .set({ revokedAt: new Date() }) + .where( + and( + eq(contentCollabInvites.id, parseEntityId(inviteId)), + eq(contentCollabInvites.documentId, parseEntityId(documentId)), + isNull(contentCollabInvites.revokedAt), + ), + ) + .returning({ id: contentCollabInvites.id }) + return rows.length > 0 + } + + async acceptInvite( + tokenHash: string, + readerId: string, + ): Promise { + return this.db.transaction(async (tx) => { + const [invite] = await tx + .select() + .from(contentCollabInvites) + .where(eq(contentCollabInvites.tokenHash, tokenHash)) + .for('update') + .limit(1) + const usable = usableInvite(invite) + if (!usable.ok || !invite) return usable + const { documentId } = usable + const [existing] = await tx + .select({ + id: contentCollaborators.id, + removedAt: contentCollaborators.removedAt, + }) + .from(contentCollaborators) + .where( + and( + eq(contentCollaborators.documentId, invite.documentId), + eq(contentCollaborators.readerId, readerId), + ), + ) + .limit(1) + if (existing && !existing.removedAt) return { documentId, ok: true } + // A link made before the removal must not undo it; only a later invite brings them back. + if (existing?.removedAt && invite.createdAt <= existing.removedAt) { + return { ok: false, reason: 'revoked' } + } + if (invite.maxUses !== null && invite.uses >= invite.maxUses) { + return { ok: false, reason: 'exhausted' } + } + if (existing) { + await tx + .update(contentCollaborators) + .set({ removedAt: null }) + .where(eq(contentCollaborators.id, existing.id)) + } else { + await tx.insert(contentCollaborators).values({ + documentId: invite.documentId, + id: this.snowflake.nextId(), + readerId, + }) + } + await tx + .update(contentCollabInvites) + .set({ uses: invite.uses + 1 }) + .where(eq(contentCollabInvites.id, invite.id)) + return { documentId, ok: true } + }) + } + + async peekInvite(tokenHash: string): Promise { + const [invite] = await this.db + .select() + .from(contentCollabInvites) + .where(eq(contentCollabInvites.tokenHash, tokenHash)) + .limit(1) + const usable = usableInvite(invite) + if (!usable.ok || !invite) return usable + if (invite.maxUses !== null && invite.uses >= invite.maxUses) { + return { ok: false, reason: 'exhausted' } + } + return usable + } + + async addCollaborator( + documentId: EntityId | string, + readerId: string, + ): Promise { + await this.db + .insert(contentCollaborators) + .values({ + documentId: parseEntityId(documentId), + id: this.snowflake.nextId(), + readerId, + }) + .onConflictDoNothing() + } + + async isCollaborator( + documentId: EntityId | string, + readerId: string, + ): Promise { + const [row] = await this.db + .select({ id: contentCollaborators.id }) + .from(contentCollaborators) + .where(this.member(documentId, readerId)) + .limit(1) + return Boolean(row) + } + + async listCollaborators( + documentId: EntityId | string, + ): Promise { + return this.db + .select({ + avatar: readers.image, + contributedAt: contentCollaborators.contributedAt, + createdAt: contentCollaborators.createdAt, + name: displayName, + readerId: contentCollaborators.readerId, + removedAt: contentCollaborators.removedAt, + showInByline: contentCollaborators.showInByline, + }) + .from(contentCollaborators) + .innerJoin(readers, eq(readers.id, contentCollaborators.readerId)) + .where(eq(contentCollaborators.documentId, parseEntityId(documentId))) + .orderBy( + asc(contentCollaborators.createdAt), + asc(contentCollaborators.id), + ) + } + + async removeCollaborator( + documentId: EntityId | string, + readerId: string, + ): Promise { + const rows = await this.db + .update(contentCollaborators) + .set({ removedAt: sql`now()` }) + .where(this.member(documentId, readerId)) + .returning({ id: contentCollaborators.id }) + return rows.length > 0 + } + + async setShowInByline( + documentId: EntityId | string, + readerId: string, + showInByline: boolean, + ): Promise { + const rows = await this.db + .update(contentCollaborators) + .set({ showInByline }) + .where(this.member(documentId, readerId)) + .returning({ id: contentCollaborators.id }) + return rows.length > 0 + } + + async markContributed( + documentId: EntityId | string, + readerId: string, + ): Promise { + await this.db + .update(contentCollaborators) + .set({ contributedAt: sql`now()` }) + .where( + and( + this.member(documentId, readerId), + isNull(contentCollaborators.contributedAt), + ), + ) + } + + async coAuthorsByRef( + refType: string, + refIds: Array, + ): Promise> { + const result = new Map() + if (!refIds.length) return result + const rows = await this.db + .select({ + avatar: readers.image, + name: displayName, + refId: contentDocuments.refId, + }) + .from(contentCollaborators) + .innerJoin( + contentDocuments, + eq(contentDocuments.id, contentCollaborators.documentId), + ) + .innerJoin(readers, eq(readers.id, contentCollaborators.readerId)) + .where( + and( + eq(contentDocuments.refType, refType), + inArray( + contentDocuments.refId, + refIds.map((id) => parseEntityId(id)), + ), + isNotNull(contentCollaborators.contributedAt), + isNull(contentCollaborators.removedAt), + eq(contentCollaborators.showInByline, true), + ), + ) + .orderBy( + asc(contentCollaborators.contributedAt), + asc(contentCollaborators.id), + ) + for (const { refId, ...author } of rows) { + const key = String(toEntityId(refId)) + const list = result.get(key) ?? [] + list.push(author) + result.set(key, list) + } + return result + } + + async presenceUserOf(readerId: string): Promise<{ + avatar: string | null + id: string + name: string + role: string + } | null> { + const [row] = await this.db + .select({ + avatar: readers.image, + id: readers.id, + name: displayName, + role: readers.role, + }) + .from(readers) + .where(eq(readers.id, readerId)) + .limit(1) + return row ?? null + } + + async ownerId(): Promise { + const [row] = await this.db + .select({ id: readers.id }) + .from(readers) + .where(eq(readers.role, 'owner')) + .orderBy(asc(readers.createdAt), asc(readers.id)) + .limit(1) + return row?.id ?? null + } + + private member(documentId: EntityId | string, readerId: string) { + return and( + eq(contentCollaborators.documentId, parseEntityId(documentId)), + eq(contentCollaborators.readerId, readerId), + isNull(contentCollaborators.removedAt), + ) + } +} diff --git a/apps/core/src/modules/draft/collab.schema.ts b/apps/core/src/modules/draft/collab.schema.ts new file mode 100644 index 00000000000..f0113b899c7 --- /dev/null +++ b/apps/core/src/modules/draft/collab.schema.ts @@ -0,0 +1,41 @@ +import { z } from 'zod' + +import { zEntityId } from '~/common/zod' + +export const CollabDocumentParamsSchema = z.object({ + documentId: zEntityId, +}) +export type CollabDocumentParamsDto = z.infer + +export const CollabInviteParamsSchema = CollabDocumentParamsSchema.extend({ + inviteId: zEntityId, +}) +export type CollabInviteParamsDto = z.infer + +export const CollabCollaboratorParamsSchema = CollabDocumentParamsSchema.extend( + { readerId: z.string().min(1).max(64) }, +) +export type CollabCollaboratorParamsDto = z.infer< + typeof CollabCollaboratorParamsSchema +> + +export const CollabTokenParamsSchema = z.object({ + token: z.string().min(16).max(128), +}) +export type CollabTokenParamsDto = z.infer + +export const CreateCollabInviteSchema = z.object({ + expiresInDays: z.number().int().min(1).max(30).optional(), + maxUses: z.number().int().min(1).max(100).nullable().optional(), +}) +export type CreateCollabInviteDto = z.infer + +export const UpdateCollaboratorSchema = z.object({ + showInByline: z.boolean(), +}) +export type UpdateCollaboratorDto = z.infer + +export const CollabCrdtQuerySchema = z.object({ + since: z.string().max(65_536).optional(), +}) +export type CollabCrdtQueryDto = z.infer diff --git a/apps/core/src/modules/draft/collab.service.ts b/apps/core/src/modules/draft/collab.service.ts new file mode 100644 index 00000000000..481d2a18f0d --- /dev/null +++ b/apps/core/src/modules/draft/collab.service.ts @@ -0,0 +1,232 @@ +import { createHash, randomBytes } from 'node:crypto' + +import { Injectable } from '@nestjs/common' + +import { AppErrorCode, createAppException } from '~/common/errors' +import { WsBusService } from '~/processors/gateway/ws/ws-bus.service' + +import { CollabRepository } from './collab.repository' +import type { CollabIdentity } from './collab-room.service' +import { CollabRoomService, CollabWsEvents } from './collab-room.service' +import { DraftRepository } from './draft.repository' +import { DraftService } from './draft.service' + +const INVITE_TTL_MS = 7 * 24 * 3600 * 1000 +const IDENTITY_TTL_MS = 60_000 + +const hashToken = (token: string) => + createHash('sha256').update(token).digest('hex') + +export type AcceptedInvite = + | { documentId: string; role: 'collaborator' } + | { + documentId: string + draftId: string + refId: string | null + refType: string + role: 'owner' + } + +@Injectable() +export class CollabService { + private readonly identities = new Map< + string, + { at: number; identity: Promise } + >() + + private ownerId: { at: number; id: Promise } | null = null + + constructor( + private readonly repository: CollabRepository, + private readonly draftRepository: DraftRepository, + private readonly drafts: DraftService, + private readonly rooms: CollabRoomService, + private readonly bus: WsBusService, + ) {} + + async createInvite( + documentId: string, + options: { expiresInDays?: number; maxUses?: number | null }, + ) { + const token = randomBytes(32).toString('base64url') + const expiresAt = new Date( + Date.now() + + (options.expiresInDays + ? options.expiresInDays * 24 * 3600 * 1000 + : INVITE_TTL_MS), + ) + const invite = await this.repository.createInvite({ + documentId, + expiresAt, + maxUses: options.maxUses ?? null, + tokenHash: hashToken(token), + }) + return { + createdAt: invite.createdAt, + expiresAt: invite.expiresAt, + id: invite.id, + maxUses: invite.maxUses, + token, + uses: invite.uses, + } + } + + async listInvites(documentId: string) { + const invites = await this.repository.listInvites(documentId) + return invites.map(({ createdAt, expiresAt, id, maxUses, uses }) => ({ + createdAt, + expiresAt, + id, + maxUses, + uses, + })) + } + + revokeInvite(documentId: string, inviteId: string) { + return this.repository.revokeInvite(documentId, inviteId) + } + + async accept(token: string, readerId: string): Promise { + const identity = await this.identityOf(readerId) + if (identity?.kind === 'owner') { + const result = await this.repository.peekInvite(hashToken(token)) + if (!result.ok) { + throw createAppException(AppErrorCode.COLLAB_INVITE_INVALID, { + reason: result.reason, + }) + } + const document = await this.draftRepository.findDocumentById( + result.documentId, + ) + return { + documentId: String(result.documentId), + draftId: await this.draftIdOf(result.documentId), + refId: document?.refId ? String(document.refId) : null, + refType: String(document?.refType ?? ''), + role: 'owner', + } + } + const result = await this.repository.acceptInvite( + hashToken(token), + readerId, + ) + if (!result.ok) { + throw createAppException(AppErrorCode.COLLAB_INVITE_INVALID, { + reason: result.reason, + }) + } + const documentId = String(result.documentId) + this.bus.publish({ + event: CollabWsEvents.collaboratorsChanged, + ns: 'admin', + payload: { documentId }, + }) + return { documentId, role: 'collaborator' } + } + + async workspaceFor(documentId: string, readerId: string) { + await this.assertMember(documentId, readerId) + const [document, draftId, collaborators, identity] = await Promise.all([ + this.draftRepository.findDocumentById(documentId), + this.draftIdOf(documentId), + this.repository.listCollaborators(documentId), + this.identityOf(readerId), + ]) + return { + collaborators: collaborators + .filter((person) => !person.removedAt) + .map(({ avatar, name }) => ({ avatar, name })), + documentId, + draftId, + me: identity!.user, + refType: String(document?.refType ?? ''), + } + } + + async exportCrdt( + documentId: string, + readerId: string, + since: Uint8Array | undefined, + ) { + await this.assertMember(documentId, readerId) + return this.drafts.exportCrdt(await this.draftIdOf(documentId), since) + } + + async importCrdt(documentId: string, readerId: string, bytes: Uint8Array) { + await this.assertMember(documentId, readerId) + await this.drafts.importCrdt( + await this.draftIdOf(documentId), + bytes, + undefined, + `human:${readerId}`, + ) + await this.repository.markContributed(documentId, readerId) + } + + listCollaborators(documentId: string) { + return this.repository.listCollaborators(documentId) + } + + setShowInByline(documentId: string, readerId: string, show: boolean) { + return this.repository.setShowInByline(documentId, readerId, show) + } + + async removeCollaborator(documentId: string, readerId: string) { + const removed = await this.repository.removeCollaborator( + documentId, + readerId, + ) + this.rooms.revoke(documentId, readerId) + return removed + } + + async ownerIdentity(): Promise { + if (!this.ownerId || Date.now() - this.ownerId.at >= IDENTITY_TTL_MS) { + const id = this.repository.ownerId() + this.ownerId = { at: Date.now(), id } + id.catch(() => { + this.ownerId = null + }) + } + const ownerId = await this.ownerId.id + return ownerId ? this.identityOf(ownerId) : null + } + + identityOf(readerId: string): Promise { + const cached = this.identities.get(readerId) + if (cached && Date.now() - cached.at < IDENTITY_TTL_MS) { + return cached.identity + } + const identity = this.repository + .presenceUserOf(readerId) + .then((row): CollabIdentity | null => { + if (!row) return null + const user = { avatar: row.avatar, id: row.id, name: row.name } + return row.role === 'owner' + ? { kind: 'owner', user } + : { kind: 'reader', readerId: row.id, user } + }) + this.identities.set(readerId, { at: Date.now(), identity }) + identity.catch(() => this.identities.delete(readerId)) + return identity + } + + private async assertMember(documentId: string, readerId: string) { + const identity = await this.identityOf(readerId) + if (identity?.kind === 'owner') return + if (!(await this.repository.isCollaborator(documentId, readerId))) { + throw createAppException(AppErrorCode.COLLAB_FORBIDDEN) + } + } + + private async draftIdOf(documentId: string) { + const [branch] = + await this.draftRepository.findBranchesByDocument(documentId) + if (!branch) { + throw createAppException(AppErrorCode.DRAFT_NOT_FOUND, { + id: documentId, + }) + } + return String(branch.id) + } +} diff --git a/apps/core/src/modules/draft/draft-content.ts b/apps/core/src/modules/draft/draft-content.ts index 5968fb00183..2478fa4a0f1 100644 --- a/apps/core/src/modules/draft/draft-content.ts +++ b/apps/core/src/modules/draft/draft-content.ts @@ -76,3 +76,15 @@ export function sameRevisionContent( JSON.stringify(normalizeJson(left)) === JSON.stringify(normalizeJson(right)) ) } + +export function pickSnapshot(input: RevisionSnapshot): RevisionSnapshot { + return { + content: input.content, + contentFormat: input.contentFormat, + images: input.images, + meta: input.meta, + text: input.text, + title: input.title, + typeSpecificData: input.typeSpecificData, + } +} diff --git a/apps/core/src/modules/draft/draft-crdt.doc.ts b/apps/core/src/modules/draft/draft-crdt.doc.ts new file mode 100644 index 00000000000..2431f5970a1 --- /dev/null +++ b/apps/core/src/modules/draft/draft-crdt.doc.ts @@ -0,0 +1 @@ +export * from '@mx-space/editor/loro' diff --git a/apps/core/src/modules/draft/draft-crdt.repository.ts b/apps/core/src/modules/draft/draft-crdt.repository.ts new file mode 100644 index 00000000000..f28d6655cf0 --- /dev/null +++ b/apps/core/src/modules/draft/draft-crdt.repository.ts @@ -0,0 +1,242 @@ +import { Inject, Injectable } from '@nestjs/common' +import { and, asc, eq, inArray, sql } from 'drizzle-orm' + +import { PG_DB_TOKEN } from '~/constants/system.constant' +import { + contentCrdtDocs, + contentCrdtUpdates, + contentRevisions, + drafts, +} from '~/database/schema' +import { + BaseRepository, + toEntityId, +} from '~/processors/database/base.repository' +import type { AppDatabase } from '~/processors/database/postgres.provider' +import { type EntityId, parseEntityId } from '~/shared/id/entity-id' +import { SnowflakeService } from '~/shared/id/snowflake.service' + +import { mapBranch, mapRevision } from './draft.repository' +import type { + CheckpointInput, + ContentRevisionRow, + CrdtDocRow, + CrdtFields, + CrdtFieldsPatch, + DraftBranchRow, +} from './draft.types' + +const mapDoc = (row: typeof contentCrdtDocs.$inferSelect): CrdtDocRow => ({ + ...row, + documentId: toEntityId(row.documentId)!, + syncedRevisionId: toEntityId(row.syncedRevisionId), +}) + +export interface CreateCrdtDocInput { + documentId: EntityId | string + fields: CrdtFields | null + snapshot: Uint8Array + snapshotVersion: Uint8Array + syncedRevisionId: EntityId | string | null +} + +export interface CompactInput { + expectedVersion: Uint8Array + foldedIds: EntityId[] + snapshot: Uint8Array + snapshotVersion: Uint8Array +} + +@Injectable() +export class DraftCrdtRepository extends BaseRepository { + constructor( + @Inject(PG_DB_TOKEN) db: AppDatabase, + private readonly snowflake: SnowflakeService, + ) { + super(db) + } + + // Holds a transaction open only to own the advisory lock; `run` uses its own connections. + async withDocumentLock( + documentId: EntityId | string, + run: () => Promise, + ): Promise { + return this.db.transaction(async (tx) => { + await tx.execute( + sql`select pg_advisory_xact_lock(hashtext(${`crdt-edit:${parseEntityId(documentId)}`}))`, + ) + return run() + }) + } + + async findDoc(documentId: EntityId | string): Promise { + const [row] = await this.db + .select() + .from(contentCrdtDocs) + .where(eq(contentCrdtDocs.documentId, parseEntityId(documentId))) + .limit(1) + return row ? mapDoc(row) : null + } + + async createDoc(input: CreateCrdtDocInput): Promise { + await this.db + .insert(contentCrdtDocs) + .values({ + documentId: parseEntityId(input.documentId), + fields: input.fields, + snapshot: input.snapshot, + snapshotVersion: input.snapshotVersion, + syncedRevisionId: input.syncedRevisionId + ? parseEntityId(input.syncedRevisionId) + : null, + }) + .onConflictDoNothing({ target: contentCrdtDocs.documentId }) + return (await this.findDoc(input.documentId))! + } + + async appendUpdate( + documentId: EntityId | string, + bytes: Uint8Array, + ): Promise { + const id = this.snowflake.nextId() + await this.db.insert(contentCrdtUpdates).values({ + bytes, + documentId: parseEntityId(documentId), + id, + }) + return toEntityId(id)! + } + + async findUpdates( + documentId: EntityId | string, + ): Promise<{ bytes: Uint8Array; id: EntityId }[]> { + const rows = await this.db + .select({ bytes: contentCrdtUpdates.bytes, id: contentCrdtUpdates.id }) + .from(contentCrdtUpdates) + .where(eq(contentCrdtUpdates.documentId, parseEntityId(documentId))) + .orderBy(asc(contentCrdtUpdates.createdAt), asc(contentCrdtUpdates.id)) + return rows.map((row) => ({ bytes: row.bytes, id: toEntityId(row.id)! })) + } + + async compact( + documentId: EntityId | string, + input: CompactInput, + ): Promise { + const documentIdValue = parseEntityId(documentId) + return this.db.transaction(async (tx) => { + const lock = await tx.execute<{ locked: boolean }>( + sql`select pg_try_advisory_xact_lock(hashtext(${`crdt-compact:${documentIdValue}`})) as locked`, + ) + if (!lock.rows[0]?.locked) return false + const updated = await tx + .update(contentCrdtDocs) + .set({ + snapshot: input.snapshot, + snapshotVersion: input.snapshotVersion, + updatedAt: new Date(), + }) + .where( + and( + eq(contentCrdtDocs.documentId, documentIdValue), + sql`${contentCrdtDocs.snapshotVersion} = ${Buffer.from(input.expectedVersion)}`, + ), + ) + .returning({ documentId: contentCrdtDocs.documentId }) + if (!updated.length) return false + if (input.foldedIds.length) { + await tx.delete(contentCrdtUpdates).where( + and( + eq(contentCrdtUpdates.documentId, documentIdValue), + inArray( + contentCrdtUpdates.id, + input.foldedIds.map((id) => parseEntityId(id)), + ), + ), + ) + } + return true + }) + } + + async setSyncedRevision( + documentId: EntityId | string, + revisionId: EntityId | string, + ): Promise { + await this.db + .update(contentCrdtDocs) + .set({ + syncedRevisionId: parseEntityId(revisionId), + updatedAt: new Date(), + }) + .where(eq(contentCrdtDocs.documentId, parseEntityId(documentId))) + } + + async mergeFields( + documentId: EntityId | string, + patch: CrdtFieldsPatch, + ): Promise { + const merge = (key: keyof CrdtFields) => + patch[key] + ? sql`(case when jsonb_typeof(${contentCrdtDocs.fields}->${key}) = 'object' then ${contentCrdtDocs.fields}->${key} else '{}'::jsonb end) || ${JSON.stringify(patch[key])}::jsonb` + : sql`coalesce(${contentCrdtDocs.fields}->${key}, 'null'::jsonb)` + const [row] = await this.db + .update(contentCrdtDocs) + .set({ + fields: sql`jsonb_build_object('meta', ${merge('meta')}, 'typeSpecificData', ${merge('typeSpecificData')})`, + updatedAt: new Date(), + }) + .where(eq(contentCrdtDocs.documentId, parseEntityId(documentId))) + .returning({ fields: contentCrdtDocs.fields }) + return row?.fields ?? { meta: null, typeSpecificData: null } + } + + async replaceFields( + documentId: EntityId | string, + fields: CrdtFields, + ): Promise { + await this.db + .update(contentCrdtDocs) + .set({ fields, updatedAt: new Date() }) + .where(eq(contentCrdtDocs.documentId, parseEntityId(documentId))) + } + + async appendCheckpoints( + branchId: EntityId | string, + inputs: CheckpointInput[], + ): Promise<{ branch: DraftBranchRow; revisions: ContentRevisionRow[] }> { + return this.db.transaction(async (tx) => { + const branchIdValue = parseEntityId(branchId) + const [branch] = await tx + .select() + .from(drafts) + .where(eq(drafts.id, branchIdValue)) + .for('update') + .limit(1) + if (!branch) throw new Error('DRAFT_NOT_FOUND') + + let parentRevisionId = branch.headRevisionId + const revisions: ContentRevisionRow[] = [] + for (const input of inputs) { + const [revision] = await tx + .insert(contentRevisions) + .values({ + ...input.snapshot, + documentId: branch.documentId, + frontiers: input.frontiers, + id: this.snowflake.nextId(), + kind: input.kind, + parentRevisionId, + }) + .returning() + revisions.push(mapRevision(revision)) + parentRevisionId = revision.id + } + const [updated] = await tx + .update(drafts) + .set({ headRevisionId: parentRevisionId, updatedAt: new Date() }) + .where(eq(drafts.id, branchIdValue)) + .returning() + return { branch: mapBranch(updated), revisions } + }) + } +} diff --git a/apps/core/src/modules/draft/draft-crdt.service.ts b/apps/core/src/modules/draft/draft-crdt.service.ts new file mode 100644 index 00000000000..c3c8c45b269 --- /dev/null +++ b/apps/core/src/modules/draft/draft-crdt.service.ts @@ -0,0 +1,797 @@ +import { mxLexicalToMarkdown } from '@mx-space/editor' +import { Injectable, Logger } from '@nestjs/common' +import { type LoroDoc, type OpId, VersionVector } from 'loro-crdt' + +import { AppErrorCode, createAppException } from '~/common/errors' +import { FileReferenceType } from '~/modules/file/file-reference.enum' +import { FileReferenceService } from '~/modules/file/file-reference.service' +import type { EntityId } from '~/shared/id/entity-id' +import { ContentFormat } from '~/shared/types/content-format.type' + +import type { DraftRefType } from './draft.enum' +import { DraftRepository } from './draft.repository' +import type { + CheckpointInput, + CheckpointKind, + ContentDocumentRow, + ContentRevisionRow, + CrdtDocRow, + CrdtFields, + CrdtFieldsPatch, + DraftBranchRow, + RevisionSnapshot, +} from './draft.types' +import { + canonicalSnapshot, + pickSnapshot, + sameRevisionContent, +} from './draft-content' +import { + createContentDoc, + createDocument, + type DocContent, + editDocAt, + readDocContent, + revertChange, + vetUpdate, +} from './draft-crdt.doc' +import { DraftCrdtRepository } from './draft-crdt.repository' + +const COMPACT_THRESHOLD = 100 +const MAX_UPDATE_BYTES = 4 * 1024 * 1024 + +const EMPTY_STATE = JSON.stringify({ + root: { + children: [], + direction: null, + format: '', + indent: 0, + type: 'root', + version: 1, + }, +}) + +interface Session { + branch: DraftBranchRow + doc: LoroDoc + document: ContentDocumentRow + row: CrdtDocRow +} + +export interface CrdtChange { + deps: OpId[] + id: OpId + lamport: number + message: string | undefined + timestamp: number +} + +export type CrdtCheckpoint = Pick< + ContentRevisionRow, + | 'contentFormat' + | 'createdAt' + | 'frontiers' + | 'id' + | 'kind' + | 'parentRevisionId' + | 'title' +> + +export interface CrdtHistory { + changes: CrdtChange[] + checkpoints: CrdtCheckpoint[] +} + +export interface DocumentChange { + documentId: string + origin?: string + update: Uint8Array +} + +export interface CheckpointResult { + branch: DraftBranchRow + revision: ContentRevisionRow +} + +const fieldsOf = (snapshot: RevisionSnapshot): CrdtFields => ({ + meta: snapshot.meta, + typeSpecificData: snapshot.typeSpecificData, +}) + +const sameBytes = (left: Uint8Array, right: Uint8Array) => + left.length === right.length && left.every((byte, i) => byte === right[i]) + +const changedKeys = ( + base: Record | null, + next: Record | null, +): Record | undefined => { + const patch: Record = {} + for (const key of new Set([ + ...Object.keys(base ?? {}), + ...Object.keys(next ?? {}), + ])) { + const value = next?.[key] ?? null + if (JSON.stringify(base?.[key] ?? null) !== JSON.stringify(value)) { + patch[key] = value + } + } + return Object.keys(patch).length ? patch : undefined +} + +@Injectable() +export class DraftCrdtService { + private readonly logger = new Logger(DraftCrdtService.name) + private readonly changeListeners = new Set<(change: DocumentChange) => void>() + + constructor( + private readonly drafts: DraftRepository, + private readonly crdt: DraftCrdtRepository, + private readonly fileReferences: FileReferenceService, + ) {} + + onDocumentChanged(listener: (change: DocumentChange) => void) { + this.changeListeners.add(listener) + return () => { + this.changeListeners.delete(listener) + } + } + + async usesCrdt(branchId: EntityId | string): Promise { + const branch = await this.drafts.findBranchById(branchId) + if (!branch) return false + if (await this.crdt.findDoc(branch.documentId)) return true + const head = await this.drafts.findRevisionById(branch.headRevisionId) + return head?.contentFormat === ContentFormat.Lexical + } + + async workingCopy(branchId: EntityId | string): Promise { + return (await this.open(branchId)).branch + } + + async workingCopyOfDocument( + documentId: EntityId | string, + ): Promise { + const document = await this.drafts.findDocumentById(documentId) + if (!document) return null + if (!(await this.crdt.findDoc(document.id))) { + const [candidate] = await this.drafts.findBranchesByDocument(document.id) + const revisionId = + candidate?.headRevisionId ?? document.publishedRevisionId + if (!revisionId) return null + const head = await this.drafts.findRevisionById(revisionId) + if (head?.contentFormat !== ContentFormat.Lexical) return null + } + return (await this.openDocument(document)).branch + } + + async exportUpdates( + branchId: EntityId | string, + since?: Uint8Array, + ): Promise { + return (await this.exportFor(branchId, since)).update + } + + async exportFor( + branchId: EntityId | string, + since?: Uint8Array, + ): Promise<{ documentId: string; update: Uint8Array }> { + const { doc, document } = await this.open(branchId) + const documentId = String(document.id) + if (!since) return { documentId, update: doc.export({ mode: 'snapshot' }) } + let from: VersionVector + try { + from = VersionVector.decode(since) + } catch { + return { documentId, update: doc.export({ mode: 'snapshot' }) } + } + return { documentId, update: doc.export({ from, mode: 'update' }) } + } + + async importUpdate( + branchId: EntityId | string, + bytes: Uint8Array, + origin?: string, + author?: string, + ): Promise<{ branch: DraftBranchRow; document: ContentDocumentRow }> { + const session = await this.open(branchId) + if (bytes.length > MAX_UPDATE_BYTES) { + this.reject(session.branch, 'update is too large') + } + try { + vetUpdate(session.doc, bytes, { author }) + } catch (error) { + this.reject(session.branch, (error as Error).message) + } + await this.crdt.appendUpdate(session.document.id, bytes) + this.notifyChanged(session.document.id, bytes, origin) + return { branch: session.branch, document: session.document } + } + + async createCheckpoint( + branchId: EntityId | string, + kind: Extract, + ): Promise { + const session = await this.open(branchId) + const head = await this.revision(session.branch.headRevisionId) + const snapshot = this.materialize(session, readDocContent(session.doc)) + if (head.frontiers && sameRevisionContent(pickSnapshot(head), snapshot)) { + return { branch: session.branch, revision: head } + } + return this.checkpoint(session, kind, snapshot) + } + + async updateFields( + branchId: EntityId | string, + patch: CrdtFieldsPatch, + ): Promise { + const { document } = await this.open(branchId) + return this.crdt.mergeFields(document.id, patch) + } + + async mergeDocumentFields( + documentId: EntityId | string, + typeSpecificData: Record, + ): Promise { + if (!(await this.crdt.findDoc(documentId))) return + await this.crdt.mergeFields(documentId, { typeSpecificData }) + } + + async legacyWrite( + branchId: EntityId | string, + baseRevisionId: EntityId | string | null, + snapshot: RevisionSnapshot, + ): Promise { + return this.mutate(branchId, async (session) => { + if (snapshot.contentFormat !== ContentFormat.Lexical) { + throw createAppException(AppErrorCode.DRAFT_CRDT_FORMAT_UNSUPPORTED, { + branchId: session.branch.id, + contentFormat: snapshot.contentFormat, + }) + } + const base = baseRevisionId + ? await this.revision(baseRevisionId, session.document.id) + : null + const target: DocContent = { + content: snapshot.content ?? EMPTY_STATE, + title: snapshot.title, + } + const writeFields = fieldsOf(snapshot) + const message = `legacy-save ${base?.id ?? session.branch.headRevisionId}` + const at = this.knownFrontiers( + session.doc, + base ? (base.frontiers ?? (await this.importedFrontiers(base))) : null, + ) + + const bytes = editDocAt( + session.doc, + at ?? session.doc.frontiers(), + target, + message, + ) + let own: CheckpointInput + if (at) { + const fork = session.doc.forkAt(at) + fork.import(bytes) + own = { + frontiers: fork.frontiers(), + kind: 'legacy-save', + snapshot: this.materialize( + session, + readDocContent(fork), + writeFields, + ), + } + } else { + own = { + frontiers: null, + kind: 'legacy-save', + snapshot: this.materialize(session, target, writeFields), + } + } + + await this.apply(session, bytes) + const fields = await this.mergeChangedFields( + session, + base ? fieldsOf(base) : session.row.fields, + writeFields, + ) + const merged = this.materialize( + session, + readDocContent(session.doc), + fields, + ) + const inputs: CheckpointInput[] = sameRevisionContent( + own.snapshot, + merged, + ) + ? [ + { + frontiers: session.doc.frontiers(), + kind: 'legacy-save', + snapshot: merged, + }, + ] + : [ + own, + { + frontiers: session.doc.frontiers(), + kind: 'legacy-save', + snapshot: merged, + }, + ] + const { branch, revisions } = await this.appendCheckpoints( + session, + inputs, + ) + return { branch, revision: revisions[0] } + }) + } + + async restore( + branchId: EntityId | string, + revisionId: EntityId | string, + ): Promise { + return this.mutate(branchId, (session) => + this.restoreIn(session, revisionId), + ) + } + + async undo( + branchId: EntityId | string, + id: OpId, + before: OpId[], + ): Promise { + return this.mutate(branchId, async (session) => { + try { + session.doc.getChangeAt(id) + } catch { + this.reject(session.branch, 'unknown change') + } + await this.apply( + session, + revertChange(session.doc, id, before, `undo ${id.peer}@${id.counter}`), + ) + return this.checkpoint( + session, + 'undo', + this.materialize(session, readDocContent(session.doc)), + ) + }) + } + + async discard(branchId: EntityId | string): Promise { + return this.mutate(branchId, async (session) => { + if (session.document.publishedRevisionId) { + const { branch } = await this.restoreIn( + session, + session.document.publishedRevisionId, + ) + return branch + } + await this.drafts.archiveBranch(session.branch.id) + await this.fileReferences.removeReferencesForDocument( + session.branch.id, + FileReferenceType.Draft, + ) + return null + }) + } + + async history(branchId: EntityId | string): Promise { + const { doc, document } = await this.open(branchId) + const checkpoints = (await this.drafts.findRevisionsByDocument(document.id)) + .sort( + (left, right) => + right.createdAt.getTime() - left.createdAt.getTime() || + Number(BigInt(right.id) - BigInt(left.id)), + ) + .map((revision): CrdtCheckpoint => ({ + contentFormat: revision.contentFormat, + createdAt: revision.createdAt, + frontiers: revision.frontiers ?? null, + id: revision.id, + kind: revision.kind ?? null, + parentRevisionId: revision.parentRevisionId, + title: revision.title, + })) + const changes = [...doc.getAllChanges().values()] + .flat() + .sort((left, right) => right.lamport - left.lamport) + .map((change) => ({ + deps: change.deps, + id: { counter: change.counter + change.length - 1, peer: change.peer }, + lamport: change.lamport, + message: change.message, + timestamp: change.timestamp, + })) + return { changes, checkpoints } + } + + private async open( + branchId: EntityId | string, + locked = false, + ): Promise { + const requested = await this.drafts.findBranchById(branchId) + if (!requested) { + throw createAppException(AppErrorCode.DRAFT_NOT_FOUND, { + id: String(branchId), + }) + } + const document = await this.drafts.findDocumentById(requested.documentId) + if (!document) { + throw createAppException(AppErrorCode.DRAFT_NOT_FOUND, { + id: requested.id, + }) + } + return this.openDocument(document, locked) + } + + private async openDocument( + document: ContentDocumentRow, + locked = false, + ): Promise { + let row = await this.crdt.findDoc(document.id) + let branch = await this.ensureWorkingCopy(document, Boolean(row)) + if (!row) ({ branch, row } = await this.importDoc(document, branch)) + + let updates = await this.crdt.findUpdates(document.id) + // A compaction committed between the two reads has deleted rows our snapshot lacks. + for (;;) { + const current = await this.crdt.findDoc(document.id) + if (!current || sameBytes(current.snapshotVersion, row.snapshotVersion)) { + break + } + row = current + updates = await this.crdt.findUpdates(document.id) + } + const doc = createDocument() + doc.import(row.snapshot) + if (updates.length) doc.importBatch(updates.map((update) => update.bytes)) + const session: Session = { branch, doc, document, row } + + const head = await this.revision(branch.headRevisionId) + if (!head.frontiers && head.id !== row.syncedRevisionId) { + if (!locked) { + return this.crdt.withDocumentLock(document.id, () => + this.openDocument(document, true), + ) + } + await this.reconcile(session, head) + } + if (updates.length >= COMPACT_THRESHOLD) { + void this.compact( + session, + updates.map((update) => update.id), + ) + } + return session + } + + private mutate( + branchId: EntityId | string, + run: (session: Session) => Promise, + ): Promise { + return this.drafts.findBranchById(branchId).then((branch) => { + if (!branch) { + throw createAppException(AppErrorCode.DRAFT_NOT_FOUND, { + id: String(branchId), + }) + } + return this.crdt.withDocumentLock(branch.documentId, async () => + run(await this.open(branchId, true)), + ) + }) + } + + private async ensureWorkingCopy( + document: ContentDocumentRow, + imported: boolean, + ): Promise { + const active = await this.drafts.findBranchesByDocument(document.id) + if (active.length) { + const [keep, ...rest] = active + if (!imported) await this.assertLexical(keep) + for (const branch of rest) await this.retire(document, branch) + return keep + } + if (!document.publishedRevisionId) { + throw createAppException(AppErrorCode.DRAFT_NOT_FOUND, { + id: document.id, + }) + } + const published = await this.revision(document.publishedRevisionId) + const { branch } = await this.drafts.createBranch( + document.id, + published.id, + pickSnapshot(published), + true, + ) + if (!imported) await this.assertLexical(branch) + return branch + } + + private async assertLexical(branch: DraftBranchRow) { + const head = await this.revision(branch.headRevisionId) + if (head.contentFormat !== ContentFormat.Lexical) { + throw createAppException(AppErrorCode.DRAFT_CRDT_FORMAT_UNSUPPORTED, { + branchId: branch.id, + contentFormat: head.contentFormat, + }) + } + } + + private async retire(document: ContentDocumentRow, branch: DraftBranchRow) { + await this.drafts.archiveBranch(branch.id) + const share = await this.drafts.findShareByDocument(document.id) + if (share?.mode === 'follow' && share.draftId === branch.id) { + await this.drafts.upsertShare(document.id, share.token, { + mode: 'pinned', + revisionId: branch.headRevisionId, + }) + } + await this.fileReferences.removeReferencesForDocument( + branch.id, + FileReferenceType.Draft, + ) + } + + private async importDoc( + document: ContentDocumentRow, + branch: DraftBranchRow, + ): Promise<{ branch: DraftBranchRow; row: CrdtDocRow }> { + const head = await this.revision(branch.headRevisionId) + const doc = createContentDoc( + { content: head.content ?? EMPTY_STATE, title: head.title }, + `import ${head.id}`, + ) + const snapshot = doc.export({ mode: 'snapshot' }) + const fields = fieldsOf(head) + const row = await this.crdt.createDoc({ + documentId: document.id, + fields, + snapshot, + snapshotVersion: doc.oplogVersion().encode(), + syncedRevisionId: head.id, + }) + if (!sameBytes(row.snapshot, snapshot)) return { branch, row } + + const { branch: moved, revisions } = await this.crdt.appendCheckpoints( + branch.id, + [ + { + frontiers: doc.frontiers(), + kind: 'import', + snapshot: this.materializeFor( + document.refType, + readDocContent(doc), + fields, + ), + }, + ], + ) + await this.crdt.setSyncedRevision(document.id, revisions[0].id) + return { + branch: moved, + row: { ...row, syncedRevisionId: revisions[0].id }, + } + } + + private async reconcile(session: Session, head: ContentRevisionRow) { + if (head.contentFormat !== ContentFormat.Lexical) { + this.logger.warn( + `Skipping non-Lexical legacy head ${head.id} on CRDT document ${session.document.id}`, + ) + return + } + const synced = session.row.syncedRevisionId + ? await this.drafts.findRevisionById(session.row.syncedRevisionId) + : null + await this.apply( + session, + editDocAt( + session.doc, + this.knownFrontiers(session.doc, synced?.frontiers) ?? + session.doc.frontiers(), + { content: head.content ?? EMPTY_STATE, title: head.title }, + `legacy-save ${head.id}`, + ), + ) + const fields = await this.mergeChangedFields( + session, + synced ? fieldsOf(synced) : session.row.fields, + fieldsOf(head), + ) + const { branch } = await this.appendCheckpoints(session, [ + { + frontiers: session.doc.frontiers(), + kind: 'legacy-save', + snapshot: this.materialize( + session, + readDocContent(session.doc), + fields, + ), + }, + ]) + session.branch = branch + } + + private async restoreIn( + session: Session, + revisionId: EntityId | string, + ): Promise { + const revision = await this.revision(revisionId, session.document.id) + if (revision.contentFormat !== ContentFormat.Lexical) { + throw createAppException(AppErrorCode.DRAFT_CRDT_FORMAT_UNSUPPORTED, { + branchId: session.branch.id, + contentFormat: revision.contentFormat, + }) + } + await this.apply( + session, + editDocAt( + session.doc, + session.doc.frontiers(), + { content: revision.content ?? EMPTY_STATE, title: revision.title }, + `restore ${revision.id}`, + ), + ) + const fields = fieldsOf(revision) + await this.crdt.replaceFields(session.document.id, fields) + session.row = { ...session.row, fields } + return this.checkpoint( + session, + 'restore', + this.materialize(session, readDocContent(session.doc), fields), + ) + } + + private async apply(session: Session, bytes: Uint8Array) { + await this.crdt.appendUpdate(session.document.id, bytes) + session.doc.import(bytes) + this.notifyChanged(session.document.id, bytes) + } + + private notifyChanged( + documentId: EntityId | string, + update: Uint8Array, + origin?: string, + ) { + const change = { documentId: String(documentId), origin, update } + for (const listener of this.changeListeners) { + try { + listener(change) + } catch (error) { + this.logger.warn( + `document change listener failed: ${(error as Error).message}`, + ) + } + } + } + + private async checkpoint( + session: Session, + kind: CheckpointKind, + snapshot: RevisionSnapshot, + ): Promise { + const { branch, revisions } = await this.appendCheckpoints(session, [ + { frontiers: session.doc.frontiers(), kind, snapshot }, + ]) + return { branch, revision: revisions[0] } + } + + private async appendCheckpoints(session: Session, inputs: CheckpointInput[]) { + const result = await this.crdt.appendCheckpoints(session.branch.id, inputs) + const head = result.revisions.at(-1)! + await this.crdt.setSyncedRevision(session.document.id, head.id) + session.row = { ...session.row, syncedRevisionId: head.id } + await this.fileReferences.updateReferencesForDocument( + head, + result.branch.id, + FileReferenceType.Draft, + ) + return result + } + + private async mergeChangedFields( + session: Session, + base: CrdtFields | null, + next: CrdtFields, + ): Promise { + const patch: CrdtFieldsPatch = { + meta: changedKeys(base?.meta ?? null, next.meta), + typeSpecificData: changedKeys( + base?.typeSpecificData ?? null, + next.typeSpecificData, + ), + } + const current = session.row.fields ?? { meta: null, typeSpecificData: null } + if (!patch.meta && !patch.typeSpecificData) return current + const fields = await this.crdt.mergeFields(session.document.id, patch) + session.row = { ...session.row, fields } + return fields + } + + private async compact(session: Session, foldedIds: EntityId[]) { + try { + await this.crdt.compact(session.document.id, { + expectedVersion: session.row.snapshotVersion, + foldedIds, + snapshot: session.doc.export({ mode: 'snapshot' }), + snapshotVersion: session.doc.oplogVersion().encode(), + }) + } catch (error) { + this.logger.warn( + `Compaction of ${session.document.id} failed: ${(error as Error).message}`, + ) + } + } + + // A legacy revision has no version of its own, but the import checkpoint made from it does. + private async importedFrontiers(revision: ContentRevisionRow) { + const checkpoints = await this.drafts.findRevisionsByDocument( + revision.documentId, + ) + return ( + checkpoints.find( + (row) => row.kind === 'import' && row.parentRevisionId === revision.id, + )?.frontiers ?? null + ) + } + + private knownFrontiers( + doc: LoroDoc, + frontiers: CheckpointInput['frontiers'] | undefined, + ) { + if (!frontiers?.length) return null + try { + doc.frontiersToVV(frontiers) + return frontiers + } catch { + return null + } + } + + private materialize( + session: Session, + content: DocContent, + fields: CrdtFields | null = session.row.fields, + ): RevisionSnapshot { + return this.materializeFor(session.document.refType, content, fields) + } + + private materializeFor( + refType: DraftRefType, + content: DocContent, + fields: CrdtFields | null, + ): RevisionSnapshot { + return canonicalSnapshot(refType, { + content: content.content, + contentFormat: ContentFormat.Lexical, + images: null, + meta: fields?.meta ?? null, + text: mxLexicalToMarkdown(content.content), + title: content.title, + typeSpecificData: fields?.typeSpecificData ?? null, + }) + } + + private async revision( + id: EntityId | string, + documentId?: EntityId, + ): Promise { + const revision = await this.drafts.findRevisionById(id) + if (!revision || (documentId && revision.documentId !== documentId)) { + throw createAppException(AppErrorCode.CONTENT_REVISION_NOT_FOUND, { + id: String(id), + }) + } + return revision + } + + private reject(branch: DraftBranchRow, reason: string): never { + throw createAppException(AppErrorCode.DRAFT_CRDT_UPDATE_REJECTED, { + branchId: branch.id, + reason, + }) + } +} diff --git a/apps/core/src/modules/draft/draft.controller.ts b/apps/core/src/modules/draft/draft.controller.ts index 3f5f77cdf35..528bebfc6fa 100644 --- a/apps/core/src/modules/draft/draft.controller.ts +++ b/apps/core/src/modules/draft/draft.controller.ts @@ -1,7 +1,23 @@ -import { Body, Delete, Get, Param, Post, Put, Query } from '@nestjs/common' +import { + Body, + Delete, + Get, + HttpCode, + Param, + Patch, + Post, + Put, + Query, + Req, + Res, +} from '@nestjs/common' +import type { FastifyReply, FastifyRequest } from 'fastify' +import type { OpId } from 'loro-crdt' import { ApiController } from '~/common/decorators/api-controller.decorator' import { Auth } from '~/common/decorators/auth.decorator' +import { WithFastifyRouteOptions } from '~/common/decorators/fastify-route-options.decorator' +import { HTTPDecorators } from '~/common/decorators/http.decorator' import { AppErrorCode, createAppException } from '~/common/errors' import { withMeta } from '~/common/response/envelope.types' import { MetaObjectBuilder } from '~/common/response/meta-builder' @@ -11,16 +27,26 @@ import { DraftRefType } from './draft.enum' import { type CreateDraftDto, CreateDraftSchema, + type DraftCheckpointDto, + DraftCheckpointSchema, + type DraftCrdtQueryDto, + DraftCrdtQuerySchema, type DraftDocumentIdDto, DraftDocumentIdSchema, + type DraftFieldsDto, + DraftFieldsSchema, type DraftPagerDto, DraftPagerSchema, type DraftRefTypeAndIdDto, DraftRefTypeAndIdSchema, type DraftRefTypeDto, DraftRefTypeSchema, + type DraftRestoreDto, + DraftRestoreSchema, type DraftShareTokenDto, DraftShareTokenSchema, + type DraftUndoDto, + DraftUndoSchema, type RevisionComparisonDto, RevisionComparisonSchema, type SetDraftShareDto, @@ -31,6 +57,8 @@ import { import { DraftService } from './draft.service' import { DraftViews } from './draft.views' +const CRDT_UPDATE_BODY_LIMIT = 4 * 1024 * 1024 + @ApiController('drafts') export class DraftController { constructor(private readonly draftService: DraftService) {} @@ -131,7 +159,7 @@ export class DraftController { @Get('/:id') @Auth() async getById(@Param({ schema: EntityIdSchema }) params: EntityIdDto) { - const draft = await this.draftService.findById(params.id) + const draft = await this.draftService.findForEditing(params.id) if (!draft) { throw createAppException(AppErrorCode.DRAFT_NOT_FOUND, { id: params.id }) } @@ -144,6 +172,87 @@ export class DraftController { return this.draftService.getBranchRevisions(params.id) } + @Get('/:id/crdt') + @Auth() + @HTTPDecorators.RawResponse + async exportCrdt( + @Param({ schema: EntityIdSchema }) params: EntityIdDto, + @Query({ schema: DraftCrdtQuerySchema }) query: DraftCrdtQueryDto, + @Res({ passthrough: true }) reply: FastifyReply, + ) { + const bytes = await this.draftService.exportCrdt( + params.id, + query.since + ? new Uint8Array(Buffer.from(query.since, 'base64')) + : undefined, + ) + reply.type('application/octet-stream') + return Buffer.from(bytes) + } + + @Post('/:id/crdt') + @Auth() + @HttpCode(204) + @WithFastifyRouteOptions({ bodyLimit: CRDT_UPDATE_BODY_LIMIT }) + async importCrdt( + @Param({ schema: EntityIdSchema }) params: EntityIdDto, + @Req() request: FastifyRequest, + ) { + if (!Buffer.isBuffer(request.body)) { + throw createAppException(AppErrorCode.DRAFT_CRDT_UPDATE_REJECTED, { + branchId: params.id, + reason: 'expected an application/octet-stream body', + }) + } + await this.draftService.importCrdt(params.id, new Uint8Array(request.body)) + } + + @Post('/:id/checkpoints') + @Auth() + createCheckpoint( + @Param({ schema: EntityIdSchema }) params: EntityIdDto, + @Body({ schema: DraftCheckpointSchema }) body: DraftCheckpointDto, + ) { + return this.draftService.createCheckpoint(params.id, body.kind) + } + + @Patch('/:id/fields') + @Auth() + updateFields( + @Param({ schema: EntityIdSchema }) params: EntityIdDto, + @Body({ schema: DraftFieldsSchema }) body: DraftFieldsDto, + ) { + return this.draftService.updateFields(params.id, body) + } + + @Get('/:id/history') + @Auth() + history(@Param({ schema: EntityIdSchema }) params: EntityIdDto) { + return this.draftService.history(params.id) + } + + @Post('/:id/restore') + @Auth() + restore( + @Param({ schema: EntityIdSchema }) params: EntityIdDto, + @Body({ schema: DraftRestoreSchema }) body: DraftRestoreDto, + ) { + return this.draftService.restore(params.id, body.revisionId) + } + + @Post('/:id/undo') + @Auth() + undo( + @Param({ schema: EntityIdSchema }) params: EntityIdDto, + @Body({ schema: DraftUndoSchema }) body: DraftUndoDto, + ) { + return this.draftService.undo( + params.id, + body.id as OpId, + body.before as OpId[], + ) + } + @Put('/:id') @Auth() update( diff --git a/apps/core/src/modules/draft/draft.module.ts b/apps/core/src/modules/draft/draft.module.ts index 98a08e28e1f..7bdbf9ef986 100644 --- a/apps/core/src/modules/draft/draft.module.ts +++ b/apps/core/src/modules/draft/draft.module.ts @@ -2,17 +2,41 @@ import { Module } from '@nestjs/common' import { DRAFT_SERVICE_TOKEN } from '~/constants/injection.constant' +import { CollabController } from './collab.controller' +import { CollabRepository } from './collab.repository' +import { CollabService } from './collab.service' +import { CollabRoomService } from './collab-room.service' +import { + collabAccessProvider, + collabRelayProvider, + CollabRoomWsHandler, +} from './collab-room.ws' import { DraftController } from './draft.controller' import { DraftRepository } from './draft.repository' import { DraftService } from './draft.service' +import { DraftCrdtRepository } from './draft-crdt.repository' +import { DraftCrdtService } from './draft-crdt.service' @Module({ - controllers: [DraftController], + controllers: [DraftController, CollabController], providers: [ DraftRepository, + DraftCrdtRepository, + DraftCrdtService, DraftService, + CollabRepository, + CollabService, + CollabRoomService, + CollabRoomWsHandler, + collabRelayProvider, + collabAccessProvider, { provide: DRAFT_SERVICE_TOKEN, useExisting: DraftService }, ], - exports: [DraftService, DraftRepository, DRAFT_SERVICE_TOKEN], + exports: [ + CollabRepository, + DraftService, + DraftRepository, + DRAFT_SERVICE_TOKEN, + ], }) export class DraftModule {} diff --git a/apps/core/src/modules/draft/draft.repository.ts b/apps/core/src/modules/draft/draft.repository.ts index bfa00f9b82e..5da763831dc 100644 --- a/apps/core/src/modules/draft/draft.repository.ts +++ b/apps/core/src/modules/draft/draft.repository.ts @@ -30,6 +30,7 @@ import { type EntityId, parseEntityId } from '~/shared/id/entity-id' import { SnowflakeService } from '~/shared/id/snowflake.service' import type { + CheckpointKind, ContentDocumentRow, ContentPublicationEventRow, ContentRevisionRow, @@ -53,12 +54,13 @@ const mapDocument = ( refType: row.refType as DraftRefType, }) -const mapRevision = ( +export const mapRevision = ( row: typeof contentRevisions.$inferSelect, ): ContentRevisionRow => ({ ...row, documentId: toEntityId(row.documentId)!, id: toEntityId(row.id)!, + kind: row.kind as CheckpointKind | null, parentRevisionId: toEntityId(row.parentRevisionId), }) @@ -83,7 +85,7 @@ const mapShare = ( revisionId: toEntityId(row.revisionId), }) -const mapBranch = (row: typeof drafts.$inferSelect): DraftBranchRow => ({ +export const mapBranch = (row: typeof drafts.$inferSelect): DraftBranchRow => ({ ...row, baseRevisionId: toEntityId(row.baseRevisionId)!, documentId: toEntityId(row.documentId)!, diff --git a/apps/core/src/modules/draft/draft.schema.ts b/apps/core/src/modules/draft/draft.schema.ts index a27c7d74f14..ca48b17c934 100644 --- a/apps/core/src/modules/draft/draft.schema.ts +++ b/apps/core/src/modules/draft/draft.schema.ts @@ -96,3 +96,44 @@ export const SetDraftShareSchema = z.discriminatedUnion('mode', [ ]) export type SetDraftShareDto = z.infer + +export const DraftCrdtQuerySchema = z.object({ + since: z.string().max(65_536).optional(), +}) + +export type DraftCrdtQueryDto = z.infer + +export const DraftCheckpointSchema = z.object({ + kind: z.enum(['save', 'publish']), +}) + +export type DraftCheckpointDto = z.infer + +export const DraftFieldsSchema = z + .object({ + meta: z.record(z.string(), z.unknown()).optional(), + typeSpecificData: z.record(z.string(), z.unknown()).optional(), + }) + .refine((value) => value.meta || value.typeSpecificData, { + message: 'Nothing to update', + }) + +export type DraftFieldsDto = z.infer + +export const DraftRestoreSchema = z.object({ + revisionId: zEntityId, +}) + +export type DraftRestoreDto = z.infer + +const OpIdSchema = z.object({ + counter: z.number().int().nonnegative(), + peer: z.string().regex(/^\d+$/), +}) + +export const DraftUndoSchema = z.object({ + before: z.array(OpIdSchema).max(64), + id: OpIdSchema, +}) + +export type DraftUndoDto = z.infer diff --git a/apps/core/src/modules/draft/draft.service.ts b/apps/core/src/modules/draft/draft.service.ts index a47cb862937..dd332ef5cb0 100644 --- a/apps/core/src/modules/draft/draft.service.ts +++ b/apps/core/src/modules/draft/draft.service.ts @@ -1,6 +1,7 @@ import { randomBytes } from 'node:crypto' import { Injectable } from '@nestjs/common' +import type { OpId } from 'loro-crdt' import { AppErrorCode, createAppException } from '~/common/errors' import { BusinessEvents, EventScope } from '~/constants/business-event.constant' @@ -14,6 +15,7 @@ import { DraftRefType } from './draft.enum' import { DraftRepository } from './draft.repository' import type { CreateDraftDto, + DraftCheckpointDto, SetDraftShareDto, UpdateDraftDto, } from './draft.schema' @@ -21,6 +23,7 @@ import type { ContentDocumentRow, ContentPublicationEventRow, ContentRevisionRow, + CrdtFieldsPatch, DocumentShareRow, DraftBranchRow, DraftBranchView, @@ -33,6 +36,7 @@ import type { VersionTreeNode, } from './draft.types' import { canonicalSnapshot, sameRevisionContent } from './draft-content' +import { type CheckpointResult, DraftCrdtService } from './draft-crdt.service' const projectVersionTree = ( document: ContentDocumentRow, @@ -142,6 +146,7 @@ export class DraftService { private readonly fileReferenceService: FileReferenceService, private readonly databaseService: DatabaseService, private readonly eventManager: EventManagerService, + private readonly crdtService: DraftCrdtService, ) {} get repository() { @@ -167,6 +172,7 @@ export class DraftService { refId: string, ): Promise { const document = await this.ensurePublishedDocument(refType, refId) + await this.crdtService.workingCopyOfDocument(document.id) return this.contextForDocument(document) } @@ -229,6 +235,17 @@ export class DraftService { base = created.revision } + if (dto.baseRevisionId || dto.refId) { + const workingCopy = await this.crdtService.workingCopyOfDocument( + document.id, + ) + if (workingCopy) { + return this.legacyView( + await this.crdtService.legacyWrite(workingCopy.id, base.id, snapshot), + ) + } + } + const created = await this.draftRepository.createBranch( document.id, base.id, @@ -257,6 +274,15 @@ export class DraftService { throw createAppException(AppErrorCode.DRAFT_NOT_FOUND, { id }) } const snapshot = canonicalSnapshot(document.refType, dto.data) + if (await this.crdtService.usesCrdt(id)) { + return this.legacyView( + await this.crdtService.legacyWrite( + id, + dto.expectedHeadRevisionId, + snapshot, + ), + ) + } const result = await this.draftRepository.saveBranch( id, dto.expectedHeadRevisionId, @@ -288,24 +314,130 @@ export class DraftService { * as a failed request — the client still gets the new head in the response. */ private broadcastUpdate(view: DraftBranchView) { + this.broadcast(view, view.document) + } + + private broadcast(branch: DraftBranchRow, document: ContentDocumentRow) { void this.eventManager.emit( BusinessEvents.DRAFT_UPDATE, { - branchId: view.id, - documentId: view.documentId, - headRevisionId: view.headRevisionId, - refId: view.document.refId, - refType: view.document.refType, + branchId: branch.id, + documentId: branch.documentId, + headRevisionId: branch.headRevisionId, + refId: document.refId, + refType: document.refType, } satisfies DraftUpdateEventPayload, { scope: EventScope.TO_ADMIN }, ) } async findById(id: string): Promise { + if (await this.crdtService.usesCrdt(id)) { + return this.hydrate(await this.crdtService.workingCopy(id)) + } const branch = await this.draftRepository.findBranchById(id) return branch ? this.hydrate(branch) : null } + async mergeArticleFields( + refType: DraftRefType, + refId: string, + typeSpecificData: Record, + ): Promise { + if (!Object.keys(typeSpecificData).length) return + const document = await this.draftRepository.findDocumentByRef( + refType, + refId, + ) + if (document) { + await this.crdtService.mergeDocumentFields(document.id, typeSpecificData) + } + } + + async findForEditing(id: string): Promise { + if (await this.crdtService.usesCrdt(id)) { + return this.createCheckpoint(id, 'save') + } + return this.findById(id) + } + + usesCrdt(id: string): Promise { + return this.crdtService.usesCrdt(id) + } + + exportCrdt(id: string, since?: Uint8Array): Promise { + return this.crdtService.exportUpdates(id, since) + } + + async documentIdOf(id: string): Promise { + const branch = await this.draftRepository.findBranchById(id) + if (!branch) { + throw createAppException(AppErrorCode.DRAFT_NOT_FOUND, { id }) + } + return String(branch.documentId) + } + + exportCrdtFor(id: string, since?: Uint8Array) { + return this.crdtService.exportFor(id, since) + } + + async importCrdt( + id: string, + bytes: Uint8Array, + origin?: string, + author?: string, + ): Promise { + const { branch, document } = await this.crdtService.importUpdate( + id, + bytes, + origin, + author, + ) + this.broadcast(branch, document) + } + + async createCheckpoint( + id: string, + kind: DraftCheckpointDto['kind'], + ): Promise { + return this.checkpointView( + await this.crdtService.createCheckpoint(id, kind), + ) + } + + updateFields(id: string, patch: CrdtFieldsPatch) { + return this.crdtService.updateFields(id, patch) + } + + history(id: string) { + return this.crdtService.history(id) + } + + async restore(id: string, revisionId: string): Promise { + return this.checkpointView(await this.crdtService.restore(id, revisionId)) + } + + async undo(id: string, change: OpId, before: OpId[]) { + return this.checkpointView(await this.crdtService.undo(id, change, before)) + } + + private async checkpointView( + result: CheckpointResult, + ): Promise { + const view = await this.hydrate(result.branch) + this.broadcastUpdate(view) + return view + } + + private async legacyView(result: CheckpointResult): Promise { + const view = await this.checkpointView(result) + return { + ...view, + headRevision: result.revision, + headRevisionId: result.revision.id, + } + } + async findRevisionById(id: string): Promise { const revision = await this.draftRepository.findRevisionById(id) if (!revision) { @@ -460,6 +592,11 @@ export class DraftService { } async delete(id: string): Promise { + if (await this.crdtService.usesCrdt(id)) { + const branch = await this.crdtService.discard(id) + if (branch) this.broadcastUpdate(await this.hydrate(branch)) + return + } const branch = await this.draftRepository.archiveBranch(id) if (!branch) { throw createAppException(AppErrorCode.DRAFT_NOT_FOUND, { id }) diff --git a/apps/core/src/modules/draft/draft.types.ts b/apps/core/src/modules/draft/draft.types.ts index 997cf4cf240..365525b7e34 100644 --- a/apps/core/src/modules/draft/draft.types.ts +++ b/apps/core/src/modules/draft/draft.types.ts @@ -29,13 +29,45 @@ export interface ContentDocumentRow { updatedAt: Date | null } +export type Frontiers = { counter: number; peer: `${number}` }[] + +export type CheckpointKind = + 'save' | 'publish' | 'restore' | 'undo' | 'import' | 'legacy-save' + export interface ContentRevisionRow extends RevisionSnapshot { createdAt: Date documentId: EntityId + frontiers?: Frontiers | null id: EntityId + kind?: CheckpointKind | null parentRevisionId: EntityId | null } +export interface CrdtFields { + meta: Record | null + typeSpecificData: Record | null +} + +export type CrdtFieldsPatch = { + [K in keyof CrdtFields]?: Record +} + +export interface CrdtDocRow { + createdAt: Date + documentId: EntityId + fields: CrdtFields | null + snapshot: Uint8Array + snapshotVersion: Uint8Array + syncedRevisionId: EntityId | null + updatedAt: Date | null +} + +export interface CheckpointInput { + frontiers: Frontiers | null + kind: CheckpointKind + snapshot: RevisionSnapshot +} + export interface ContentPublicationEventRow { createdAt: Date documentId: EntityId diff --git a/apps/core/src/modules/note/note.controller.ts b/apps/core/src/modules/note/note.controller.ts index 1b64682defb..ae434cb042d 100644 --- a/apps/core/src/modules/note/note.controller.ts +++ b/apps/core/src/modules/note/note.controller.ts @@ -32,6 +32,8 @@ import { AiInsightsService } from '../ai/ai-insights/ai-insights.service' import { parseLanguageCode } from '../ai/ai-language.util' import { AiSummaryService } from '../ai/ai-summary/ai-summary.service' import { AiTtsQueryService } from '../ai/ai-tts/ai-tts-query.service' +import { CollabRepository } from '../draft/collab.repository' +import { DraftRefType } from '../draft/draft.enum' import { EnrichmentService } from '../enrichment/enrichment.service' import { type ListQueryDto, @@ -85,8 +87,17 @@ export class NoteController { private readonly lexicalService: LexicalService, private readonly enrichmentService: EnrichmentService, private readonly translationEntryService: TranslationEntryService, + private readonly collabRepository: CollabRepository, ) {} + private async coAuthorsOf(noteId: string) { + const byRef = await this.collabRepository.coAuthorsByRef( + DraftRefType.Note, + [noteId], + ) + return byRef.get(noteId) ?? [] + } + private toArticleTranslationInput(note: NoteModel): ArticleTranslationInput { return { id: String(note.id), @@ -285,7 +296,10 @@ export class NoteController { const { enrichments, ...noteData } = await this.enrichmentService.attachEnrichments(current) metaBuilder.enrichments(enrichments as Record) - return withMeta(noteData, metaBuilder.build()) + return withMeta( + { ...noteData, coAuthors: await this.coAuthorsOf(String(current.id)) }, + metaBuilder.build(), + ) } const [[prev], [next]] = await Promise.all([ @@ -327,6 +341,7 @@ export class NoteController { return withMeta( { ...noteData, + coAuthors: await this.coAuthorsOf(String(current.id)), next: this.toAdjacentNavNote(next), prev: this.toAdjacentNavNote(prev), }, @@ -592,9 +607,9 @@ export class NoteController { @Body({ schema: PartialNoteSchema }) body: PartialNoteDto, @Param({ schema: EntityIdSchema }) params: EntityIdDto, ) { - await this.noteService.updateById( + await this.noteService.patchById( params.id, - body as unknown as Partial, + body as Parameters[1], ) } diff --git a/apps/core/src/modules/note/note.service.ts b/apps/core/src/modules/note/note.service.ts index 189c816ac47..aea095def74 100644 --- a/apps/core/src/modules/note/note.service.ts +++ b/apps/core/src/modules/note/note.service.ts @@ -353,6 +353,23 @@ export class NoteService { return note } + public async patchById( + id: string, + patch: Partial< + Pick + >, + ) { + const updated = await this.updateById(id, patch) + await this.draftService.mergeArticleFields( + DraftRefType.Note, + id, + Object.fromEntries( + Object.entries(patch).filter(([, value]) => value !== undefined), + ), + ) + return updated + } + public async updateById( id: string, data: Partial & { diff --git a/apps/core/src/modules/post/post.controller.ts b/apps/core/src/modules/post/post.controller.ts index 76e58d38187..276b4a749c6 100644 --- a/apps/core/src/modules/post/post.controller.ts +++ b/apps/core/src/modules/post/post.controller.ts @@ -42,6 +42,8 @@ import { AiInsightsService } from '../ai/ai-insights/ai-insights.service' import { parseLanguageCode } from '../ai/ai-language.util' import { AiSummaryService } from '../ai/ai-summary/ai-summary.service' import { AiTtsQueryService } from '../ai/ai-tts/ai-tts-query.service' +import { CollabRepository } from '../draft/collab.repository' +import { DraftRefType } from '../draft/draft.enum' import { EnrichmentService } from '../enrichment/enrichment.service' import { SnippetService } from '../snippet/snippet.service' import { @@ -57,7 +59,6 @@ import { SetPostPublishStatusSchema, } from './post.schema' import { PostService } from './post.service' -import type { PostModel } from './post.types' import { PostMetaBuilder } from './post-meta-builder' import { readPaywallMeta } from './post-paywall.util' @@ -91,6 +92,7 @@ export class PostController { private readonly translationEntryService: TranslationEntryService, private readonly snippetService: SnippetService, private readonly entitlementService: EntitlementService, + private readonly collabRepository: CollabRepository, ) {} private async applyPaywall( @@ -551,7 +553,15 @@ export class PostController { if (paywall) metaBuilder.paywall(paywall) - return withMeta(postData, metaBuilder.build()) + const postId = String(postDocument.id) + const coAuthors = await this.collabRepository.coAuthorsByRef( + DraftRefType.Post, + [postId], + ) + return withMeta( + { ...postData, coAuthors: coAuthors.get(postId) ?? [] }, + metaBuilder.build(), + ) } @Patch('/:id') @@ -560,10 +570,7 @@ export class PostController { @Param({ schema: EntityIdSchema }) params: EntityIdDto, @Body({ schema: PartialPostSchema }) body: PartialPostDto, ) { - await this.postService.updateById( - params.id, - body as unknown as Partial, - ) + await this.postService.patchById(params.id, body) } @Delete('/:id') diff --git a/apps/core/src/modules/post/post.service.ts b/apps/core/src/modules/post/post.service.ts index 42c2ce82b8e..1730995e6b2 100644 --- a/apps/core/src/modules/post/post.service.ts +++ b/apps/core/src/modules/post/post.service.ts @@ -347,6 +347,19 @@ export class PostService implements OnApplicationBootstrap { return trackedPost } + async patchById( + id: string, + patch: { categoryId?: string; pinAt?: Date | null }, + ) { + await this.updateById(id, patch as Partial) + await this.draftService.mergeArticleFields(DraftRefType.Post, id, { + ...(patch.categoryId === undefined + ? {} + : { categoryId: patch.categoryId }), + ...(patch.pinAt === undefined ? {} : { pin: Boolean(patch.pinAt) }), + }) + } + async updateById( id: string, data: Partial & { diff --git a/apps/core/src/modules/publish/publish.service.ts b/apps/core/src/modules/publish/publish.service.ts index 787f5c498d3..47ab58d8ab4 100644 --- a/apps/core/src/modules/publish/publish.service.ts +++ b/apps/core/src/modules/publish/publish.service.ts @@ -79,7 +79,17 @@ export class PublishService implements OnModuleInit { id: dto.branchId, }) } - if (branch.headRevisionId !== dto.revisionId) { + const workingCopy = await this.drafts.usesCrdt(branch.id) + let revision = branch.headRevision + if (workingCopy) { + const selected = await this.drafts.findRevisionById(dto.revisionId) + if (!selected || selected.documentId !== branch.documentId) { + throw createAppException(AppErrorCode.CONTENT_REVISION_NOT_FOUND, { + id: dto.revisionId, + }) + } + revision = selected + } else if (branch.headRevisionId !== dto.revisionId) { throw createAppException(AppErrorCode.DRAFT_HEAD_CONFLICT, { actualHeadRevisionId: branch.headRevisionId, branchId: branch.id, @@ -95,6 +105,7 @@ export class PublishService implements OnModuleInit { }) } if ( + !workingCopy && branch.relationToPublished && ['descendant', 'diverged'].includes(branch.relationToPublished) && !dto.confirmDiverged @@ -114,12 +125,13 @@ export class PublishService implements OnModuleInit { let migration: PublishTaskPayload['migration'] if ( existing?.contentFormat === ContentFormat.Markdown && - branch.headRevision.contentFormat === ContentFormat.Lexical + revision.contentFormat === ContentFormat.Lexical ) { const result = await this.contentMigrations.dryRunMarkdownToLexical({ branchId: branch.id, profile: 'yohaku-v1', refId: existing.id, + revisionId: revision.id, refType: branch.document.refType, sourceText: existing.text ?? '', }) @@ -160,8 +172,8 @@ export class PublishService implements OnModuleInit { : 'republish', refId: branch.document.refId, refType: branch.document.refType, - revisionId: branch.headRevision.id, - snapshot: structuredClone(branch.headRevision), + revisionId: revision.id, + snapshot: structuredClone(revision), } return this.tasks.createTask({ diff --git a/apps/core/src/processors/gateway/admin/admin-ws-handler.registry.ts b/apps/core/src/processors/gateway/admin/admin-ws-handler.registry.ts index c6a12ae63d3..a0a3201da48 100644 --- a/apps/core/src/processors/gateway/admin/admin-ws-handler.registry.ts +++ b/apps/core/src/processors/gateway/admin/admin-ws-handler.registry.ts @@ -1,54 +1,14 @@ -import type { WsAckPayload } from '@mx-space/ws-client/protocol' -import { Injectable, Logger } from '@nestjs/common' +import { Injectable } from '@nestjs/common' -export interface AdminWsContext { - connectionId: string - send: (event: string, payload: unknown) => void -} +import { + type WsHandler, + type WsHandlerContext, + WsHandlerRegistry, +} from '../ws/ws-handler.registry' -export type AdminWsHandler = ( - context: AdminWsContext, - payload: unknown, -) => WsAckPayload | Promise +export type AdminWsContext = WsHandlerContext -@Injectable() -export class AdminWsHandlerRegistry { - private readonly logger = new Logger(AdminWsHandlerRegistry.name) - private readonly handlers = new Map() - private readonly disconnectListeners = new Set< - (connectionId: string) => void - >() - - register(event: string, handler: AdminWsHandler) { - this.handlers.set(event, handler) - return () => { - if (this.handlers.get(event) === handler) this.handlers.delete(event) - } - } - - onDisconnect(listener: (connectionId: string) => void) { - this.disconnectListeners.add(listener) - return () => { - this.disconnectListeners.delete(listener) - } - } +export type AdminWsHandler = WsHandler - async dispatch( - event: string, - context: AdminWsContext, - payload: unknown, - ): Promise { - const handler = this.handlers.get(event) - if (!handler) return { ok: false, code: 'UNSUPPORTED_EVENT' } - try { - return await handler(context, payload) - } catch (error) { - this.logger.warn(`${event} handler failed: ${(error as Error).message}`) - return { ok: false, code: 'INTERNAL_ERROR' } - } - } - - notifyDisconnect(connectionId: string) { - for (const listener of this.disconnectListeners) listener(connectionId) - } -} +@Injectable() +export class AdminWsHandlerRegistry extends WsHandlerRegistry {} diff --git a/apps/core/src/processors/gateway/admin/events.gateway.ts b/apps/core/src/processors/gateway/admin/events.gateway.ts index 2a760bf33fb..e7643a00ebc 100644 --- a/apps/core/src/processors/gateway/admin/events.gateway.ts +++ b/apps/core/src/processors/gateway/admin/events.gateway.ts @@ -34,6 +34,13 @@ import { AdminWsHandlerRegistry } from './admin-ws-handler.registry' const AI_TASK_ROOM_PREFIX = 'ai-task:' +const CollabEvents = { + join: 'collab.join', + leave: 'collab.leave', + presence: 'collab.presence', + update: 'collab.update', +} as const + const AuthGateway = createAuthGateway({ namespace: 'admin' }) @WebSocketGateway({ path: '/ws/admin', maxPayload: 1024 * 1024 }) @@ -97,6 +104,38 @@ export class AdminEventsGateway return this.dispatchExtension(AgentWsEvents.steer, envelope, ws) } + @SubscribeMessage(CollabEvents.join) + handleCollabJoin( + @MessageBody() envelope: WsEnvelope, + @ConnectedSocket() ws: WebSocket, + ) { + return this.dispatchExtension(CollabEvents.join, envelope, ws) + } + + @SubscribeMessage(CollabEvents.update) + handleCollabUpdate( + @MessageBody() envelope: WsEnvelope, + @ConnectedSocket() ws: WebSocket, + ) { + return this.dispatchExtension(CollabEvents.update, envelope, ws) + } + + @SubscribeMessage(CollabEvents.presence) + handleCollabPresence( + @MessageBody() envelope: WsEnvelope, + @ConnectedSocket() ws: WebSocket, + ) { + return this.dispatchExtension(CollabEvents.presence, envelope, ws) + } + + @SubscribeMessage(CollabEvents.leave) + handleCollabLeave( + @MessageBody() envelope: WsEnvelope, + @ConnectedSocket() ws: WebSocket, + ) { + return this.dispatchExtension(CollabEvents.leave, envelope, ws) + } + private async dispatchExtension( event: string, envelope: WsEnvelope, @@ -104,10 +143,22 @@ export class AdminEventsGateway ) { const conn = this.resolveConnection(ws) if (!conn) return ack(envelope, { ok: false, code: 'NOT_CONNECTED' }) + if (!this.isAuthenticated(conn)) { + return ack(envelope, { ok: false, code: 'NOT_AUTHENTICATED' }) + } const result = await this.handlers.dispatch( event, { + alive: () => this.resolveConnection(ws) === conn, connectionId: conn.id, + join: async (room) => { + this.roomManager.join(room, conn) + await this.presence.joinRoom('admin', room, conn.id) + }, + leave: async (room) => { + this.roomManager.leave(room, conn) + await this.presence.leaveRoom('admin', room, conn.id) + }, send: (name, payload) => { if (conn.ws.readyState !== conn.ws.OPEN) return conn.ws.send(serializeEnvelope(buildEnvelope(name, payload))) @@ -125,7 +176,7 @@ export class AdminEventsGateway ) { const conn = this.resolveConnection(ws) const parsed = aiAgentPayloadSchema.safeParse(envelope.payload) - if (!conn || !parsed.success) { + if (!conn || !this.isAuthenticated(conn) || !parsed.success) { return ack(envelope, { ok: false, code: 'VALIDATION_FAILED' }) } @@ -142,7 +193,7 @@ export class AdminEventsGateway ) { const conn = this.resolveConnection(ws) const parsed = aiAgentPayloadSchema.safeParse(envelope.payload) - if (!conn || !parsed.success) { + if (!conn || !this.isAuthenticated(conn) || !parsed.success) { return ack(envelope, { ok: false, code: 'VALIDATION_FAILED' }) } @@ -159,7 +210,7 @@ export class AdminEventsGateway ) { const conn = this.resolveConnection(ws) const parsed = aiTaskPayloadSchema.safeParse(envelope.payload) - if (!conn || !parsed.success) { + if (!conn || !this.isAuthenticated(conn) || !parsed.success) { return ack(envelope, { ok: false, code: 'VALIDATION_FAILED' }) } @@ -181,7 +232,7 @@ export class AdminEventsGateway ) { const conn = this.resolveConnection(ws) const parsed = aiTaskPayloadSchema.safeParse(envelope.payload) - if (!conn || !parsed.success) { + if (!conn || !this.isAuthenticated(conn) || !parsed.success) { return ack(envelope, { ok: false, code: 'VALIDATION_FAILED' }) } diff --git a/apps/core/src/processors/gateway/gateway.module.ts b/apps/core/src/processors/gateway/gateway.module.ts index 5aa9902af36..7e9dcaabb62 100644 --- a/apps/core/src/processors/gateway/gateway.module.ts +++ b/apps/core/src/processors/gateway/gateway.module.ts @@ -14,6 +14,7 @@ import { GatewayService } from './gateway.service' import { SharedGateway } from './shared/events.gateway' import { WebEventsGateway } from './web/events.gateway' import { VisitorEventDispatchService } from './web/visitor-event-dispatch.service' +import { WebWsHandlerRegistry } from './web/web-ws-handler.registry' import { WsBusService } from './ws/ws-bus.service' import { WsPresenceService } from './ws/ws-presence.service' @@ -24,6 +25,7 @@ import { WsPresenceService } from './ws/ws-presence.service' AdminEventsGateway, AdminWsHandlerRegistry, WebEventsGateway, + WebWsHandlerRegistry, SharedGateway, GatewayService, @@ -37,6 +39,7 @@ import { WsPresenceService } from './ws/ws-presence.service' AdminEventsGateway, AdminWsHandlerRegistry, WebEventsGateway, + WebWsHandlerRegistry, SharedGateway, GatewayService, diff --git a/apps/core/src/processors/gateway/shared/auth.gateway.ts b/apps/core/src/processors/gateway/shared/auth.gateway.ts index 6864860917d..30e57f908d8 100644 --- a/apps/core/src/processors/gateway/shared/auth.gateway.ts +++ b/apps/core/src/processors/gateway/shared/auth.gateway.ts @@ -31,6 +31,13 @@ export const createAuthGateway = (options: AuthGatewayOptions) => { } private roleGeneration = 0 + private readonly authenticated = new WeakSet() + + // Message handlers run as soon as the socket is tracked, before the auth lookup settles, + // and keep running while a rejected socket is closing. + protected isAuthenticated(conn: WsConnection) { + return this.authenticated.has(conn) + } tokenSocketIdMap = new Map>() private readonly socketIdTokenMap = new Map() @@ -51,6 +58,7 @@ export const createAuthGateway = (options: AuthGatewayOptions) => { } authFailed(conn: WsConnection) { + this.authenticated.delete(conn) this.sendTo(conn, BusinessEvents.AUTH_FAILED, 'Authentication failed') conn.ws.close(AUTH_FAILED_CLOSE_CODE, 'auth failed') } @@ -108,6 +116,7 @@ export const createAuthGateway = (options: AuthGatewayOptions) => { ) { if (!this.resolveConnection(conn.ws)) return if (generation !== this.roleGeneration) return this.authFailed(conn) + this.authenticated.add(conn) this.sendConnectGreeting(conn) if (token) this.bindToken(token, conn) } diff --git a/apps/core/src/processors/gateway/web/events.gateway.ts b/apps/core/src/processors/gateway/web/events.gateway.ts index 192260460c9..d1193687541 100644 --- a/apps/core/src/processors/gateway/web/events.gateway.ts +++ b/apps/core/src/processors/gateway/web/events.gateway.ts @@ -1,6 +1,10 @@ import type { IncomingMessage } from 'node:http' -import type { WsEnvelope } from '@mx-space/ws-client/protocol' +import { + buildEnvelope, + type WsAckPayload, + type WsEnvelope, +} from '@mx-space/ws-client/protocol' import { Logger } from '@nestjs/common' import type { OnGatewayConnection, @@ -27,7 +31,7 @@ import type { SocketLike } from '../gateway.service' import { GatewayService } from '../gateway.service' import type { WsConnection } from '../ws/ws.types' import { WsBusService } from '../ws/ws-bus.service' -import { buildAck } from '../ws/ws-envelope' +import { buildAck, serializeEnvelope } from '../ws/ws-envelope' import { LANG_PATTERN, langUpdatePayloadSchema, @@ -38,6 +42,7 @@ import { import { WsGatewayBase } from '../ws/ws-gateway.base' import { WsPresenceService } from '../ws/ws-presence.service' import type { EventGatewayHooks } from './hook.interface' +import { WebWsHandlerRegistry } from './web-ws-handler.registry' declare module '~/types/socket-meta' { interface SocketMetadata { @@ -52,6 +57,15 @@ declare module '~/types/socket-meta' { const langRoom = (lang: string) => `lang:${lang}` +const COLLAB_ROOM_PREFIX = 'collab:' + +const CollabEvents = { + join: 'collab.join', + leave: 'collab.leave', + presence: 'collab.presence', + update: 'collab.update', +} as const + // ws defaults maxPayload to 100 MiB; cap at the ~1 MiB the socket.io // transport enforced so unauthenticated clients cannot buffer-bomb the node. @WebSocketGateway({ path: '/ws/web', maxPayload: 1024 * 1024 }) @@ -60,6 +74,10 @@ export class WebEventsGateway implements OnGatewayConnection, OnGatewayDisconnect { private readonly logger = new Logger(WebEventsGateway.name) + private readonly readerIds = new WeakMap< + WsConnection, + Promise + >() constructor( private readonly redisService: RedisService, @@ -68,6 +86,8 @@ export class WebEventsGateway private readonly authService: AuthService, + private readonly handlers: WebWsHandlerRegistry, + bus: WsBusService, presence: WsPresenceService, ) { @@ -113,6 +133,8 @@ export class WebEventsGateway async handleConnection(ws: WebSocket, request: IncomingMessage) { const conn = this.trackConnection(ws) const query = parseQuery(request) + const resolvingReader = this.resolveReaderId(request) + this.readerIds.set(conn, resolvingReader) const sessionId = query.get('socket_session_id') || conn.id const rawLang = query.get('lang') @@ -130,7 +152,7 @@ export class WebEventsGateway await this.presence.joinRoom('web', langRoom(lang), conn.id) } - const readerId = await this.resolveReaderId(request) + const readerId = await resolvingReader // A close anywhere above already ran the disconnect cleanup, which cannot // see writes this handler issues afterwards. Undo them or they outlive the @@ -160,6 +182,7 @@ export class WebEventsGateway const conn = this.resolveConnection(ws) if (!conn) return + this.handlers.notifyDisconnect(conn.id) const meta = await this.gatewayService.getSocketMetadata(conn) const leftRooms = await this.releaseConnection(conn) @@ -194,6 +217,9 @@ export class WebEventsGateway } const room = parsed.data.room + if (room.startsWith(COLLAB_ROOM_PREFIX)) { + return ack(envelope, { ok: false, code: 'ROOM_INVALID' }) + } this.roomManager.join(room, conn) await this.presence.joinRoom('web', room, conn.id) this.logger.log(`Connection ${conn.id} joined room [${room}]`) @@ -268,6 +294,69 @@ export class WebEventsGateway return ack(envelope, { ok: true }) } + @SubscribeMessage(CollabEvents.join) + handleCollabJoin( + @MessageBody() envelope: WsEnvelope, + @ConnectedSocket() ws: WebSocket, + ) { + return this.dispatchExtension(CollabEvents.join, envelope, ws) + } + + @SubscribeMessage(CollabEvents.update) + handleCollabUpdate( + @MessageBody() envelope: WsEnvelope, + @ConnectedSocket() ws: WebSocket, + ) { + return this.dispatchExtension(CollabEvents.update, envelope, ws) + } + + @SubscribeMessage(CollabEvents.presence) + handleCollabPresence( + @MessageBody() envelope: WsEnvelope, + @ConnectedSocket() ws: WebSocket, + ) { + return this.dispatchExtension(CollabEvents.presence, envelope, ws) + } + + @SubscribeMessage(CollabEvents.leave) + handleCollabLeave( + @MessageBody() envelope: WsEnvelope, + @ConnectedSocket() ws: WebSocket, + ) { + return this.dispatchExtension(CollabEvents.leave, envelope, ws) + } + + private async dispatchExtension( + event: string, + envelope: WsEnvelope, + ws: WebSocket, + ) { + const conn = this.resolveConnection(ws) + if (!conn) return ack(envelope, { ok: false, code: 'NOT_CONNECTED' }) + const result: WsAckPayload = await this.handlers.dispatch( + event, + { + alive: () => this.resolveConnection(ws) === conn, + connectionId: conn.id, + join: async (room) => { + this.roomManager.join(room, conn) + await this.presence.joinRoom('web', room, conn.id) + }, + leave: async (room) => { + this.roomManager.leave(room, conn) + await this.presence.leaveRoom('web', room, conn.id) + }, + readerId: await this.readerIds.get(conn), + send: (name, payload) => { + if (conn.ws.readyState !== conn.ws.OPEN) return + conn.ws.send(serializeEnvelope(buildEnvelope(name, payload))) + }, + }, + envelope.payload, + ) + return ack(envelope, result) + } + private notifyMessageHooks(conn: WsConnection, envelope: WsEnvelope) { this.hooks.onMessage.forEach((fn) => fn(conn, envelope)) } @@ -369,7 +458,7 @@ export class WebEventsGateway } } -function ack(envelope: WsEnvelope, payload: { ok: boolean; code?: string }) { +function ack(envelope: WsEnvelope, payload: WsAckPayload) { return envelope.id ? buildAck(envelope.id, payload) : undefined } diff --git a/apps/core/src/processors/gateway/web/web-ws-handler.registry.ts b/apps/core/src/processors/gateway/web/web-ws-handler.registry.ts new file mode 100644 index 00000000000..7e1494de2e3 --- /dev/null +++ b/apps/core/src/processors/gateway/web/web-ws-handler.registry.ts @@ -0,0 +1,16 @@ +import { Injectable } from '@nestjs/common' + +import { + type WsHandler, + type WsHandlerContext, + WsHandlerRegistry, +} from '../ws/ws-handler.registry' + +export interface WebWsContext extends WsHandlerContext { + readerId: string | undefined +} + +export type WebWsHandler = WsHandler + +@Injectable() +export class WebWsHandlerRegistry extends WsHandlerRegistry {} diff --git a/apps/core/src/processors/gateway/ws/ws-bus.service.ts b/apps/core/src/processors/gateway/ws/ws-bus.service.ts index 8565103cc86..c1e140d86ea 100644 --- a/apps/core/src/processors/gateway/ws/ws-bus.service.ts +++ b/apps/core/src/processors/gateway/ws/ws-bus.service.ts @@ -10,14 +10,14 @@ import { RedisKeys } from '~/constants/cache.constant' import { getRedisKey } from '~/utils/redis.util' import { RedisService } from '../../redis/redis.service' -import type { WsBusFrame, WsNamespace } from './ws.types' +import type { WsBusChannel, WsBusFrame } from './ws.types' @Injectable() export class WsBusService implements OnModuleInit, OnModuleDestroy { private readonly logger = new Logger(WsBusService.name) private readonly channel = getRedisKey(RedisKeys.WsBus) private readonly handlers = new Map< - WsNamespace, + WsBusChannel, (frame: WsBusFrame) => void >() @@ -68,7 +68,7 @@ export class WsBusService implements OnModuleInit, OnModuleDestroy { await subClient.quit().catch(() => undefined) } - register(ns: WsNamespace, deliver: (frame: WsBusFrame) => void): () => void { + register(ns: WsBusChannel, deliver: (frame: WsBusFrame) => void): () => void { this.handlers.set(ns, deliver) return () => { if (this.handlers.get(ns) === deliver) this.handlers.delete(ns) diff --git a/apps/core/src/processors/gateway/ws/ws-handler.registry.ts b/apps/core/src/processors/gateway/ws/ws-handler.registry.ts new file mode 100644 index 00000000000..c3198bfb40d --- /dev/null +++ b/apps/core/src/processors/gateway/ws/ws-handler.registry.ts @@ -0,0 +1,56 @@ +import type { WsAckPayload } from '@mx-space/ws-client/protocol' +import { Logger } from '@nestjs/common' + +export interface WsHandlerContext { + alive: () => boolean + connectionId: string + join: (room: string) => Promise + leave: (room: string) => Promise + send: (event: string, payload: unknown) => void +} + +export type WsHandler = ( + context: C, + payload: unknown, +) => WsAckPayload | Promise + +export class WsHandlerRegistry { + private readonly logger = new Logger(this.constructor.name) + private readonly handlers = new Map>() + private readonly disconnectListeners = new Set< + (connectionId: string) => void + >() + + register(event: string, handler: WsHandler) { + this.handlers.set(event, handler) + return () => { + if (this.handlers.get(event) === handler) this.handlers.delete(event) + } + } + + onDisconnect(listener: (connectionId: string) => void) { + this.disconnectListeners.add(listener) + return () => { + this.disconnectListeners.delete(listener) + } + } + + async dispatch( + event: string, + context: C, + payload: unknown, + ): Promise { + const handler = this.handlers.get(event) + if (!handler) return { ok: false, code: 'UNSUPPORTED_EVENT' } + try { + return await handler(context, payload) + } catch (error) { + this.logger.warn(`${event} handler failed: ${(error as Error).message}`) + return { ok: false, code: 'INTERNAL_ERROR' } + } + } + + notifyDisconnect(connectionId: string) { + for (const listener of this.disconnectListeners) listener(connectionId) + } +} diff --git a/apps/core/src/processors/gateway/ws/ws.types.ts b/apps/core/src/processors/gateway/ws/ws.types.ts index b08b589c172..72bbd321382 100644 --- a/apps/core/src/processors/gateway/ws/ws.types.ts +++ b/apps/core/src/processors/gateway/ws/ws.types.ts @@ -2,13 +2,15 @@ import type { WebSocket } from 'ws' export type WsNamespace = 'web' | 'admin' +export type WsBusChannel = WsNamespace | 'collab' + export interface WsConnection { id: string ws: WebSocket } export interface WsBusFrame { - ns: WsNamespace + ns: WsBusChannel event: string payload?: unknown rooms?: string[] diff --git a/apps/core/test/mock/modules/collab.mock.ts b/apps/core/test/mock/modules/collab.mock.ts new file mode 100644 index 00000000000..d6978b2a465 --- /dev/null +++ b/apps/core/test/mock/modules/collab.mock.ts @@ -0,0 +1,19 @@ +import { defineProvider } from 'test/helper/defineProvider' + +import type { CoAuthor } from '~/modules/draft/collab.repository' +import { CollabRepository } from '~/modules/draft/collab.repository' + +export const collabCoAuthors = new Map() + +export const collabRepositoryProvider = defineProvider({ + provide: CollabRepository, + useValue: { + async coAuthorsByRef(_refType: string, refIds: string[]) { + return new Map( + refIds + .filter((id) => collabCoAuthors.has(String(id))) + .map((id) => [String(id), collabCoAuthors.get(String(id))!]), + ) + }, + }, +}) diff --git a/apps/core/test/src/contracts/admin/drafts-admin.contract.spec.ts b/apps/core/test/src/contracts/admin/drafts-admin.contract.spec.ts index 39378ef54d0..d7067139a8d 100644 --- a/apps/core/test/src/contracts/admin/drafts-admin.contract.spec.ts +++ b/apps/core/test/src/contracts/admin/drafts-admin.contract.spec.ts @@ -46,6 +46,7 @@ describe('DraftController Admin branch contract (e2e)', () => { provide: DraftService, useValue: { findById: async () => fixture, + findForEditing: async () => fixture, list: async () => ({ data: [fixture], pagination: { currentPage: 1, size: 10, total: 1, totalPage: 1 }, diff --git a/apps/core/test/src/contracts/admin/drafts-crdt-admin.contract.spec.ts b/apps/core/test/src/contracts/admin/drafts-crdt-admin.contract.spec.ts new file mode 100644 index 00000000000..0f80aa0fd42 --- /dev/null +++ b/apps/core/test/src/contracts/admin/drafts-crdt-admin.contract.spec.ts @@ -0,0 +1,146 @@ +import { describe, expect, test, vi } from 'vitest' + +import { apiRoutePrefix } from '~/common/decorators/api-controller.decorator' +import { DraftController } from '~/modules/draft/draft.controller' +import { DraftService } from '~/modules/draft/draft.service' + +import { createE2EApp } from '../../../helper/create-e2e-app' +import { authPassHeader } from '../../../mock/guard/auth.guard' + +const id = '7000000000000000010' +const view = { headRevisionId: '7000000000000000011', id } + +const service = { + createCheckpoint: vi.fn(async () => view), + exportCrdt: vi.fn(async () => new Uint8Array([1, 2, 3])), + history: vi.fn(async () => ({ + changes: [ + { + deps: [], + id: { counter: 4, peer: '99' }, + lamport: 4, + message: 'human:owner', + timestamp: 1, + }, + ], + checkpoints: [{ id: view.headRevisionId, kind: 'save' }], + })), + importCrdt: vi.fn(async () => undefined), + restore: vi.fn(async () => view), + undo: vi.fn(async () => view), + updateFields: vi.fn(async () => ({ + meta: null, + typeSpecificData: { categoryId: 'c' }, + })), +} + +describe('DraftController CRDT endpoints (e2e)', () => { + const proxy = createE2EApp({ + controllers: [DraftController], + providers: [{ provide: DraftService, useValue: service }], + }) + + test('GET /drafts/:id/crdt returns raw bytes and decodes ?since', async () => { + const since = Buffer.from([7, 8]).toString('base64') + const res = await proxy.app.inject({ + headers: authPassHeader, + method: 'GET', + url: `${apiRoutePrefix}/drafts/${id}/crdt?since=${encodeURIComponent(since)}`, + }) + + expect(res.statusCode).toBe(200) + expect(res.headers['content-type']).toContain('application/octet-stream') + expect([...res.rawPayload]).toEqual([1, 2, 3]) + expect([ + ...((service.exportCrdt.mock.calls[0] as unknown[])[1] as Uint8Array), + ]).toEqual([7, 8]) + }) + + test('POST /drafts/:id/crdt imports an octet-stream body', async () => { + const res = await proxy.app.inject({ + headers: { + ...authPassHeader, + 'content-type': 'application/octet-stream', + }, + method: 'POST', + payload: Buffer.from([5, 6]), + url: `${apiRoutePrefix}/drafts/${id}/crdt`, + }) + + expect(res.statusCode).toBe(204) + const [calledId, bytes] = service.importCrdt.mock.calls[0] as unknown as [ + string, + Uint8Array, + ] + expect(calledId).toBe(id) + expect([...bytes]).toEqual([5, 6]) + }) + + test('POST /drafts/:id/checkpoints creates a checkpoint of a kind', async () => { + const res = await proxy.app.inject({ + headers: authPassHeader, + method: 'POST', + payload: { kind: 'publish' }, + url: `${apiRoutePrefix}/drafts/${id}/checkpoints`, + }) + + expect(res.statusCode).toBe(201) + expect(res.json().data).toMatchObject({ + head_revision_id: view.headRevisionId, + }) + expect(service.createCheckpoint).toHaveBeenCalledWith(id, 'publish') + }) + + test('PATCH /drafts/:id/fields keeps nested keys verbatim', async () => { + const res = await proxy.app.inject({ + headers: authPassHeader, + method: 'PATCH', + payload: { typeSpecificData: { categoryId: 'c' } }, + url: `${apiRoutePrefix}/drafts/${id}/fields`, + }) + + expect(res.statusCode).toBe(200) + expect(service.updateFields).toHaveBeenCalledWith(id, { + typeSpecificData: { categoryId: 'c' }, + }) + }) + + test('GET /drafts/:id/history lists checkpoints and changes', async () => { + const res = await proxy.app.inject({ + headers: authPassHeader, + method: 'GET', + url: `${apiRoutePrefix}/drafts/${id}/history`, + }) + + expect(res.statusCode).toBe(200) + expect(res.json().data.changes[0]).toMatchObject({ + id: { counter: 4, peer: '99' }, + message: 'human:owner', + }) + }) + + test('POST /drafts/:id/restore and /undo apply as new commits', async () => { + const restore = await proxy.app.inject({ + headers: authPassHeader, + method: 'POST', + payload: { revisionId: view.headRevisionId }, + url: `${apiRoutePrefix}/drafts/${id}/restore`, + }) + const undo = await proxy.app.inject({ + headers: authPassHeader, + method: 'POST', + payload: { + before: [{ counter: 1, peer: '99' }], + id: { counter: 4, peer: '99' }, + }, + url: `${apiRoutePrefix}/drafts/${id}/undo`, + }) + + expect(restore.statusCode).toBe(201) + expect(undo.statusCode).toBe(201) + expect(service.restore).toHaveBeenCalledWith(id, view.headRevisionId) + expect(service.undo).toHaveBeenCalledWith(id, { counter: 4, peer: '99' }, [ + { counter: 1, peer: '99' }, + ]) + }) +}) diff --git a/apps/core/test/src/contracts/admin/notes-admin.contract.spec.ts b/apps/core/test/src/contracts/admin/notes-admin.contract.spec.ts index 31e43ac3356..75aa360c1fa 100644 --- a/apps/core/test/src/contracts/admin/notes-admin.contract.spec.ts +++ b/apps/core/test/src/contracts/admin/notes-admin.contract.spec.ts @@ -29,6 +29,7 @@ import { } from '../../../helper/api-shape' import { createE2EApp } from '../../../helper/create-e2e-app' import { authPassHeader } from '../../../mock/guard/auth.guard' +import { collabRepositoryProvider } from '../../../mock/modules/collab.mock' import { enrichmentProvider } from '../../../mock/modules/enrichment.mock' import { countingServiceProvider } from '../../../mock/processors/counting.mock' import { @@ -204,6 +205,7 @@ describe('NoteController admin contract (e2e)', () => { translationProvider, translationEntryProvider, enrichmentProvider, + collabRepositoryProvider, aiSummaryProvider, aiInsightsProvider, aiTtsQueryProvider, diff --git a/apps/core/test/src/contracts/admin/posts-admin.contract.spec.ts b/apps/core/test/src/contracts/admin/posts-admin.contract.spec.ts index 731e49d4d56..27aa00f5baa 100644 --- a/apps/core/test/src/contracts/admin/posts-admin.contract.spec.ts +++ b/apps/core/test/src/contracts/admin/posts-admin.contract.spec.ts @@ -31,6 +31,7 @@ import { } from '../../../helper/api-shape' import { createE2EApp } from '../../../helper/create-e2e-app' import { authPassHeader } from '../../../mock/guard/auth.guard' +import { collabRepositoryProvider } from '../../../mock/modules/collab.mock' import { enrichmentProvider } from '../../../mock/modules/enrichment.mock' import { entitlementProvider } from '../../../mock/modules/entitlement.mock' import { snippetProvider } from '../../../mock/modules/snippet.mock' @@ -165,6 +166,7 @@ describe('PostController admin contract (e2e)', () => { translationProvider, translationEntryProvider, enrichmentProvider, + collabRepositoryProvider, aiInsightsProvider, aiSummaryProvider, aiTtsQueryProvider, diff --git a/apps/core/test/src/contracts/collab.contract.spec.ts b/apps/core/test/src/contracts/collab.contract.spec.ts new file mode 100644 index 00000000000..a642810c35b --- /dev/null +++ b/apps/core/test/src/contracts/collab.contract.spec.ts @@ -0,0 +1,183 @@ +import { describe, expect, test, vi } from 'vitest' + +import { apiRoutePrefix } from '~/common/decorators/api-controller.decorator' +import { AuthService } from '~/modules/auth/auth.service' +import { CollabController } from '~/modules/draft/collab.controller' +import { CollabService } from '~/modules/draft/collab.service' + +import { createE2EApp } from '../../helper/create-e2e-app' +import { authPassHeader } from '../../mock/guard/auth.guard' + +const documentId = '7000000000000000020' + +const service = { + accept: vi.fn(async () => ({ documentId, role: 'collaborator' })), + createInvite: vi.fn(async () => ({ + createdAt: new Date('2026-10-25T00:00:00.000Z'), + expiresAt: new Date('2026-11-01T00:00:00.000Z'), + id: '7000000000000000021', + maxUses: null, + token: 'secret-token-secret-token', + uses: 0, + })), + exportCrdt: vi.fn(async () => new Uint8Array([1, 2])), + importCrdt: vi.fn(async () => undefined), + listCollaborators: vi.fn(async () => [ + { + avatar: null, + contributedAt: null, + createdAt: new Date('2026-10-25T00:00:00.000Z'), + name: 'Bob', + readerId: 'r1', + showInByline: true, + }, + ]), + listInvites: vi.fn(async () => []), + removeCollaborator: vi.fn(async () => true), + revokeInvite: vi.fn(async () => true), + setShowInByline: vi.fn(async () => true), + workspaceFor: vi.fn(async () => ({ + collaborators: [{ avatar: null, name: 'Bob' }], + documentId, + draftId: '7000000000000000022', + me: { avatar: null, id: 'r1', name: 'Bob' }, + refType: 'post', + })), +} + +const readerHeader = { 'x-test-reader': 'r1' } + +const authService = { + getSessionUser: vi.fn(async (req: { headers: Record }) => { + const id = req.headers['x-test-reader'] + return id ? { user: { id, role: 'reader' } } : null + }), +} + +describe('CollabController (e2e)', () => { + const proxy = createE2EApp({ + controllers: [CollabController], + providers: [ + { provide: CollabService, useValue: service }, + { provide: AuthService, useValue: authService }, + ], + }) + + test('POST /collab/documents/:id/invites returns the token once, snake_cased', async () => { + const res = await proxy.app.inject({ + headers: authPassHeader, + method: 'POST', + payload: { max_uses: 5 }, + url: `${apiRoutePrefix}/collab/documents/${documentId}/invites`, + }) + + expect(res.statusCode).toBe(201) + expect(res.json().data).toMatchObject({ + expires_at: '2026-11-01T00:00:00.000Z', + max_uses: null, + token: 'secret-token-secret-token', + }) + expect(service.createInvite).toHaveBeenCalledWith(documentId, { + maxUses: 5, + }) + }) + + test('owner routes refuse requests without the owner session', async () => { + const res = await proxy.app.inject({ + headers: readerHeader, + method: 'GET', + url: `${apiRoutePrefix}/collab/documents/${documentId}/collaborators`, + }) + + expect(res.statusCode).toBe(401) + }) + + test('PATCH and DELETE collaborators reach the service', async () => { + const patch = await proxy.app.inject({ + headers: authPassHeader, + method: 'PATCH', + payload: { show_in_byline: false }, + url: `${apiRoutePrefix}/collab/documents/${documentId}/collaborators/r1`, + }) + const remove = await proxy.app.inject({ + headers: authPassHeader, + method: 'DELETE', + url: `${apiRoutePrefix}/collab/documents/${documentId}/collaborators/r1`, + }) + + expect(patch.statusCode).toBe(204) + expect(remove.statusCode).toBe(204) + expect(service.setShowInByline).toHaveBeenCalledWith( + documentId, + 'r1', + false, + ) + expect(service.removeCollaborator).toHaveBeenCalledWith(documentId, 'r1') + }) + + test('POST /collab/invites/:token/accept needs a reader session', async () => { + const anonymous = await proxy.app.inject({ + method: 'POST', + url: `${apiRoutePrefix}/collab/invites/secret-token-secret-token/accept`, + }) + const signedIn = await proxy.app.inject({ + headers: readerHeader, + method: 'POST', + url: `${apiRoutePrefix}/collab/invites/secret-token-secret-token/accept`, + }) + + expect(anonymous.statusCode).toBe(401) + expect(signedIn.statusCode).toBe(201) + expect(signedIn.json().data).toEqual({ + document_id: documentId, + role: 'collaborator', + }) + expect(service.accept).toHaveBeenCalledWith( + 'secret-token-secret-token', + 'r1', + ) + }) + + test('GET /collab/documents/:id returns the co-author workspace', async () => { + const res = await proxy.app.inject({ + headers: readerHeader, + method: 'GET', + url: `${apiRoutePrefix}/collab/documents/${documentId}`, + }) + + expect(res.statusCode).toBe(200) + expect(res.json().data).toMatchObject({ + document_id: documentId, + draft_id: '7000000000000000022', + me: { id: 'r1', name: 'Bob' }, + }) + }) + + test('co-author CRDT sync goes over raw bytes', async () => { + const pulled = await proxy.app.inject({ + headers: readerHeader, + method: 'GET', + url: `${apiRoutePrefix}/collab/documents/${documentId}/crdt`, + }) + const pushed = await proxy.app.inject({ + headers: { + ...readerHeader, + 'content-type': 'application/octet-stream', + }, + method: 'POST', + payload: Buffer.from([9]), + url: `${apiRoutePrefix}/collab/documents/${documentId}/crdt`, + }) + + expect(pulled.headers['content-type']).toContain('application/octet-stream') + expect([...pulled.rawPayload]).toEqual([1, 2]) + expect(pushed.statusCode).toBe(204) + const [calledDocument, reader, bytes] = service.importCrdt.mock + .calls[0] as unknown as [string, string, Uint8Array] + expect([calledDocument, reader, [...bytes]]).toEqual([ + documentId, + 'r1', + [9], + ]) + }) +}) diff --git a/apps/core/test/src/contracts/draft.contract.spec.ts b/apps/core/test/src/contracts/draft.contract.spec.ts index 87d33d55c4d..e047982999f 100644 --- a/apps/core/test/src/contracts/draft.contract.spec.ts +++ b/apps/core/test/src/contracts/draft.contract.spec.ts @@ -59,6 +59,7 @@ const provider = { right: revision(), }), findById: async () => branch, + findForEditing: async () => branch, findNewDrafts: async () => [branch], findRevisionById: async () => revision(), getBranchRevisions: async () => [revision()], diff --git a/apps/core/test/src/contracts/note.contract.spec.ts b/apps/core/test/src/contracts/note.contract.spec.ts index cfe9a7ab8c9..4c2c8573da0 100644 --- a/apps/core/test/src/contracts/note.contract.spec.ts +++ b/apps/core/test/src/contracts/note.contract.spec.ts @@ -14,6 +14,7 @@ import { assertPgTimestamps, } from '../../helper/api-shape' import { createE2EApp } from '../../helper/create-e2e-app' +import { collabRepositoryProvider } from '../../mock/modules/collab.mock' import { enrichmentProvider } from '../../mock/modules/enrichment.mock' import { countingServiceProvider } from '../../mock/processors/counting.mock' import { @@ -188,6 +189,7 @@ describe('NoteController contract (e2e)', () => { translationProvider, translationEntryProvider, enrichmentProvider, + collabRepositoryProvider, aiSummaryProvider, aiInsightsProvider, aiTtsQueryProvider, diff --git a/apps/core/test/src/contracts/post.contract.spec.ts b/apps/core/test/src/contracts/post.contract.spec.ts index 2cfadd26d90..6431de78b76 100644 --- a/apps/core/test/src/contracts/post.contract.spec.ts +++ b/apps/core/test/src/contracts/post.contract.spec.ts @@ -14,6 +14,7 @@ import { assertPgTimestamps, } from '../../helper/api-shape' import { createE2EApp } from '../../helper/create-e2e-app' +import { collabRepositoryProvider } from '../../mock/modules/collab.mock' import { enrichmentProvider } from '../../mock/modules/enrichment.mock' import { entitlementProvider } from '../../mock/modules/entitlement.mock' import { snippetProvider } from '../../mock/modules/snippet.mock' @@ -145,6 +146,7 @@ describe('PostController contract (e2e)', () => { translationProvider, translationEntryProvider, enrichmentProvider, + collabRepositoryProvider, aiInsightsProvider, aiSummaryProvider, aiTtsQueryProvider, diff --git a/apps/core/test/src/contracts/yohaku/note-detail.contract.spec.ts b/apps/core/test/src/contracts/yohaku/note-detail.contract.spec.ts index 86817eac0c1..19de67beb48 100644 --- a/apps/core/test/src/contracts/yohaku/note-detail.contract.spec.ts +++ b/apps/core/test/src/contracts/yohaku/note-detail.contract.spec.ts @@ -29,6 +29,10 @@ import { assertPgTimestamps, } from '../../../helper/api-shape' import { createE2EApp } from '../../../helper/create-e2e-app' +import { + collabCoAuthors, + collabRepositoryProvider, +} from '../../../mock/modules/collab.mock' import { enrichmentProvider } from '../../../mock/modules/enrichment.mock' import { countingServiceProvider } from '../../../mock/processors/counting.mock' import { @@ -171,6 +175,7 @@ describe('Yohaku contract — note detail (e2e)', () => { translationProvider, translationEntryProvider, enrichmentProvider, + collabRepositoryProvider, aiSummaryProvider, aiInsightsProvider, aiTtsQueryProvider, @@ -242,4 +247,30 @@ describe('Yohaku contract — note detail (e2e)', () => { expect(body.meta.enrichments).toBeDefined() expect(typeof body.meta.insights?.has_in_locale).toBe('boolean') }) + + test('lists the co-authors credited in the byline', async () => { + collabCoAuthors.set('7000000000000000070', [ + { avatar: 'https://a.test/bob.png', name: 'Bob' }, + ]) + try { + const res = await proxy.app.inject({ + method: 'GET', + url: `${apiRoutePrefix}/notes/nid/42`, + }) + expect(res.statusCode).toBe(200) + expect(res.json().data.co_authors).toEqual([ + { avatar: 'https://a.test/bob.png', name: 'Bob' }, + ]) + } finally { + collabCoAuthors.clear() + } + }) + + test('sends an empty byline when nobody else edited', async () => { + const res = await proxy.app.inject({ + method: 'GET', + url: `${apiRoutePrefix}/notes/nid/42`, + }) + expect(res.json().data.co_authors).toEqual([]) + }) }) diff --git a/apps/core/test/src/contracts/yohaku/post-detail.contract.spec.ts b/apps/core/test/src/contracts/yohaku/post-detail.contract.spec.ts index d15f240640d..ae97d9213ac 100644 --- a/apps/core/test/src/contracts/yohaku/post-detail.contract.spec.ts +++ b/apps/core/test/src/contracts/yohaku/post-detail.contract.spec.ts @@ -33,6 +33,10 @@ import { assertPgTimestamps, } from '../../../helper/api-shape' import { createE2EApp } from '../../../helper/create-e2e-app' +import { + collabCoAuthors, + collabRepositoryProvider, +} from '../../../mock/modules/collab.mock' import { enrichmentProvider } from '../../../mock/modules/enrichment.mock' import { entitlementProvider } from '../../../mock/modules/entitlement.mock' import { snippetProvider } from '../../../mock/modules/snippet.mock' @@ -142,6 +146,7 @@ describe('Yohaku contract — post detail (e2e)', () => { translationProvider, translationEntryProvider, enrichmentProvider, + collabRepositoryProvider, aiInsightsProvider, aiSummaryProvider, aiTtsQueryProvider, @@ -194,4 +199,30 @@ describe('Yohaku contract — post detail (e2e)', () => { // direct destructuring. assertHasKeysDeep(body.data, ['category.slug', 'category.name']) }) + + test('lists the co-authors credited in the byline', async () => { + collabCoAuthors.set('7000000000000000050', [ + { avatar: 'https://a.test/bob.png', name: 'Bob' }, + ]) + try { + const res = await proxy.app.inject({ + method: 'GET', + url: `${apiRoutePrefix}/posts/tech/yohaku-post`, + }) + expect(res.statusCode).toBe(200) + expect(res.json().data.co_authors).toEqual([ + { avatar: 'https://a.test/bob.png', name: 'Bob' }, + ]) + } finally { + collabCoAuthors.clear() + } + }) + + test('sends an empty byline when nobody else edited', async () => { + const res = await proxy.app.inject({ + method: 'GET', + url: `${apiRoutePrefix}/posts/tech/yohaku-post`, + }) + expect(res.json().data.co_authors).toEqual([]) + }) }) diff --git a/apps/core/test/src/contracts/yohaku/post-list.contract.spec.ts b/apps/core/test/src/contracts/yohaku/post-list.contract.spec.ts index 99c6cd677a1..328e9186584 100644 --- a/apps/core/test/src/contracts/yohaku/post-list.contract.spec.ts +++ b/apps/core/test/src/contracts/yohaku/post-list.contract.spec.ts @@ -26,6 +26,7 @@ import { assertPgTimestamps, } from '../../../helper/api-shape' import { createE2EApp } from '../../../helper/create-e2e-app' +import { collabRepositoryProvider } from '../../../mock/modules/collab.mock' import { enrichmentProvider } from '../../../mock/modules/enrichment.mock' import { entitlementProvider } from '../../../mock/modules/entitlement.mock' import { snippetProvider } from '../../../mock/modules/snippet.mock' @@ -120,6 +121,7 @@ describe('Yohaku contract — post list (e2e)', () => { translationProvider, translationEntryProvider, enrichmentProvider, + collabRepositoryProvider, aiInsightsProvider, aiSummaryProvider, aiTtsQueryProvider, diff --git a/apps/core/test/src/modules/content-migration/content-migration-commit.service.spec.ts b/apps/core/test/src/modules/content-migration/content-migration-commit.service.spec.ts index 22b39911fcd..e96d17ddcda 100644 --- a/apps/core/test/src/modules/content-migration/content-migration-commit.service.spec.ts +++ b/apps/core/test/src/modules/content-migration/content-migration-commit.service.spec.ts @@ -270,4 +270,41 @@ describe('ContentMigrationCommitService', () => { ) expect(database.state).toEqual(before) }) + + it('accepts a checkpoint precondition after the working copy head moved on', async () => { + const rows = branchRows() + const database = createTransactionalDatabase({ + source: sourceRow(), + ...rows, + branch: { ...rows.branch, headRevisionId: '501' }, + revision: { ...rows.revision, kind: 'publish' }, + translations: [], + }) + const service = new ContentMigrationCommitService( + database as never, + new LexicalService(), + ) + + await service.commitMarkdownToLexical(buildCommitInput([])) + + expect(database.state.source.contentFormat).toBe(ContentFormat.Lexical) + }) + + it('still rejects a moved head when the precondition revision is not a checkpoint', async () => { + const rows = branchRows() + const database = createTransactionalDatabase({ + source: sourceRow(), + ...rows, + branch: { ...rows.branch, headRevisionId: '501' }, + translations: [], + }) + const service = new ContentMigrationCommitService( + database as never, + new LexicalService(), + ) + + await expect( + service.commitMarkdownToLexical(buildCommitInput([])), + ).rejects.toBeInstanceOf(ConflictException) + }) }) diff --git a/apps/core/test/src/modules/content-migration/content-migration.service.spec.ts b/apps/core/test/src/modules/content-migration/content-migration.service.spec.ts index 5539ea84abf..36718a1a536 100644 --- a/apps/core/test/src/modules/content-migration/content-migration.service.spec.ts +++ b/apps/core/test/src/modules/content-migration/content-migration.service.spec.ts @@ -33,6 +33,7 @@ function createService( options: { source?: Record | null draft?: Record | null + revisions?: Record> translations?: Record[] } = {}, ) { @@ -53,6 +54,7 @@ function createService( const pageService = { findById: vi.fn() } const draftService = { findById: vi.fn().mockResolvedValue(options.draft ?? null), + findRevisionById: vi.fn(async (id: string) => options.revisions?.[id]), } const translationRepository = { listByRefId: vi @@ -208,4 +210,36 @@ describe('ContentMigrationService', () => { service.dryRunMarkdownToLexical({ ...request, branchId: '300' }), ).rejects.toBeInstanceOf(BadRequestException) }) + + it('pins the branch precondition to the checkpoint being published', async () => { + const lexicalContent = JSON.stringify({ root: { children: [] } }) + const { service } = createService({ + draft: { + document: { id: '400', refId: '100', refType: DraftRefType.Post }, + documentId: '400', + headRevision: { content: null, contentFormat: 'markdown', text: 'x' }, + headRevisionId: '501', + id: '300', + }, + revisions: { + '500': { + content: lexicalContent, + contentFormat: 'lexical', + documentId: '400', + id: '500', + text: '', + }, + }, + }) + + const result = await service.dryRunMarkdownToLexical({ + ...request, + branchId: '300', + revisionId: '500', + }) + + expect( + result.preconditions.find((item) => item.kind === 'branch'), + ).toMatchObject({ headRevisionId: '500' }) + }) }) diff --git a/apps/core/test/src/modules/draft/collab-room.service.spec.ts b/apps/core/test/src/modules/draft/collab-room.service.spec.ts new file mode 100644 index 00000000000..6586b3fdf54 --- /dev/null +++ b/apps/core/test/src/modules/draft/collab-room.service.spec.ts @@ -0,0 +1,461 @@ +import { describe, expect, it, vi } from 'vitest' + +import { createAppException } from '~/common/errors' +import { AppErrorCode } from '~/common/errors/app-error-code' +import { + type CollabIdentity, + type CollabMember, + CollabRoomService, + CollabWsEvents, +} from '~/modules/draft/collab-room.service' +import type { DocumentChange } from '~/modules/draft/draft-crdt.service' + +const b64 = (bytes: number[]) => Buffer.from(bytes).toString('base64') + +const owner: CollabIdentity = { + kind: 'owner', + user: { avatar: 'https://a.test/owner.png', id: 'owner-1', name: 'Owner' }, +} +const reader = (readerId: string): CollabIdentity => ({ + kind: 'reader', + readerId, + user: { avatar: null, id: readerId, name: `Reader ${readerId}` }, +}) + +const setup = (access: (documentId: string) => boolean = () => true) => { + const order: string[] = [] + let revokeListener: + ((documentId: string, readerId: string) => void) | undefined + const contributions: Array<[string, string]> = [] + let listener: ((change: DocumentChange) => void) | undefined + const crdt = { + onDocumentChanged: vi.fn((next: (change: DocumentChange) => void) => { + listener = next + return () => { + listener = undefined + } + }), + } + const drafts = { + documentIdOf: vi.fn(async () => 'doc-1'), + exportCrdtFor: vi.fn(async () => { + order.push('export') + return { documentId: 'doc-1', update: new Uint8Array([1, 2, 3]) } + }), + importCrdt: vi.fn(async () => undefined), + } + const published: { + event: string + exclude?: string + payload: unknown + room: string + }[] = [] + const relay = { + onRevoke: (listener: (documentId: string, readerId: string) => void) => { + revokeListener = listener + return () => { + revokeListener = undefined + } + }, + publish: vi.fn( + (event: string, payload: unknown, room: string, exclude?: string) => { + published.push({ event, exclude, payload, room }) + }, + ), + revoke: vi.fn((documentId: string, readerId: string) => + revokeListener?.(documentId, readerId), + ), + } + const service = new CollabRoomService(drafts as never, crdt as never, relay, { + canJoin: async (_member, documentId) => access(documentId), + recordContribution: async (documentId, readerId) => { + contributions.push([documentId, readerId]) + }, + }) + service.onModuleInit() + const member = ( + connectionId: string, + alive = () => true, + identity: CollabIdentity = owner, + ): CollabMember => ({ + alive, + connectionId, + identity, + join: vi.fn(async () => { + order.push('join') + }), + leave: vi.fn(async () => undefined), + send: vi.fn(), + }) + return { + contributions, + order, + relay, + change: (change: DocumentChange) => listener?.(change), + drafts, + member, + published, + service, + } +} + +describe('CollabRoomService', () => { + it('joins the document room and returns what the member lacks', async () => { + const { drafts, member, published, service } = setup() + const alice = member('conn-a') + + const ack = await service.join(alice, { draftId: 'd1', since: b64([9]) }) + + expect(ack).toEqual({ + data: { + connectionId: 'conn-a', + documentId: 'doc-1', + update: b64([1, 2, 3]), + }, + ok: true, + }) + expect(drafts.exportCrdtFor).toHaveBeenCalledWith('d1', new Uint8Array([9])) + expect(alice.join).toHaveBeenCalledWith('collab:doc-1') + expect(published).toEqual([ + { + event: CollabWsEvents.peerJoined, + exclude: 'conn-a', + payload: { connectionId: 'conn-a', documentId: 'doc-1' }, + room: 'collab:doc-1', + }, + ]) + }) + + it('stores an update with its origin and relays stored changes to everyone else', async () => { + const { change, drafts, member, published, service } = setup() + const alice = member('conn-a') + await service.join(alice, { draftId: 'd1' }) + published.length = 0 + + const ack = await service.update(alice, { + draftId: 'd1', + update: b64([7, 7]), + }) + change({ + documentId: 'doc-1', + origin: 'conn-a', + update: new Uint8Array([7, 7]), + }) + change({ documentId: 'doc-1', update: new Uint8Array([8]) }) + + expect(ack).toEqual({ ok: true }) + expect(drafts.importCrdt).toHaveBeenCalledWith( + 'd1', + new Uint8Array([7, 7]), + 'conn-a', + undefined, + ) + expect(published).toEqual([ + { + event: CollabWsEvents.update, + exclude: 'conn-a', + payload: { documentId: 'doc-1', update: b64([7, 7]) }, + room: 'collab:doc-1', + }, + { + event: CollabWsEvents.update, + exclude: undefined, + payload: { documentId: 'doc-1', update: b64([8]) }, + room: 'collab:doc-1', + }, + ]) + }) + + it('acks a rejected update with its code', async () => { + const { drafts, member, service } = setup() + const alice = member('conn-a') + await service.join(alice, { draftId: 'd1' }) + drafts.importCrdt.mockRejectedValueOnce( + createAppException(AppErrorCode.DRAFT_CRDT_UPDATE_REJECTED, { + branchId: 'd1', + reason: 'lineage', + }), + ) + + const ack = await service.update(alice, { + draftId: 'd1', + update: b64([1]), + }) + + expect(ack).toEqual({ code: 'DRAFT_CRDT_UPDATE_REJECTED', ok: false }) + }) + + it('joins the room before reading the snapshot, so no relayed change falls between', async () => { + const { member, order, service } = setup() + + await service.join(member('conn-a'), { draftId: 'd1' }) + + expect(order).toEqual(['join', 'export']) + }) + + it('undoes a join whose connection closed while it was in flight', async () => { + const { member, published, service } = setup() + let open = true + const alice = member('conn-a', () => open) + ;(alice.join as ReturnType).mockImplementation(async () => { + open = false + }) + + const ack = await service.join(alice, { draftId: 'd1' }) + + expect(ack).toEqual({ code: 'NOT_CONNECTED', ok: false }) + expect(alice.leave).toHaveBeenCalledWith('collab:doc-1') + expect(published).toEqual([]) + expect( + await service.presence(alice, { caret: null, draftId: 'd1' }), + ).toEqual({ code: 'NOT_JOINED', ok: false }) + }) + + it('refuses members the access check rejects', async () => { + const { drafts, member, service } = setup(() => false) + const alice = member('conn-a') + + const ack = await service.join(alice, { draftId: 'd1' }) + + expect(ack).toEqual({ code: 'FORBIDDEN', ok: false }) + expect(alice.join).not.toHaveBeenCalled() + expect(drafts.exportCrdtFor).not.toHaveBeenCalled() + }) + + it('refuses updates from a connection that has not joined the document', async () => { + const { drafts, member, service } = setup() + + const ack = await service.update(member('conn-a'), { + draftId: 'd1', + update: b64([1]), + }) + + expect(ack).toEqual({ code: 'NOT_JOINED', ok: false }) + expect(drafts.importCrdt).not.toHaveBeenCalled() + }) + + it('refuses an update over the size limit without storing it', async () => { + const { drafts, member, service } = setup() + + const ack = await service.update(member('conn-a'), { + draftId: 'd1', + update: Buffer.alloc(4 * 1024 * 1024 + 1).toString('base64'), + }) + + expect(ack).toEqual({ code: 'DRAFT_CRDT_UPDATE_REJECTED', ok: false }) + expect(drafts.importCrdt).not.toHaveBeenCalled() + }) + + it('relays presence stamped with the identity the server holds', async () => { + const { member, published, service } = setup() + const alice = member('conn-a') + const caret = { anchor: { id: 'p1', offset: 2 } } + + expect(await service.presence(alice, { caret, draftId: 'd1' })).toEqual({ + code: 'NOT_JOINED', + ok: false, + }) + + await service.join(alice, { draftId: 'd1' }) + published.length = 0 + const ack = await service.presence(alice, { + caret, + draftId: 'd1', + user: { name: 'Mallory' }, + }) + await service.presence(alice, { + caret: null, + draftId: 'd1', + entry: 'agent:42', + }) + + expect(ack).toEqual({ ok: true }) + expect(published).toEqual([ + { + event: CollabWsEvents.presence, + exclude: 'conn-a', + payload: { + caret, + connectionId: 'conn-a', + documentId: 'doc-1', + entry: null, + user: owner.user, + }, + room: 'collab:doc-1', + }, + { + event: CollabWsEvents.presence, + exclude: 'conn-a', + payload: { + caret: null, + connectionId: 'conn-a', + documentId: 'doc-1', + entry: 'agent:42', + user: { avatar: null, id: 'agent:42', name: 'AI' }, + }, + room: 'collab:doc-1', + }, + ]) + }) + + it('refuses agent presence entries and oversized carets from co-authors', async () => { + const { member, service } = setup() + const bob = member('conn-b', () => true, reader('r1')) + await service.join(bob, { draftId: 'd1' }) + + expect( + await service.presence(bob, { + caret: null, + draftId: 'd1', + entry: 'agent:1', + }), + ).toEqual({ code: 'FORBIDDEN', ok: false }) + expect( + await service.presence(bob, { caret: 'x'.repeat(5000), draftId: 'd1' }), + ).toEqual({ code: 'VALIDATION_FAILED', ok: false }) + }) + + it('binds a co-author update to their reader id and records the first contribution', async () => { + const { contributions, drafts, member, service } = setup() + const bob = member('conn-b', () => true, reader('r1')) + await service.join(bob, { draftId: 'd1' }) + + await service.update(bob, { draftId: 'd1', update: b64([1]) }) + await service.update(bob, { draftId: 'd1', update: b64([2]) }) + + expect(drafts.importCrdt).toHaveBeenCalledWith( + 'd1', + new Uint8Array([1]), + 'conn-b', + 'human:r1', + ) + expect(contributions).toEqual([['doc-1', 'r1']]) + }) + + it("drops co-author frames past the rate limit but never the owner's", async () => { + const { member, service } = setup() + const now = vi.spyOn(Date, 'now').mockReturnValue(1_000_000) + const bob = member('conn-b', () => true, reader('r1')) + const alice = member('conn-a') + await service.join(bob, { draftId: 'd1' }) + await service.join(alice, { draftId: 'd1' }) + + const send = (who: CollabMember) => + service.presence(who, { caret: null, draftId: 'd1' }) + for (let index = 0; index < 120; index += 1) await send(bob) + const limited = await send(bob) + now.mockReturnValue(1_000_000 + 1000) + const refilled = await send(bob) + for (let index = 0; index < 200; index += 1) await send(alice) + const ownerAck = await send(alice) + now.mockRestore() + + expect(limited).toEqual({ code: 'RATE_LIMITED', ok: false }) + expect(refilled).toEqual({ ok: true }) + expect(ownerAck).toEqual({ ok: true }) + }) + + it('drops a removed co-author from the room on every pod', async () => { + const { member, published, relay, service } = setup() + const bob = member('conn-b', () => true, reader('r1')) + const carol = member('conn-c', () => true, reader('r2')) + await service.join(bob, { draftId: 'd1' }) + await service.join(carol, { draftId: 'd1' }) + published.length = 0 + + service.revoke('doc-1', 'r1') + + expect(relay.revoke).toHaveBeenCalledWith('doc-1', 'r1') + expect(bob.leave).toHaveBeenCalledWith('collab:doc-1') + expect(bob.send).toHaveBeenCalledWith(CollabWsEvents.revoked, { + documentId: 'doc-1', + }) + expect(carol.leave).not.toHaveBeenCalled() + expect(published.map((frame) => [frame.event, frame.payload])).toEqual([ + [ + CollabWsEvents.peerLeft, + { connectionId: 'conn-b', documentId: 'doc-1' }, + ], + ]) + expect( + await service.update(bob, { draftId: 'd1', update: b64([1]) }), + ).toEqual({ code: 'NOT_JOINED', ok: false }) + }) + + it('announces a member leaving on leave and on disconnect', async () => { + const { member, published, service } = setup() + const alice = member('conn-a') + const bob = member('conn-b') + await service.join(alice, { draftId: 'd1' }) + await service.join(bob, { draftId: 'd1' }) + published.length = 0 + + await service.leave(alice, { draftId: 'd1' }) + service.disconnect('conn-b') + service.disconnect('conn-b') + + expect(alice.leave).toHaveBeenCalledWith('collab:doc-1') + expect(published.map((frame) => [frame.event, frame.payload])).toEqual([ + [ + CollabWsEvents.peerLeft, + { connectionId: 'conn-a', documentId: 'doc-1' }, + ], + [ + CollabWsEvents.peerLeft, + { connectionId: 'conn-b', documentId: 'doc-1' }, + ], + ]) + }) + + it('rejects malformed payloads', async () => { + const { member, service } = setup() + + expect(await service.join(member('conn-a'), { draftId: 3 })).toEqual({ + code: 'VALIDATION_FAILED', + ok: false, + }) + }) + + it('drops a removed co-author on this pod even if the broadcast is lost', async () => { + const { member, relay, service } = setup() + relay.revoke.mockImplementation(() => undefined) + const bob = member('conn-b', () => true, reader('r1')) + await service.join(bob, { draftId: 'd1' }) + + service.revoke('doc-1', 'r1') + + expect(bob.leave).toHaveBeenCalledWith('collab:doc-1') + expect( + await service.update(bob, { draftId: 'd1', update: b64([1]) }), + ).toEqual({ code: 'NOT_JOINED', ok: false }) + }) + + it('backs out a join whose access was removed while it was loading', async () => { + let allowed = true + const { drafts, member, service } = setup(() => allowed) + drafts.exportCrdtFor.mockImplementationOnce(async () => { + allowed = false + return { documentId: 'doc-1', update: new Uint8Array([1]) } + }) + const bob = member('conn-b', () => true, reader('r1')) + + const ack = await service.join(bob, { draftId: 'd1' }) + + expect(ack).toEqual({ code: 'FORBIDDEN', ok: false }) + expect(bob.leave).toHaveBeenCalledWith('collab:doc-1') + expect( + await service.update(bob, { draftId: 'd1', update: b64([1]) }), + ).toEqual({ code: 'NOT_JOINED', ok: false }) + }) + + it('leaves the room again when loading the document fails', async () => { + const { drafts, member, service } = setup() + drafts.exportCrdtFor.mockRejectedValueOnce(new Error('boom')) + const bob = member('conn-b', () => true, reader('r1')) + + const ack = await service.join(bob, { draftId: 'd1' }) + + expect(ack).toEqual({ code: 'INTERNAL_ERROR', ok: false }) + expect(bob.leave).toHaveBeenCalledWith('collab:doc-1') + }) +}) diff --git a/apps/core/test/src/modules/draft/collab.repository.pg.e2e.spec.ts b/apps/core/test/src/modules/draft/collab.repository.pg.e2e.spec.ts new file mode 100644 index 00000000000..99363beb779 --- /dev/null +++ b/apps/core/test/src/modules/draft/collab.repository.pg.e2e.spec.ts @@ -0,0 +1,278 @@ +import { SnowflakeGenerator } from '@mx-space/db-schema/id' +import * as schema from '@mx-space/db-schema/schema' +import { drizzle } from 'drizzle-orm/node-postgres' +import { Pool } from 'pg' +import { createIsolatedPgDatabase } from 'test/helper/pg-testcontainer' +import { afterAll, beforeAll, describe, expect, it } from 'vitest' + +import { CollabRepository } from '~/modules/draft/collab.repository' +import { DraftRefType } from '~/modules/draft/draft.enum' +import { DraftRepository } from '~/modules/draft/draft.repository' +import type { AppDatabase } from '~/processors/database/postgres.provider' +import type { SnowflakeService } from '~/shared/id/snowflake.service' + +const snapshot = { + content: null, + contentFormat: 'lexical', + images: null, + meta: null, + text: 'Root', + title: 'Root', + typeSpecificData: null, +} + +const later = (ms: number) => new Date(Date.now() + ms) + +describe('CollabRepository (real PG)', () => { + let pool: Pool + let db: AppDatabase + let database: Awaited> + let drafts: DraftRepository + let repository: CollabRepository + let readerSeq = 0 + + const seedDocument = async () => + ( + await drafts.createDocumentWithRoot( + DraftRefType.Post, + null, + snapshot, + false, + ) + ).document.id + + const seedReader = async (name: string) => { + readerSeq += 1 + const id = `reader-${readerSeq}` + await db.insert(schema.readers).values({ + id, + image: `https://avatars.test/${id}.png`, + name, + }) + return id + } + + beforeAll(async () => { + database = await createIsolatedPgDatabase() + pool = new Pool({ connectionString: database.getConnectionUri(), max: 4 }) + db = drizzle(pool, { schema }) as unknown as AppDatabase + const snowflake = new SnowflakeGenerator({ + workerId: 45, + }) as SnowflakeService + drafts = new DraftRepository(db, snowflake) + repository = new CollabRepository(db, snowflake) + }, 120_000) + + afterAll(async () => { + await pool?.end() + await database?.drop() + }) + + it('accepts an invite once per reader and counts each new reader', async () => { + const documentId = await seedDocument() + const alice = await seedReader('Alice') + const bob = await seedReader('Bob') + await repository.createInvite({ + documentId, + expiresAt: later(60_000), + maxUses: 3, + tokenHash: 'hash-a', + }) + + expect(await repository.peekInvite('hash-a')).toEqual({ + documentId, + ok: true, + }) + expect(await repository.acceptInvite('hash-a', alice)).toEqual({ + documentId, + ok: true, + }) + expect(await repository.acceptInvite('hash-a', alice)).toEqual({ + documentId, + ok: true, + }) + expect(await repository.acceptInvite('hash-a', bob)).toEqual({ + documentId, + ok: true, + }) + + const [invite] = await repository.listInvites(documentId) + expect(invite.uses).toBe(2) + expect(await repository.isCollaborator(documentId, alice)).toBe(true) + expect(await repository.isCollaborator(documentId, bob)).toBe(true) + }) + + it('refuses expired, revoked, exhausted and unknown invites', async () => { + const documentId = await seedDocument() + const reader = await seedReader('Carol') + const other = await seedReader('Dan') + await repository.createInvite({ + documentId, + expiresAt: later(-1000), + maxUses: null, + tokenHash: 'hash-expired', + }) + const revoked = await repository.createInvite({ + documentId, + expiresAt: later(60_000), + maxUses: null, + tokenHash: 'hash-revoked', + }) + await repository.revokeInvite(documentId, revoked.id) + await repository.createInvite({ + documentId, + expiresAt: later(60_000), + maxUses: 1, + tokenHash: 'hash-once', + }) + await repository.acceptInvite('hash-once', other) + + expect(await repository.acceptInvite('hash-expired', reader)).toEqual({ + ok: false, + reason: 'expired', + }) + expect(await repository.acceptInvite('hash-revoked', reader)).toEqual({ + ok: false, + reason: 'revoked', + }) + expect(await repository.acceptInvite('hash-once', reader)).toEqual({ + ok: false, + reason: 'exhausted', + }) + expect(await repository.acceptInvite('hash-none', reader)).toEqual({ + ok: false, + reason: 'not_found', + }) + expect(await repository.peekInvite('hash-revoked')).toEqual({ + ok: false, + reason: 'revoked', + }) + expect(await repository.isCollaborator(documentId, reader)).toBe(false) + expect(await repository.listInvites(documentId)).toEqual([]) + }) + + it('lists contributors who stay in the byline, in order of first edit', async () => { + const refId = '7700000000000000009' + const documentId = ( + await drafts.createDocumentWithRoot( + DraftRefType.Post, + refId, + snapshot, + false, + ) + ).document.id + const first = await seedReader('First') + const second = await seedReader('Second') + const hidden = await seedReader('Hidden') + const idle = await seedReader('Idle') + for (const reader of [first, second, hidden, idle]) { + await repository.addCollaborator(documentId, reader) + } + await repository.markContributed(documentId, second) + await repository.markContributed(documentId, first) + await repository.markContributed(documentId, second) + await repository.markContributed(documentId, hidden) + await repository.setShowInByline(documentId, hidden, false) + + const byline = await repository.coAuthorsByRef(DraftRefType.Post, [refId]) + + expect(byline.get(refId)?.map((author) => author.name)).toEqual([ + 'Second', + 'First', + ]) + expect(byline.get(refId)?.[0]).toMatchObject({ + avatar: `https://avatars.test/${second}.png`, + }) + expect( + (await repository.listCollaborators(documentId)).map((row) => ({ + contributed: row.contributedAt !== null, + name: row.name, + showInByline: row.showInByline, + })), + ).toEqual([ + { contributed: true, name: 'First', showInByline: true }, + { contributed: true, name: 'Second', showInByline: true }, + { contributed: true, name: 'Hidden', showInByline: false }, + { contributed: false, name: 'Idle', showInByline: true }, + ]) + }) + + it('removes a collaborator and reports whether one was removed', async () => { + const documentId = await seedDocument() + const reader = await seedReader('Eve') + await repository.addCollaborator(documentId, reader) + + expect(await repository.removeCollaborator(documentId, reader)).toBe(true) + expect(await repository.removeCollaborator(documentId, reader)).toBe(false) + expect(await repository.isCollaborator(documentId, reader)).toBe(false) + }) + + it('resolves the name, avatar and role a reader shows in the room', async () => { + const reader = await seedReader('Frank') + + expect(await repository.presenceUserOf(reader)).toEqual({ + avatar: `https://avatars.test/${reader}.png`, + id: reader, + name: 'Frank', + role: 'reader', + }) + expect(await repository.presenceUserOf('missing')).toBeNull() + }) + + it('finds co-authors by the content the document belongs to', async () => { + const { document } = await drafts.createDocumentWithRoot( + DraftRefType.Post, + '7700000000000000001', + snapshot, + false, + ) + const reader = await seedReader('Grace') + await repository.addCollaborator(document.id, reader) + await repository.markContributed(document.id, reader) + + const byRef = await repository.coAuthorsByRef(DraftRefType.Post, [ + '7700000000000000001', + '7700000000000000002', + ]) + + expect(byRef.get('7700000000000000001')).toEqual([ + { avatar: `https://avatars.test/${reader}.png`, name: 'Grace' }, + ]) + expect(byRef.has('7700000000000000002')).toBe(false) + }) + + it('keeps a removed reader out through old links but lets a newer link bring them back', async () => { + const documentId = await seedDocument() + const reader = await seedReader('Heidi') + await repository.createInvite({ + documentId, + expiresAt: later(60_000), + maxUses: null, + tokenHash: 'hash-old', + }) + await repository.acceptInvite('hash-old', reader) + await repository.markContributed(documentId, reader) + await repository.removeCollaborator(documentId, reader) + + expect(await repository.acceptInvite('hash-old', reader)).toEqual({ + ok: false, + reason: 'revoked', + }) + expect(await repository.isCollaborator(documentId, reader)).toBe(false) + expect(await repository.listCollaborators(documentId)).toEqual([ + expect.objectContaining({ name: 'Heidi', removedAt: expect.any(Date) }), + ]) + + await repository.createInvite({ + documentId, + expiresAt: later(60_000), + maxUses: null, + tokenHash: 'hash-new', + }) + expect(await repository.acceptInvite('hash-new', reader)).toEqual({ + documentId, + ok: true, + }) + expect(await repository.isCollaborator(documentId, reader)).toBe(true) + }) +}) diff --git a/apps/core/test/src/modules/draft/collab.service.spec.ts b/apps/core/test/src/modules/draft/collab.service.spec.ts new file mode 100644 index 00000000000..1d39fc9d4ca --- /dev/null +++ b/apps/core/test/src/modules/draft/collab.service.spec.ts @@ -0,0 +1,242 @@ +import { createHash } from 'node:crypto' + +import { describe, expect, it, vi } from 'vitest' + +import { AppErrorCode } from '~/common/errors/app-error-code' +import { CollabService } from '~/modules/draft/collab.service' + +const sha256 = (value: string) => + createHash('sha256').update(value).digest('hex') + +const setup = () => { + const repository = { + acceptInvite: vi.fn(async () => ({ documentId: 'doc-1', ok: true })), + createInvite: vi.fn(async (input: any) => ({ + createdAt: new Date(0), + documentId: input.documentId, + expiresAt: input.expiresAt, + id: 'inv-1', + maxUses: input.maxUses, + tokenHash: input.tokenHash, + uses: 0, + })), + isCollaborator: vi.fn(async () => true), + listCollaborators: vi.fn(async () => [ + { + avatar: 'https://a.test/bob.png', + contributedAt: null, + createdAt: new Date(0), + name: 'Bob', + readerId: 'r1', + showInByline: true, + }, + ]), + markContributed: vi.fn(async () => undefined), + ownerId: vi.fn(async () => 'owner-1'), + peekInvite: vi.fn(async () => ({ documentId: 'doc-1', ok: true })), + presenceUserOf: vi.fn(async (id: string) => + id === 'owner-1' + ? { avatar: null, id, name: 'Owner', role: 'owner' } + : { avatar: null, id, name: `Reader ${id}`, role: 'reader' }, + ), + removeCollaborator: vi.fn(async () => true), + } + const draftRepository = { + findBranchesByDocument: vi.fn(async () => [{ id: 'draft-1' }]), + findDocumentById: vi.fn(async () => ({ + id: 'doc-1', + refId: 'post-1', + refType: 'post', + })), + } + const drafts = { + exportCrdt: vi.fn(async () => new Uint8Array([1])), + importCrdt: vi.fn(async () => undefined), + } + const rooms = { revoke: vi.fn() } + const bus = { publish: vi.fn() } + const service = new CollabService( + repository as never, + draftRepository as never, + drafts as never, + rooms as never, + bus as never, + ) + return { bus, draftRepository, drafts, repository, rooms, service } +} + +describe('CollabService', () => { + it('stores only the hash of a fresh invite token and returns the token once', async () => { + const { repository, service } = setup() + + const invite = await service.createInvite('doc-1', { maxUses: 3 }) + + expect(invite.token).toMatch(/^[\w-]{43}$/) + expect(repository.createInvite).toHaveBeenCalledWith( + expect.objectContaining({ + documentId: 'doc-1', + maxUses: 3, + tokenHash: sha256(invite.token), + }), + ) + expect(invite.expiresAt.getTime() - Date.now()).toBeGreaterThan( + 6.9 * 24 * 3600 * 1000, + ) + }) + + it('accepts an invite for a reader and points them at the document', async () => { + const { repository, service } = setup() + + const result = await service.accept('tok', 'r1') + + expect(repository.acceptInvite).toHaveBeenCalledWith(sha256('tok'), 'r1') + expect(result).toEqual({ documentId: 'doc-1', role: 'collaborator' }) + }) + + it('sends the owner opening an invite to their own draft without joining', async () => { + const { repository, service } = setup() + repository.peekInvite.mockResolvedValueOnce({ + documentId: 'doc-9', + ok: true, + }) + + const result = await service.accept('tok', 'owner-1') + + expect(repository.acceptInvite).not.toHaveBeenCalled() + expect(result).toEqual({ + documentId: 'doc-9', + draftId: 'draft-1', + refId: 'post-1', + refType: 'post', + role: 'owner', + }) + }) + + it('reports why an invite cannot be used', async () => { + const { repository, service } = setup() + repository.acceptInvite.mockResolvedValueOnce({ + ok: false, + reason: 'expired', + } as never) + + await expect(service.accept('tok', 'r1')).rejects.toMatchObject({ + code: AppErrorCode.COLLAB_INVITE_INVALID, + }) + }) + + it('opens the shared draft only for collaborators', async () => { + const { repository, service } = setup() + + expect(await service.workspaceFor('doc-1', 'r1')).toEqual({ + collaborators: [{ avatar: 'https://a.test/bob.png', name: 'Bob' }], + documentId: 'doc-1', + draftId: 'draft-1', + me: { avatar: null, id: 'r1', name: 'Reader r1' }, + refType: 'post', + }) + + repository.isCollaborator.mockResolvedValueOnce(false) + await expect(service.workspaceFor('doc-1', 'r2')).rejects.toMatchObject({ + code: AppErrorCode.COLLAB_FORBIDDEN, + }) + }) + + it('signs a co-author HTTP update with their reader id and records it', async () => { + const { drafts, repository, service } = setup() + + await service.importCrdt('doc-1', 'r1', new Uint8Array([5])) + + expect(drafts.importCrdt).toHaveBeenCalledWith( + 'draft-1', + new Uint8Array([5]), + undefined, + 'human:r1', + ) + expect(repository.markContributed).toHaveBeenCalledWith('doc-1', 'r1') + }) + + it('refuses HTTP sync for readers who are not collaborators', async () => { + const { drafts, repository, service } = setup() + repository.isCollaborator.mockResolvedValue(false) + + await expect( + service.exportCrdt('doc-1', 'r2', undefined), + ).rejects.toMatchObject({ code: AppErrorCode.COLLAB_FORBIDDEN }) + await expect( + service.importCrdt('doc-1', 'r2', new Uint8Array([1])), + ).rejects.toMatchObject({ code: AppErrorCode.COLLAB_FORBIDDEN }) + expect(drafts.importCrdt).not.toHaveBeenCalled() + }) + + it('drops a removed collaborator from the live room', async () => { + const { rooms, service } = setup() + + await service.removeCollaborator('doc-1', 'r1') + + expect(rooms.revoke).toHaveBeenCalledWith('doc-1', 'r1') + }) + + it('resolves the owner and readers to the identities the room stamps', async () => { + const { service } = setup() + + expect(await service.identityOf('owner-1')).toEqual({ + kind: 'owner', + user: { avatar: null, id: 'owner-1', name: 'Owner' }, + }) + expect(await service.identityOf('r1')).toEqual({ + kind: 'reader', + readerId: 'r1', + user: { avatar: null, id: 'r1', name: 'Reader r1' }, + }) + }) + + it('looks the owner up once rather than on every frame', async () => { + const { repository, service } = setup() + + await service.ownerIdentity() + await service.ownerIdentity() + await service.ownerIdentity() + + expect(repository.ownerId).toHaveBeenCalledTimes(1) + expect(repository.presenceUserOf).toHaveBeenCalledTimes(1) + }) + + it('leaves removed collaborators out of the co-author workspace', async () => { + const { repository, service } = setup() + repository.listCollaborators.mockResolvedValueOnce([ + { + avatar: null, + contributedAt: null, + createdAt: new Date(0), + name: 'Gone', + readerId: 'r9', + removedAt: new Date(0), + showInByline: true, + } as never, + ]) + + const workspace = await service.workspaceFor('doc-1', 'r1') + + expect(workspace.collaborators).toEqual([]) + }) + + it('tells the owner admin tabs when a reader joins the document', async () => { + const { bus, service } = setup() + + await service.accept('tok', 'r1') + + expect(bus.publish).toHaveBeenCalledWith({ + event: 'collab.collaborators.changed', + ns: 'admin', + payload: { documentId: 'doc-1' }, + }) + }) + + it('stays quiet when the owner only peeks at an invite', async () => { + const { bus, service } = setup() + + await service.accept('tok', 'owner-1') + + expect(bus.publish).not.toHaveBeenCalled() + }) +}) diff --git a/apps/core/test/src/modules/draft/draft-crdt.doc.spec.ts b/apps/core/test/src/modules/draft/draft-crdt.doc.spec.ts new file mode 100644 index 00000000000..6e6639e0e24 --- /dev/null +++ b/apps/core/test/src/modules/draft/draft-crdt.doc.spec.ts @@ -0,0 +1,319 @@ +import { LoroDoc } from 'loro-crdt' +import { describe, expect, it } from 'vitest' + +import { + createContentDoc, + editDocAt, + readDocContent, + revertChange, + vetUpdate, +} from '~/modules/draft/draft-crdt.doc' + +const text = (value: string) => ({ + detail: 0, + format: 0, + mode: 'normal', + style: '', + text: value, + type: 'text', + version: 1, +}) + +const paragraph = (value: string) => ({ + children: [text(value)], + direction: null, + format: '', + indent: 0, + textFormat: 0, + textStyle: '', + type: 'paragraph', + version: 1, +}) + +const state = (...children: object[]) => + JSON.stringify({ + root: { + children, + direction: null, + format: '', + indent: 0, + type: 'root', + version: 1, + }, + }) + +const texts = (content: string) => + ( + JSON.parse(content) as { + root: { children: { children?: { text?: string }[]; type: string }[] } + } + ).root.children.map((child) => + child.children + ? child.children.map((leaf) => leaf.text).join('') + : child.type, + ) + +describe('draft CRDT document', () => { + it('round-trips title and body, keeping blocks the headless set does not know', () => { + const map = { type: 'map', version: 1, lat: 1, lng: 2, zoom: 3 } + const mystery = { type: 'mystery-block', version: 1, payload: { a: [1] } } + const doc = createContentDoc( + { content: state(paragraph('Hello'), map, mystery), title: 'Title' }, + 'import 1', + ) + + const read = readDocContent(doc) + const children = JSON.parse(read.content).root.children + + expect(read.title).toBe('Title') + expect(texts(read.content)).toEqual(['Hello', 'map', 'mystery-block']) + expect(children[1]).toMatchObject(map) + expect(children[2]).toMatchObject(mystery) + }) + + it('keeps block ids and nested lists through the tree', () => { + const list = { + children: [ + { + children: [text('item')], + direction: null, + format: '', + indent: 0, + type: 'listitem', + value: 1, + version: 1, + }, + ], + direction: null, + format: '', + indent: 0, + listType: 'bullet', + start: 1, + tag: 'ul', + type: 'list', + version: 1, + } + const doc = createContentDoc( + { + content: state({ ...paragraph('p'), $: { blockId: 'blk1' } }, list), + title: '', + }, + 'import 1', + ) + + const children = JSON.parse(readDocContent(doc).content).root.children + + expect(children[0].$).toEqual({ blockId: 'blk1' }) + expect(children[1].children[0].children[0].text).toBe('item') + }) + + it('merges a whole-document write made at an older version with newer edits', () => { + const doc = createContentDoc( + { content: state(paragraph('one'), paragraph('two')), title: 'T' }, + 'import 1', + ) + const base = doc.frontiers() + doc.import( + editDocAt( + doc, + doc.frontiers(), + { content: state(paragraph('one'), paragraph('two!')), title: 'T' }, + 'human:owner', + ), + ) + + doc.import( + editDocAt( + doc, + base, + { content: state(paragraph('ONE'), paragraph('two')), title: 'T2' }, + 'legacy-save 9', + ), + ) + + const read = readDocContent(doc) + expect(texts(read.content)).toEqual(['ONE', 'two!']) + expect(read.title).toBe('T2') + }) + + it('commits a write under the peer it is given', () => { + const doc = createContentDoc( + { content: state(paragraph('one')), title: 'T' }, + 'import 1', + ) + + const update = editDocAt( + doc, + doc.frontiers(), + { content: state(paragraph('one'), paragraph('agent')), title: 'T' }, + 'agent:add a line', + { peer: '424242' }, + ) + doc.import(update) + + const agentChanges = doc.getAllChanges().get('424242') + expect(agentChanges?.map((change) => change.message)).toEqual([ + 'agent:add a line', + ]) + expect(texts(readDocContent(doc).content)).toEqual(['one', 'agent']) + }) + + it('reverts one change while edits before and after it survive', () => { + const doc = createContentDoc( + { content: state(paragraph('a'), paragraph('b')), title: 'T' }, + 'import 1', + ) + const step = (content: string, message: string) => { + doc.import( + editDocAt(doc, doc.frontiers(), { content, title: 'T' }, message), + ) + } + step(state(paragraph('a1'), paragraph('b')), 'human:owner') + const before = doc.frontiers() + step( + state(paragraph('a1'), paragraph('b'), paragraph('agent')), + 'agent:add', + ) + const agentHead = doc.frontiers() + step( + state(paragraph('a1'), paragraph('b2'), paragraph('agent')), + 'human:owner', + ) + + doc.import(revertChange(doc, agentHead[0], before, 'undo agent')) + + expect(texts(readDocContent(doc).content)).toEqual(['a1', 'b2']) + }) + + it('rejects an update from a different document lineage', () => { + const doc = createContentDoc( + { content: state(paragraph('a')), title: 'T' }, + 'import 1', + ) + const stranger = createContentDoc( + { content: state(paragraph('b')), title: 'T' }, + 'import 2', + ) + + expect(() => + vetUpdate(doc, stranger.export({ mode: 'update' })), + ).toThrowError(/lineage/) + }) + + it('rejects an update whose dependencies the document lacks', () => { + const doc = createContentDoc( + { content: state(paragraph('a')), title: 'T' }, + 'import 1', + ) + const peer = new LoroDoc() + peer.import(doc.export({ mode: 'snapshot' })) + const edit = (content: string) => + peer.import( + editDocAt(peer, peer.frontiers(), { content, title: 'T' }, 'human'), + ) + edit(state(paragraph('a'), paragraph('b'))) + const afterFirst = peer.oplogVersion() + edit(state(paragraph('a'), paragraph('b'), paragraph('c'))) + + expect(() => + vetUpdate(doc, peer.export({ from: afterFirst, mode: 'update' })), + ).toThrowError(/missing/) + }) + + it('accepts a well-formed peer update and returns the merged probe', () => { + const doc = createContentDoc( + { content: state(paragraph('a')), title: 'T' }, + 'import 1', + ) + const peer = new LoroDoc() + peer.import(doc.export({ mode: 'snapshot' })) + peer.import( + editDocAt( + peer, + peer.frontiers(), + { content: state(paragraph('a'), paragraph('b')), title: 'T' }, + 'human:owner', + ), + ) + + const probe = vetUpdate( + doc, + peer.export({ from: doc.oplogVersion(), mode: 'update' }), + ) + + expect(texts(readDocContent(probe).content)).toEqual(['a', 'b']) + expect(texts(readDocContent(doc).content)).toEqual(['a']) + }) + + it('refuses an update carrying changes that are not signed by the expected author', () => { + const doc = createContentDoc( + { content: state(paragraph('a')), title: 'T' }, + 'import 1', + ) + const peer = new LoroDoc() + peer.import(doc.export({ mode: 'snapshot' })) + const edit = (message: string, ...values: string[]) => + peer.import( + editDocAt( + peer, + peer.frontiers(), + { content: state(...values.map(paragraph)), title: 'T' }, + message, + ), + ) + edit('human:r1', 'a', 'b') + const signed = peer.export({ from: doc.oplogVersion(), mode: 'update' }) + edit('agent:rewrite', 'a', 'b', 'c') + const mixed = peer.export({ from: doc.oplogVersion(), mode: 'update' }) + + expect(() => vetUpdate(doc, signed, { author: 'human:r1' })).not.toThrow() + expect(() => vetUpdate(doc, signed, { author: 'human:r2' })).toThrowError( + /author/, + ) + expect(() => vetUpdate(doc, mixed, { author: 'human:r1' })).toThrowError( + /author/, + ) + }) + + it('refuses author-signed changes written under a peer someone else already uses', () => { + const doc = createContentDoc( + { content: state(paragraph('a')), title: 'T' }, + 'import 1', + ) + const owner = new LoroDoc() + owner.import(doc.export({ mode: 'snapshot' })) + doc.import( + editDocAt( + owner, + owner.frontiers(), + { content: state(paragraph('a'), paragraph('o')), title: 'T' }, + 'human:owner', + ), + ) + const ownerPeer = [...doc.getAllChanges().entries()].find(([, changes]) => + changes.some((change) => change.message === 'human:owner'), + )![0] + const forger = new LoroDoc() + forger.import(doc.export({ mode: 'snapshot' })) + forger.setPeerId(ownerPeer) + const from = doc.oplogVersion() + forger.import( + editDocAt( + forger, + forger.frontiers(), + { + content: state(paragraph('a'), paragraph('o'), paragraph('x')), + title: 'T', + }, + 'human:r1', + { peer: ownerPeer }, + ), + ) + + expect(() => + vetUpdate(doc, forger.export({ from, mode: 'update' }), { + author: 'human:r1', + }), + ).toThrowError(/author/) + }) +}) diff --git a/apps/core/test/src/modules/draft/draft-crdt.repository.pg.e2e.spec.ts b/apps/core/test/src/modules/draft/draft-crdt.repository.pg.e2e.spec.ts new file mode 100644 index 00000000000..37441f896c0 --- /dev/null +++ b/apps/core/test/src/modules/draft/draft-crdt.repository.pg.e2e.spec.ts @@ -0,0 +1,240 @@ +import { SnowflakeGenerator } from '@mx-space/db-schema/id' +import * as schema from '@mx-space/db-schema/schema' +import { eq } from 'drizzle-orm' +import { drizzle } from 'drizzle-orm/node-postgres' +import { Pool } from 'pg' +import { createIsolatedPgDatabase } from 'test/helper/pg-testcontainer' +import { afterAll, beforeAll, describe, expect, it } from 'vitest' + +import { DraftRefType } from '~/modules/draft/draft.enum' +import { DraftRepository } from '~/modules/draft/draft.repository' +import { DraftCrdtRepository } from '~/modules/draft/draft-crdt.repository' +import type { AppDatabase } from '~/processors/database/postgres.provider' +import type { SnowflakeService } from '~/shared/id/snowflake.service' + +const snapshot = (title: string) => ({ + content: null, + contentFormat: 'lexical', + images: null, + meta: null, + text: title, + title, + typeSpecificData: null, +}) + +const bytes = (...values: number[]) => new Uint8Array(values) + +describe('DraftCrdtRepository (real PG)', () => { + let pool: Pool + let db: AppDatabase + let database: Awaited> + let drafts: DraftRepository + let repository: DraftCrdtRepository + + const seedDocument = async () => { + const { document, revision } = await drafts.createDocumentWithRoot( + DraftRefType.Post, + null, + snapshot('Root'), + false, + ) + const { branch } = await drafts.createBranch( + document.id, + revision.id, + snapshot('Root'), + true, + ) + return { branch, document, revision } + } + + beforeAll(async () => { + database = await createIsolatedPgDatabase() + pool = new Pool({ connectionString: database.getConnectionUri(), max: 4 }) + db = drizzle(pool, { schema }) as unknown as AppDatabase + const snowflake = new SnowflakeGenerator({ + workerId: 44, + }) as SnowflakeService + drafts = new DraftRepository(db, snowflake) + repository = new DraftCrdtRepository(db, snowflake) + }, 120_000) + + afterAll(async () => { + await pool?.end() + await database?.drop() + }) + + it('creates a doc once and returns the stored row on a second create', async () => { + const { document, revision } = await seedDocument() + + const first = await repository.createDoc({ + documentId: document.id, + fields: { meta: null, typeSpecificData: { slug: 'a' } }, + snapshot: bytes(1, 2, 3), + snapshotVersion: bytes(9), + syncedRevisionId: revision.id, + }) + const second = await repository.createDoc({ + documentId: document.id, + fields: null, + snapshot: bytes(7), + snapshotVersion: bytes(7), + syncedRevisionId: null, + }) + + expect(first.snapshot).toEqual(bytes(1, 2, 3)) + expect(second.snapshot).toEqual(bytes(1, 2, 3)) + expect(second.syncedRevisionId).toBe(revision.id) + expect(await repository.findDoc(document.id)).toMatchObject({ + fields: { meta: null, typeSpecificData: { slug: 'a' } }, + snapshotVersion: bytes(9), + }) + }) + + it('appends updates and compacts exactly the folded rows', async () => { + const { document } = await seedDocument() + await repository.createDoc({ + documentId: document.id, + fields: null, + snapshot: bytes(0), + snapshotVersion: bytes(0), + syncedRevisionId: null, + }) + const a = await repository.appendUpdate(document.id, bytes(1)) + const b = await repository.appendUpdate(document.id, bytes(2)) + + const loaded = await repository.findUpdates(document.id) + expect(loaded.map((row) => row.bytes)).toEqual([bytes(1), bytes(2)]) + + const late = await repository.appendUpdate(document.id, bytes(3)) + const compacted = await repository.compact(document.id, { + expectedVersion: bytes(0), + foldedIds: [a, b], + snapshot: bytes(1, 2), + snapshotVersion: bytes(2), + }) + + expect(compacted).toBe(true) + expect( + (await repository.findUpdates(document.id)).map((row) => row.id), + ).toEqual([late]) + expect((await repository.findDoc(document.id))?.snapshot).toEqual( + bytes(1, 2), + ) + }) + + it('refuses a compaction built from a stale snapshot', async () => { + const { document } = await seedDocument() + await repository.createDoc({ + documentId: document.id, + fields: null, + snapshot: bytes(0), + snapshotVersion: bytes(5), + syncedRevisionId: null, + }) + const a = await repository.appendUpdate(document.id, bytes(1)) + + const compacted = await repository.compact(document.id, { + expectedVersion: bytes(4), + foldedIds: [a], + snapshot: bytes(1), + snapshotVersion: bytes(6), + }) + + expect(compacted).toBe(false) + expect(await repository.findUpdates(document.id)).toHaveLength(1) + expect((await repository.findDoc(document.id))?.snapshot).toEqual(bytes(0)) + }) + + it('merges fields per top-level key without clobbering other keys', async () => { + const { document } = await seedDocument() + await repository.createDoc({ + documentId: document.id, + fields: { + meta: { cover: 'a.png', keep: 1 }, + typeSpecificData: { slug: 'old', tags: ['x'] }, + }, + snapshot: bytes(0), + snapshotVersion: bytes(0), + syncedRevisionId: null, + }) + + await repository.mergeFields(document.id, { + typeSpecificData: { slug: 'new' }, + }) + const merged = await repository.mergeFields(document.id, { + meta: { cover: null }, + }) + + expect(merged).toEqual({ + meta: { cover: null, keep: 1 }, + typeSpecificData: { slug: 'new', tags: ['x'] }, + }) + }) + + it('merges into a section that is still null', async () => { + const { document } = await seedDocument() + await repository.createDoc({ + documentId: document.id, + fields: { meta: null, typeSpecificData: null }, + snapshot: bytes(0), + snapshotVersion: bytes(0), + syncedRevisionId: null, + }) + + expect( + await repository.mergeFields(document.id, { meta: { cover: 'c' } }), + ).toEqual({ meta: { cover: 'c' }, typeSpecificData: null }) + }) + + it('chains checkpoints linearly and moves the working copy head to the last', async () => { + const { branch, revision } = await seedDocument() + const frontiers = [{ counter: 3, peer: '12' as const }] + + const { branch: moved, revisions } = await repository.appendCheckpoints( + branch.id, + [ + { frontiers, kind: 'legacy-save', snapshot: snapshot('Own') }, + { frontiers: null, kind: 'save', snapshot: snapshot('Head') }, + ], + ) + + const [own, head] = revisions + expect(own).toMatchObject({ + frontiers, + kind: 'legacy-save', + parentRevisionId: revision.id, + title: 'Own', + }) + expect(head.parentRevisionId).toBe(own.id) + expect(moved.headRevisionId).toBe(head.id) + const [row] = await db + .select() + .from(schema.drafts) + .where(eq(schema.drafts.id, branch.id)) + expect(row.headRevisionId).toBe(head.id) + }) + + it('serializes work on the same document under the document lock', async () => { + const { document } = await seedDocument() + const order: string[] = [] + let release!: () => void + const gate = new Promise((resolve) => { + release = resolve + }) + + const first = repository.withDocumentLock(document.id, async () => { + order.push('first:start') + await gate + order.push('first:end') + }) + await new Promise((resolve) => setTimeout(resolve, 50)) + const second = repository.withDocumentLock(document.id, async () => { + order.push('second') + }) + await new Promise((resolve) => setTimeout(resolve, 100)) + release() + await Promise.all([first, second]) + + expect(order).toEqual(['first:start', 'first:end', 'second']) + }) +}) diff --git a/apps/core/test/src/modules/draft/draft-crdt.service.pg.e2e.spec.ts b/apps/core/test/src/modules/draft/draft-crdt.service.pg.e2e.spec.ts new file mode 100644 index 00000000000..9491a0c4416 --- /dev/null +++ b/apps/core/test/src/modules/draft/draft-crdt.service.pg.e2e.spec.ts @@ -0,0 +1,526 @@ +import { SnowflakeGenerator } from '@mx-space/db-schema/id' +import * as schema from '@mx-space/db-schema/schema' +import { eq } from 'drizzle-orm' +import { drizzle } from 'drizzle-orm/node-postgres' +import { LoroDoc } from 'loro-crdt' +import { Pool } from 'pg' +import { createIsolatedPgDatabase } from 'test/helper/pg-testcontainer' +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' + +import { AppErrorCode } from '~/common/errors/app-error-code' +import { DraftRefType } from '~/modules/draft/draft.enum' +import { DraftRepository } from '~/modules/draft/draft.repository' +import type { RevisionSnapshot } from '~/modules/draft/draft.types' +import { editDocAt, readDocContent } from '~/modules/draft/draft-crdt.doc' +import { DraftCrdtRepository } from '~/modules/draft/draft-crdt.repository' +import { DraftCrdtService } from '~/modules/draft/draft-crdt.service' +import type { AppDatabase } from '~/processors/database/postgres.provider' +import type { SnowflakeService } from '~/shared/id/snowflake.service' + +const paragraph = (value: string) => ({ + children: [ + { + detail: 0, + format: 0, + mode: 'normal', + style: '', + text: value, + type: 'text', + version: 1, + }, + ], + direction: null, + format: '', + indent: 0, + textFormat: 0, + textStyle: '', + type: 'paragraph', + version: 1, +}) + +const body = (...values: string[]) => + JSON.stringify({ + root: { + children: values.map(paragraph), + direction: null, + format: '', + indent: 0, + type: 'root', + version: 1, + }, + }) + +const texts = (content: string | null) => + ( + JSON.parse(content ?? '{"root":{"children":[]}}') as { + root: { children: { children: { text: string }[] }[] } + } + ).root.children.map((child) => child.children.map((t) => t.text).join('')) + +const lexical = ( + title: string, + values: string[], + typeSpecificData: Record | null = { slug: 's' }, +): RevisionSnapshot => ({ + content: body(...values), + contentFormat: 'lexical', + images: null, + meta: null, + text: values.join('\n\n'), + title, + typeSpecificData, +}) + +describe('DraftCrdtService (real PG)', () => { + let pool: Pool + let db: AppDatabase + let database: Awaited> + let drafts: DraftRepository + let crdt: DraftCrdtRepository + let service: DraftCrdtService + const fileReferences = { + removeReferencesForDocument: vi.fn(), + updateReferencesForDocument: vi.fn(), + } + + const seed = async (published = lexical('Pub', ['p1', 'p2'])) => { + const { document, revision } = await drafts.createDocumentWithRoot( + DraftRefType.Post, + null, + published, + true, + ) + const { branch } = await drafts.createBranch( + document.id, + revision.id, + published, + true, + ) + return { branch, document, published: revision } + } + + const clientOf = async (branchId: string) => { + const doc = new LoroDoc() + doc.import(await service.exportUpdates(branchId)) + return doc + } + + const clientEdit = async ( + branchId: string, + doc: LoroDoc, + title: string, + values: string[], + ) => { + const from = doc.oplogVersion() + doc.import( + editDocAt( + doc, + doc.frontiers(), + { content: body(...values), title }, + 'human:owner', + ), + ) + await service.importUpdate(branchId, doc.export({ from, mode: 'update' })) + } + + beforeAll(async () => { + database = await createIsolatedPgDatabase() + pool = new Pool({ connectionString: database.getConnectionUri(), max: 4 }) + db = drizzle(pool, { schema }) as unknown as AppDatabase + const snowflake = new SnowflakeGenerator({ + workerId: 45, + }) as SnowflakeService + drafts = new DraftRepository(db, snowflake) + crdt = new DraftCrdtRepository(db, snowflake) + service = new DraftCrdtService(drafts, crdt, fileReferences as never) + }, 120_000) + + afterAll(async () => { + await pool?.end() + await database?.drop() + }) + + it('imports a document lazily with an import checkpoint the doc matches', async () => { + const { branch, document } = await seed() + + const workingCopy = await service.workingCopy(branch.id) + + const head = await drafts.findRevisionById(workingCopy.headRevisionId) + expect(head).toMatchObject({ kind: 'import', title: 'Pub' }) + expect(head?.frontiers?.length).toBeGreaterThan(0) + expect(texts(head!.content)).toEqual(['p1', 'p2']) + const row = await crdt.findDoc(document.id) + expect(row?.syncedRevisionId).toBe(head!.id) + expect(row?.fields).toEqual({ meta: null, typeSpecificData: { slug: 's' } }) + expect(readDocContent(await clientOf(branch.id)).title).toBe('Pub') + }) + + it('keeps one working copy, archiving older branches and pinning their follow shares', async () => { + const { branch: older, document, published } = await seed() + const { branch: newer } = await drafts.createBranch( + document.id, + published.id, + lexical('Newer', ['n']), + false, + ) + await drafts.upsertShare(document.id, 'tok-older', { + draftId: older.id, + mode: 'follow', + }) + + const workingCopy = await service.workingCopy(older.id) + + expect(workingCopy.id).toBe(newer.id) + expect((await drafts.findBranchById(older.id))?.status).toBe('archived') + expect(await drafts.findShareByToken('tok-older')).toMatchObject({ + draftId: null, + mode: 'pinned', + revisionId: older.headRevisionId, + }) + }) + + it('creates a working copy from the published revision when none is active', async () => { + const { branch, document } = await seed() + await drafts.archiveBranch(branch.id) + + const workingCopy = await service.workingCopy(branch.id) + + expect(workingCopy.id).not.toBe(branch.id) + expect(workingCopy.status).toBe('active') + expect(workingCopy.documentId).toBe(document.id) + }) + + it('syncs peer updates and checkpoints the merged content, skipping no-op saves', async () => { + const { branch } = await seed() + const tabA = await clientOf(branch.id) + const tabB = await clientOf(branch.id) + + await clientEdit(branch.id, tabA, 'Pub', ['p1 from A', 'p2']) + await clientEdit(branch.id, tabB, 'Pub', ['p1', 'p2 from B']) + const { revision } = await service.createCheckpoint(branch.id, 'save') + const again = await service.createCheckpoint(branch.id, 'save') + + expect(revision.kind).toBe('save') + expect(texts(revision.content)).toEqual(['p1 from A', 'p2 from B']) + expect(revision.text).toContain('p1 from A') + expect(again.revision.id).toBe(revision.id) + expect(fileReferences.updateReferencesForDocument).toHaveBeenCalled() + }) + + it('serves only the updates a client lacks', async () => { + const { branch } = await seed() + const tab = await clientOf(branch.id) + const since = tab.oplogVersion().encode() + const other = await clientOf(branch.id) + await clientEdit(branch.id, other, 'Pub', ['p1', 'p2', 'p3']) + + tab.import(await service.exportUpdates(branch.id, since)) + + expect(texts(readDocContent(tab).content)).toEqual(['p1', 'p2', 'p3']) + }) + + it('compacts a long update log into the snapshot without losing edits', async () => { + const { branch, document } = await seed() + const tab = await clientOf(branch.id) + for (let i = 0; i < 100; i++) { + await clientEdit(branch.id, tab, 'Pub', ['p1', `p2 ${i}`]) + } + + await service.workingCopy(branch.id) + await vi.waitFor(async () => { + expect(await crdt.findUpdates(document.id)).toHaveLength(0) + }) + + const fresh = await clientOf(branch.id) + expect(texts(readDocContent(fresh).content)).toEqual(['p1', 'p2 99']) + }) + + it('reloads when another replica compacts between reading the snapshot and the updates', async () => { + const { branch, document } = await seed() + const tab = await clientOf(branch.id) + await clientEdit(branch.id, tab, 'Pub', ['p1', 'p2 edited']) + const racing = Object.create(crdt) as DraftCrdtRepository + let raced = false + racing.findUpdates = async (documentId) => { + if (!raced) { + raced = true + const row = await crdt.findDoc(documentId) + const updates = await crdt.findUpdates(documentId) + const full = new LoroDoc() + full.import(row!.snapshot) + full.importBatch(updates.map((update) => update.bytes)) + await crdt.compact(documentId, { + expectedVersion: row!.snapshotVersion, + foldedIds: updates.map((update) => update.id), + snapshot: full.export({ mode: 'snapshot' }), + snapshotVersion: full.oplogVersion().encode(), + }) + } + return crdt.findUpdates(documentId) + } + const replica = new DraftCrdtService( + drafts, + racing, + fileReferences as never, + ) + + const { revision } = await replica.createCheckpoint(branch.id, 'save') + + expect(raced).toBe(true) + expect(texts(revision.content)).toEqual(['p1', 'p2 edited']) + expect(await crdt.findUpdates(document.id)).toHaveLength(0) + }) + + it('rejects an update from another document lineage without storing it', async () => { + const { branch, document } = await seed() + await service.workingCopy(branch.id) + const { branch: strangerBranch } = await seed(lexical('X', ['x'])) + const stranger = await clientOf(strangerBranch.id) + + await expect( + service.importUpdate(branch.id, stranger.export({ mode: 'update' })), + ).rejects.toMatchObject({ + code: AppErrorCode.DRAFT_CRDT_UPDATE_REJECTED, + }) + expect(await crdt.findUpdates(document.id)).toHaveLength(0) + }) + + it('rejects an update not signed by the author the caller expects', async () => { + const { branch, document } = await seed() + await service.workingCopy(branch.id) + const tab = await clientOf(branch.id) + const from = tab.oplogVersion() + tab.import( + editDocAt( + tab, + tab.frontiers(), + { content: body('p1', 'p2', 'p3'), title: 'T' }, + 'human:owner', + ), + ) + + await expect( + service.importUpdate( + branch.id, + tab.export({ from, mode: 'update' }), + undefined, + 'human:reader-1', + ), + ).rejects.toMatchObject({ + code: AppErrorCode.DRAFT_CRDT_UPDATE_REJECTED, + }) + expect(await crdt.findUpdates(document.id)).toHaveLength(0) + }) + + it('applies a legacy write at its base version and returns its own checkpoint', async () => { + const { branch } = await seed() + const workingCopy = await service.workingCopy(branch.id) + const base = workingCopy.headRevisionId + const tab = await clientOf(branch.id) + await clientEdit(branch.id, tab, 'Pub', ['p1', 'p2 unfinished']) + await service.createCheckpoint(branch.id, 'save') + + const { branch: after, revision } = await service.legacyWrite( + branch.id, + base, + lexical('Pub', ['p1 cli', 'p2'], { slug: 'cli' }), + ) + + expect(texts(revision.content)).toEqual(['p1 cli', 'p2']) + expect(revision.typeSpecificData).toEqual({ slug: 'cli' }) + expect(revision.kind).toBe('legacy-save') + const head = await drafts.findRevisionById(after.headRevisionId) + expect(texts(head!.content)).toEqual(['p1 cli', 'p2 unfinished']) + expect(head!.typeSpecificData).toEqual({ slug: 'cli' }) + expect(head!.parentRevisionId).toBe(revision.id) + }) + + it('three-way merges a write based on the revision the doc was imported from', async () => { + const { branch, published } = await seed() + await service.workingCopy(branch.id) + const tab = await clientOf(branch.id) + await clientEdit(branch.id, tab, 'Pub', ['p1', 'p2 owner']) + + const { branch: after } = await service.legacyWrite( + branch.id, + published.id, + lexical('Pub', ['p1 cli', 'p2']), + ) + + const head = await drafts.findRevisionById(after.headRevisionId) + expect(texts(head!.content)).toEqual(['p1 cli', 'p2 owner']) + }) + + it('rejects a Markdown legacy write to a Loro document', async () => { + const { branch } = await seed() + await service.workingCopy(branch.id) + + await expect( + service.legacyWrite(branch.id, null, { + ...lexical('M', []), + content: null, + contentFormat: 'markdown', + text: '# md', + }), + ).rejects.toMatchObject({ + code: AppErrorCode.DRAFT_CRDT_FORMAT_UNSUPPORTED, + }) + }) + + it('folds in a save an old pod made outside the Loro doc', async () => { + const { branch } = await seed() + const workingCopy = await service.workingCopy(branch.id) + const tab = await clientOf(branch.id) + await clientEdit(branch.id, tab, 'Pub', ['p1', 'p2 new']) + const saved = await drafts.saveBranch( + branch.id, + workingCopy.headRevisionId, + lexical('Old pod', ['p1 old pod', 'p2']), + ) + expect(saved.kind).toBe('ok') + + const reconciled = await service.workingCopy(branch.id) + + const head = await drafts.findRevisionById(reconciled.headRevisionId) + expect(head?.kind).toBe('legacy-save') + expect(head?.title).toBe('Old pod') + expect(texts(head!.content)).toEqual(['p1 old pod', 'p2 new']) + }) + + it('restores a checkpoint as a new commit, fields included', async () => { + const { branch } = await seed() + const imported = await service.workingCopy(branch.id) + const tab = await clientOf(branch.id) + await clientEdit(branch.id, tab, 'Changed', ['changed']) + await service.updateFields(branch.id, { typeSpecificData: { slug: 'x' } }) + await service.createCheckpoint(branch.id, 'save') + + const { revision } = await service.restore( + branch.id, + imported.headRevisionId, + ) + + expect(revision.kind).toBe('restore') + expect(revision.title).toBe('Pub') + expect(texts(revision.content)).toEqual(['p1', 'p2']) + expect(revision.typeSpecificData).toEqual({ slug: 's' }) + }) + + it('reports every stored change with its origin', async () => { + const { branch } = await seed() + const working = await service.workingCopy(branch.id) + const seen: { documentId: string; origin?: string; update: Uint8Array }[] = + [] + const stop = service.onDocumentChanged((change) => { + seen.push(change) + }) + const tab = await clientOf(branch.id) + const mirror = await clientOf(branch.id) + const from = tab.oplogVersion() + tab.import( + editDocAt( + tab, + tab.frontiers(), + { content: body('typed'), title: 'Pub' }, + 'human:owner', + ), + ) + await service.importUpdate( + branch.id, + tab.export({ from, mode: 'update' }), + 'conn-1', + ) + await service.restore(branch.id, working.headRevisionId) + stop() + + expect(seen.map((change) => change.origin)).toEqual(['conn-1', undefined]) + expect( + seen.every((change) => change.documentId === branch.documentId), + ).toBe(true) + for (const change of seen) mirror.import(change.update) + expect(texts(readDocContent(mirror).content)).toEqual(['p1', 'p2']) + }) + + it('undoes one step while later edits survive', async () => { + const { branch } = await seed() + await service.workingCopy(branch.id) + const tab = await clientOf(branch.id) + await clientEdit(branch.id, tab, 'Pub', ['p1', 'p2', 'agent']) + const { changes } = await service.history(branch.id) + const step = changes.find((change) => change.message === 'human:owner')! + await clientEdit(branch.id, tab, 'Pub', ['p1 later', 'p2', 'agent']) + + const { revision } = await service.undo(branch.id, step.id, step.deps) + + expect(revision.kind).toBe('undo') + expect(texts(revision.content)).toEqual(['p1 later', 'p2']) + }) + + it('merges metadata fields into the next checkpoint', async () => { + const { branch } = await seed() + await service.workingCopy(branch.id) + + await service.updateFields(branch.id, { + meta: { cover: 'c.png' }, + typeSpecificData: { tags: ['a'] }, + }) + const { revision } = await service.createCheckpoint(branch.id, 'save') + + expect(revision.meta).toEqual({ cover: 'c.png' }) + expect(revision.typeSpecificData).toEqual({ slug: 's', tags: ['a'] }) + }) + + it('lists checkpoints and change log newest first', async () => { + const { branch } = await seed() + await service.workingCopy(branch.id) + const tab = await clientOf(branch.id) + await clientEdit(branch.id, tab, 'Pub', ['p1', 'p2', 'p3']) + await service.createCheckpoint(branch.id, 'save') + + const { changes, checkpoints } = await service.history(branch.id) + + expect(checkpoints.map((row) => row.kind)).toEqual(['save', 'import', null]) + expect(checkpoints[0]).not.toHaveProperty('content') + expect(checkpoints[0]).toMatchObject({ title: 'Pub' }) + expect(changes[0].message).toBe('human:owner') + expect(changes.at(-1)?.message).toMatch(/^import /) + }) + + it('discards the working copy back to the published revision', async () => { + const { branch, published } = await seed() + await service.workingCopy(branch.id) + const tab = await clientOf(branch.id) + await clientEdit(branch.id, tab, 'Draft', ['unpublished']) + + const discarded = await service.discard(branch.id) + + expect(discarded?.status).toBe('active') + const head = await drafts.findRevisionById(discarded!.headRevisionId) + expect(head?.kind).toBe('restore') + expect(head?.title).toBe(published.title) + expect(texts(head!.content)).toEqual(['p1', 'p2']) + }) + + it('archives a never-published working copy on discard', async () => { + const { document, revision } = await drafts.createDocumentWithRoot( + DraftRefType.Note, + null, + lexical('New', ['n']), + false, + ) + const { branch } = await drafts.createBranch( + document.id, + revision.id, + lexical('New', ['n']), + true, + ) + await service.workingCopy(branch.id) + + expect(await service.discard(branch.id)).toBeNull() + const [row] = await db + .select() + .from(schema.drafts) + .where(eq(schema.drafts.id, branch.id)) + expect(row.status).toBe('archived') + }) +}) diff --git a/apps/core/test/src/modules/draft/draft.service.crdt.pg.e2e.spec.ts b/apps/core/test/src/modules/draft/draft.service.crdt.pg.e2e.spec.ts new file mode 100644 index 00000000000..9341646147b --- /dev/null +++ b/apps/core/test/src/modules/draft/draft.service.crdt.pg.e2e.spec.ts @@ -0,0 +1,291 @@ +import { SnowflakeGenerator } from '@mx-space/db-schema/id' +import * as schema from '@mx-space/db-schema/schema' +import { drizzle } from 'drizzle-orm/node-postgres' +import { LoroDoc } from 'loro-crdt' +import { Pool } from 'pg' +import { createIsolatedPgDatabase } from 'test/helper/pg-testcontainer' +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' + +import { AppErrorCode } from '~/common/errors/app-error-code' +import { BusinessEvents } from '~/constants/business-event.constant' +import { DraftRefType } from '~/modules/draft/draft.enum' +import { DraftRepository } from '~/modules/draft/draft.repository' +import { DraftService } from '~/modules/draft/draft.service' +import type { RevisionSnapshot } from '~/modules/draft/draft.types' +import { editDocAt } from '~/modules/draft/draft-crdt.doc' +import { DraftCrdtRepository } from '~/modules/draft/draft-crdt.repository' +import { DraftCrdtService } from '~/modules/draft/draft-crdt.service' +import type { AppDatabase } from '~/processors/database/postgres.provider' +import type { SnowflakeService } from '~/shared/id/snowflake.service' + +const body = (...values: string[]) => + JSON.stringify({ + root: { + children: values.map((value) => ({ + children: [ + { + detail: 0, + format: 0, + mode: 'normal', + style: '', + text: value, + type: 'text', + version: 1, + }, + ], + direction: null, + format: '', + indent: 0, + textFormat: 0, + textStyle: '', + type: 'paragraph', + version: 1, + })), + direction: null, + format: '', + indent: 0, + type: 'root', + version: 1, + }, + }) + +const texts = (content: string | null) => + ( + JSON.parse(content!) as { + root: { children: { children: { text: string }[] }[] } + } + ).root.children.map((child) => child.children.map((t) => t.text).join('')) + +const lexical = (title: string, values: string[]): RevisionSnapshot => ({ + content: body(...values), + contentFormat: 'lexical', + images: null, + meta: null, + text: values.join('\n\n'), + title, + typeSpecificData: { slug: 's' }, +}) + +const markdown = (title: string): RevisionSnapshot => ({ + content: null, + contentFormat: 'markdown', + images: null, + meta: null, + text: title, + title, + typeSpecificData: null, +}) + +describe('DraftService on the Loro path (real PG)', () => { + let pool: Pool + let database: Awaited> + let drafts: DraftRepository + let service: DraftService + let refCounter = 7_100_000_000_000_000_000n + const emit = vi.fn() + const fileReferences = { + removeReferencesForDocument: vi.fn(), + updateReferencesForDocument: vi.fn(), + } + + const seed = async (published: RevisionSnapshot) => { + const refId = String(refCounter++) + const { document, revision } = await drafts.createDocumentWithRoot( + DraftRefType.Post, + refId, + published, + true, + ) + const { branch } = await drafts.createBranch( + document.id, + revision.id, + published, + true, + ) + return { branch, document, published: revision, refId } + } + + beforeAll(async () => { + database = await createIsolatedPgDatabase() + pool = new Pool({ connectionString: database.getConnectionUri(), max: 6 }) + const db = drizzle(pool, { schema }) as unknown as AppDatabase + const snowflake = new SnowflakeGenerator({ + workerId: 46, + }) as SnowflakeService + drafts = new DraftRepository(db, snowflake) + const crdt = new DraftCrdtService( + drafts, + new DraftCrdtRepository(db, snowflake), + fileReferences as never, + ) + service = new DraftService( + drafts, + fileReferences as never, + {} as never, + { emit } as never, + crdt, + ) + }, 120_000) + + afterAll(async () => { + await pool?.end() + await database?.drop() + }) + + it('never conflicts on a stale PUT and returns the write’s own checkpoint', async () => { + const { branch, published } = await seed(lexical('Pub', ['a', 'b'])) + await service.update(branch.id, { + data: lexical('Pub', ['a', 'b owner']), + expectedHeadRevisionId: published.id, + } as never) + + const view = await service.update(branch.id, { + data: lexical('Pub', ['a cli', 'b']), + expectedHeadRevisionId: published.id, + } as never) + + expect(texts(view.headRevision.content)).toEqual(['a cli', 'b']) + const stored = await drafts.findBranchById(view.id) + const head = await drafts.findRevisionById(stored!.headRevisionId) + expect(texts(head!.content)).toEqual(['a cli', 'b owner']) + expect(emit).toHaveBeenCalledWith( + BusinessEvents.DRAFT_UPDATE, + expect.objectContaining({ headRevisionId: stored!.headRevisionId }), + expect.anything(), + ) + }) + + it('writes a CLI save against the published base into the working copy', async () => { + const { branch, published, refId } = await seed(lexical('Pub', ['a'])) + + const view = await service.create({ + baseRevisionId: published.id, + data: lexical('Pub', ['a', 'cli']), + refId, + refType: DraftRefType.Post, + } as never) + + expect(view.id).toBe(branch.id) + expect(texts(view.headRevision.content)).toEqual(['a', 'cli']) + expect(await drafts.findBranchesByDocument(branch.documentId)).toHaveLength( + 1, + ) + }) + + it('keeps the branch flow and its conflicts for Markdown documents', async () => { + const { branch, published } = await seed(markdown('Md')) + await service.update(branch.id, { + data: markdown('Md 2'), + expectedHeadRevisionId: published.id, + } as never) + + await expect( + service.update(branch.id, { + data: markdown('Md 3'), + expectedHeadRevisionId: published.id, + } as never), + ).rejects.toMatchObject({ code: AppErrorCode.DRAFT_HEAD_CONFLICT }) + }) + + it('collapses a Lexical document to one branch when its context is read', async () => { + const { document, published, refId } = await seed(lexical('Pub', ['a'])) + await drafts.createBranch( + document.id, + published.id, + lexical('Other', ['o']), + false, + ) + + const context = await service.getContext(DraftRefType.Post, refId) + + expect(context.branches).toHaveLength(1) + expect(context.branches[0].headRevision.kind).toBe('import') + }) + + it('discards a Lexical working copy back to the published content', async () => { + const { branch } = await seed(lexical('Pub', ['a'])) + await service.update(branch.id, { + data: lexical('Draft', ['changed']), + expectedHeadRevisionId: branch.headRevisionId, + } as never) + + await service.delete(branch.id) + + const view = await service.findById(branch.id) + expect(view?.status).toBe('active') + expect(view?.headRevision).toMatchObject({ kind: 'restore', title: 'Pub' }) + }) + + it('syncs binary updates and broadcasts the working copy', async () => { + const { branch } = await seed(lexical('Pub', ['a'])) + const tab = new LoroDoc() + tab.import(await service.exportCrdt(branch.id)) + const from = tab.oplogVersion() + tab.import( + editDocAt( + tab, + tab.frontiers(), + { content: body('a', 'typed'), title: 'Pub' }, + 'human:owner', + ), + ) + emit.mockClear() + + await service.importCrdt(branch.id, tab.export({ from, mode: 'update' })) + const { headRevision } = await service.createCheckpoint(branch.id, 'save') + + expect(texts(headRevision.content)).toEqual(['a', 'typed']) + expect(emit).toHaveBeenCalledWith( + BusinessEvents.DRAFT_UPDATE, + expect.objectContaining({ branchId: branch.id }), + expect.anything(), + ) + }) + + it('merges live article edits into the working copy fields', async () => { + const { branch, refId } = await seed(lexical('Pub', ['a'])) + await service.findById(branch.id) + + await service.mergeArticleFields(DraftRefType.Post, refId, { + categoryId: 'cat-2', + pin: true, + }) + const view = await service.createCheckpoint(branch.id, 'save') + + expect(view.headRevision.typeSpecificData).toEqual({ + categoryId: 'cat-2', + pin: true, + slug: 's', + }) + }) + + it('leaves documents without a Loro doc alone on live edits', async () => { + const { branch, refId } = await seed(markdown('Md')) + + await service.mergeArticleFields(DraftRefType.Post, refId, { pin: true }) + + const view = await service.findById(branch.id) + expect(view?.headRevision.typeSpecificData).toBeNull() + }) + + it('opens a Lexical draft for editing at a fresh checkpoint', async () => { + const { branch } = await seed(lexical('Pub', ['a'])) + await service.findById(branch.id) + await service.updateFields(branch.id, { meta: { cover: 'late.png' } }) + + const view = await service.findForEditing(branch.id) + + expect(view?.headRevision.meta).toEqual({ cover: 'late.png' }) + expect((await service.findForEditing(branch.id))?.headRevisionId).toBe( + view?.headRevisionId, + ) + }) + + it('opens a Markdown draft for editing without touching it', async () => { + const { branch } = await seed(markdown('Md')) + + const view = await service.findForEditing(branch.id) + + expect(view?.headRevisionId).toBe(branch.headRevisionId) + }) +}) diff --git a/apps/core/test/src/modules/draft/draft.service.spec.ts b/apps/core/test/src/modules/draft/draft.service.spec.ts index e0a091563cc..660c318cdc9 100644 --- a/apps/core/test/src/modules/draft/draft.service.spec.ts +++ b/apps/core/test/src/modules/draft/draft.service.spec.ts @@ -74,6 +74,10 @@ const harness = () => { fileReferences as never, {} as never, { emit: vi.fn() } as never, + { + usesCrdt: vi.fn(async () => false), + workingCopyOfDocument: vi.fn(async () => null), + } as never, ) return { fileReferences, repository, service } } diff --git a/apps/core/test/src/modules/note/note.controller.e2e-spec.ts b/apps/core/test/src/modules/note/note.controller.e2e-spec.ts index 2bbf6a43449..9d739cf6c38 100644 --- a/apps/core/test/src/modules/note/note.controller.e2e-spec.ts +++ b/apps/core/test/src/modules/note/note.controller.e2e-spec.ts @@ -136,6 +136,8 @@ const createController = ( {} as any, enrichmentService as any, translationEntryService as any, + + { coAuthorsByRef: async () => new Map() } as any, ) return { diff --git a/apps/core/test/src/modules/post/post-paywall.e2e-spec.ts b/apps/core/test/src/modules/post/post-paywall.e2e-spec.ts index cccb2a8f5a5..b1e14679dbc 100644 --- a/apps/core/test/src/modules/post/post-paywall.e2e-spec.ts +++ b/apps/core/test/src/modules/post/post-paywall.e2e-spec.ts @@ -4,6 +4,7 @@ import { APP_GUARD } from '@nestjs/core' import { drizzle } from 'drizzle-orm/node-postgres' import { Pool } from 'pg' import { createIsolatedPgDatabase } from 'test/helper/pg-testcontainer' +import { collabRepositoryProvider } from 'test/mock/modules/collab.mock' import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' import { RolesGuard } from '~/common/guards/roles.guard' @@ -176,6 +177,7 @@ const authServiceMock = { const postModule: ModuleMetadata = { controllers: [PostController], providers: [ + collabRepositoryProvider, { provide: PostService, useValue: postService }, { provide: CountingService, diff --git a/apps/core/test/src/modules/post/post-skill.e2e-spec.ts b/apps/core/test/src/modules/post/post-skill.e2e-spec.ts index 93764ad6dbc..e9728f048a0 100644 --- a/apps/core/test/src/modules/post/post-skill.e2e-spec.ts +++ b/apps/core/test/src/modules/post/post-skill.e2e-spec.ts @@ -1,5 +1,6 @@ import { createE2EApp } from 'test/helper/create-e2e-app' import { authPassHeader } from 'test/mock/guard/auth.guard' +import { collabRepositoryProvider } from 'test/mock/modules/collab.mock' import { describe, expect, it, vi } from 'vitest' import { AiInsightsService } from '~/modules/ai/ai-insights/ai-insights.service' @@ -60,6 +61,7 @@ const snippetService = { const proxy = createE2EApp({ controllers: [PostController], providers: [ + collabRepositoryProvider, { provide: PostService, useValue: postService }, { provide: CountingService, diff --git a/apps/core/test/src/modules/post/post.controller.spec.ts b/apps/core/test/src/modules/post/post.controller.spec.ts index fdd7b96c09b..fc3321fc812 100644 --- a/apps/core/test/src/modules/post/post.controller.spec.ts +++ b/apps/core/test/src/modules/post/post.controller.spec.ts @@ -169,6 +169,8 @@ const createController = (opts: CreateControllerOptions = {}) => { translationEntryService as any, snippetService as any, entitlementService as any, + + { coAuthorsByRef: async () => new Map() } as any, ) return { diff --git a/apps/core/test/src/modules/publish/publish.service.spec.ts b/apps/core/test/src/modules/publish/publish.service.spec.ts index 71a8e1a4dff..2229301b168 100644 --- a/apps/core/test/src/modules/publish/publish.service.spec.ts +++ b/apps/core/test/src/modules/publish/publish.service.spec.ts @@ -42,13 +42,27 @@ const makeBranch = () => ({ status: 'active', }) -const harness = (options: { online?: boolean } = {}) => { +const harness = (options: { crdt?: boolean; online?: boolean } = {}) => { const branch = makeBranch() + const checkpoints = new Map>([ + [ + 'checkpoint-1', + { + ...snapshot, + documentId: 'document-1', + id: 'checkpoint-1', + kind: 'publish', + title: 'Checkpoint title', + }, + ], + ]) const order: string[] = [] let handler: TaskHandler | undefined let payload: PublishTaskPayload | undefined const drafts = { findById: vi.fn(async () => branch), + findRevisionById: vi.fn(async (id: string) => checkpoints.get(id) ?? null), + usesCrdt: vi.fn(async () => options.crdt ?? false), linkDocument: vi.fn(), recordPublication: vi.fn(async () => ({ kind: 'ok' })), setPublishAiResources: vi.fn(), @@ -104,13 +118,17 @@ const harness = (options: { online?: boolean } = {}) => { ) service.onModuleInit() - const create = (confirmDiverged = false, aiResources?: unknown) => + const create = ( + confirmDiverged = false, + aiResources?: unknown, + revisionId: string = branch.headRevisionId, + ) => service.create({ aiResources, branchId: branch.id, confirmDiverged, expectedPublishedRevisionId: branch.document.publishedRevisionId, - revisionId: branch.headRevisionId, + revisionId, } as never) const run = async (signal = new AbortController().signal) => { const setResult = vi.fn() @@ -142,6 +160,44 @@ const harness = (options: { online?: boolean } = {}) => { } } +describe('PublishService on a Loro working copy', () => { + it('publishes a checkpoint that is not the working copy head', async () => { + const { create, payload } = harness({ crdt: true }) + + await create(false, undefined, 'checkpoint-1') + + expect(payload()).toMatchObject({ + revisionId: 'checkpoint-1', + snapshot: { title: 'Checkpoint title' }, + }) + }) + + it('ignores the divergence confirmation for a working copy', async () => { + const { branch, create, tasks } = harness({ crdt: true }) + branch.relationToPublished = 'diverged' + + await create(false, undefined, 'checkpoint-1') + + expect(tasks.createTask).toHaveBeenCalled() + }) + + it('rejects a revision from another document', async () => { + const { create } = harness({ crdt: true }) + + await expect( + create(false, undefined, 'unknown-revision'), + ).rejects.toMatchObject({ code: AppErrorCode.CONTENT_REVISION_NOT_FOUND }) + }) + + it('still requires the head revision for a Markdown branch', async () => { + const { create } = harness() + + await expect( + create(false, undefined, 'checkpoint-1'), + ).rejects.toMatchObject({ code: AppErrorCode.DRAFT_HEAD_CONFLICT }) + }) +}) + describe('PublishService tree selection', () => { it('requires confirmation when publishing a branch from an older base', async () => { const { branch, create, tasks } = harness() diff --git a/apps/core/test/src/processors/gateway/e2e/collab.e2e-spec.ts b/apps/core/test/src/processors/gateway/e2e/collab.e2e-spec.ts new file mode 100644 index 00000000000..d36aeff752a --- /dev/null +++ b/apps/core/test/src/processors/gateway/e2e/collab.e2e-spec.ts @@ -0,0 +1,319 @@ +import { randomUUID } from 'node:crypto' + +import { redisHelper } from 'test/helper/redis-mock.helper' +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' + +import { BusinessEvents } from '~/constants/business-event.constant' +import { CollabRepository } from '~/modules/draft/collab.repository' +import { CollabService } from '~/modules/draft/collab.service' +import { + CollabRoomService, + CollabWsEvents, +} from '~/modules/draft/collab-room.service' +import { + collabAccessProvider, + collabRelayProvider, + CollabRoomWsHandler, +} from '~/modules/draft/collab-room.ws' +import { DraftService } from '~/modules/draft/draft.service' +import { + type DocumentChange, + DraftCrdtService, +} from '~/modules/draft/draft-crdt.service' + +import { + createGatewayApp, + type GatewayTestApp, + WsTestClient, +} from './gateway-e2e.harness' + +vi.setConfig({ testTimeout: 15_000 }) + +const listeners = new Set<(change: DocumentChange) => void>() +const crdtStub = { + onDocumentChanged: (listener: (change: DocumentChange) => void) => { + listeners.add(listener) + return () => listeners.delete(listener) + }, +} +const draftStub = { + documentIdOf: vi.fn(async () => 'doc-1'), + exportCrdtFor: vi.fn(async () => ({ + documentId: 'doc-1', + update: new Uint8Array([1]), + })), + importCrdt: vi.fn( + async (_id: string, update: Uint8Array, origin?: string) => { + for (const listener of listeners) { + listener({ documentId: 'doc-1', origin, update }) + } + }, + ), +} + +const ownerUser = { avatar: null, id: 'owner-1', name: 'Owner' } +const collabStub = { + identityOf: vi.fn(async (readerId: string) => ({ + kind: 'reader' as const, + readerId, + user: { avatar: null, id: readerId, name: `Reader ${readerId}` }, + })), + ownerIdentity: vi.fn(async () => ({ + kind: 'owner' as const, + user: ownerUser, + })), +} +const collaborators = new Set(['r1']) +const repositoryStub = { + isCollaborator: vi.fn(async (_documentId: string, readerId: string) => + collaborators.has(readerId), + ), + markContributed: vi.fn(async () => undefined), +} + +const b64 = (bytes: number[]) => Buffer.from(bytes).toString('base64') + +describe('collab room over the admin and web gateways', () => { + let testApp: GatewayTestApp + + beforeAll(async () => { + const helper = await redisHelper + const opts = helper.CacheService.getClient().options + testApp = await createGatewayApp( + { host: opts.host as string, port: opts.port as number }, + [ + CollabRoomService, + CollabRoomWsHandler, + collabRelayProvider, + collabAccessProvider, + { provide: DraftService, useValue: draftStub }, + { provide: DraftCrdtService, useValue: crdtStub }, + { provide: CollabService, useValue: collabStub }, + { provide: CollabRepository, useValue: repositoryStub }, + ], + ) + }) + + afterAll(async () => { + await testApp.close() + }) + + const connect = async (key: string) => { + testApp.authService.verifyApiKey.mockResolvedValueOnce({ + referenceId: key, + }) + testApp.authService.isOwnerReaderId.mockResolvedValueOnce(true) + const client = await WsTestClient.connect(testApp.adminUrl(), { + headers: { 'x-api-key': key }, + }) + await client.waitFor((e) => e.event === BusinessEvents.GATEWAY_CONNECT) + return client + } + + const connectReader = async (readerId: string | null) => { + if (readerId) { + testApp.authService.getSessionUserFromHeaders.mockResolvedValueOnce({ + user: { id: readerId }, + } as never) + } + const client = await WsTestClient.connect(testApp.webUrl(), { + headers: { cookie: 'better-auth.session_token=x' }, + }) + await client.waitFor((e) => e.event === BusinessEvents.GATEWAY_CONNECT) + return client + } + + const request = async ( + client: WsTestClient, + event: string, + payload: unknown, + ) => { + const id = randomUUID() + client.send(event, payload, id) + return (await client.waitForAck(id)).payload as Record + } + + it('relays updates and presence to the other member and announces leaving', async () => { + const alice = await connect('alice') + const bob = await connect('bob') + try { + expect( + await request(alice, CollabWsEvents.join, { draftId: 'd1' }), + ).toMatchObject({ + data: { documentId: 'doc-1', update: b64([1]) }, + ok: true, + }) + await request(bob, CollabWsEvents.join, { draftId: 'd1' }) + await alice.waitFor((e) => e.event === CollabWsEvents.peerJoined) + + expect( + await request(alice, CollabWsEvents.update, { + draftId: 'd1', + update: b64([4, 2]), + }), + ).toEqual({ ok: true }) + const update = await bob.waitFor((e) => e.event === CollabWsEvents.update) + expect(update.payload).toEqual({ + documentId: 'doc-1', + update: b64([4, 2]), + }) + await alice.assertNotDelivered((e) => e.event === CollabWsEvents.update) + + await request(bob, CollabWsEvents.presence, { + caret: { id: 'p1', offset: 1 }, + draftId: 'd1', + }) + const presence = await alice.waitFor( + (e) => e.event === CollabWsEvents.presence, + ) + expect(presence.payload).toMatchObject({ + caret: { id: 'p1', offset: 1 }, + documentId: 'doc-1', + entry: null, + user: ownerUser, + }) + + await alice.closeAndWait() + const left = await bob.waitFor((e) => e.event === CollabWsEvents.peerLeft) + expect(left.payload).toMatchObject({ documentId: 'doc-1' }) + } finally { + await bob.closeAndWait() + } + }) + + it('ignores collab frames from a socket that has not authenticated', async () => { + testApp.authService.verifyApiKey.mockReturnValueOnce( + new Promise(() => undefined) as never, + ) + const stranger = await WsTestClient.connect(testApp.adminUrl(), { + headers: { 'x-api-key': 'pending' }, + }) + try { + draftStub.importCrdt.mockClear() + draftStub.exportCrdtFor.mockClear() + expect( + await request(stranger, CollabWsEvents.join, { draftId: 'd1' }), + ).toEqual({ code: 'NOT_AUTHENTICATED', ok: false }) + expect( + await request(stranger, CollabWsEvents.update, { + draftId: 'd1', + update: b64([1]), + }), + ).toEqual({ code: 'NOT_AUTHENTICATED', ok: false }) + expect(draftStub.exportCrdtFor).not.toHaveBeenCalled() + expect(draftStub.importCrdt).not.toHaveBeenCalled() + } finally { + await stranger.closeAndWait() + } + }) + + it('lets a co-author on the web socket edit with the owner in one room', async () => { + const owner = await connect('owner') + const reader = await connectReader('r1') + try { + await request(owner, CollabWsEvents.join, { draftId: 'd1' }) + expect( + await request(reader, CollabWsEvents.join, { draftId: 'd1' }), + ).toMatchObject({ data: { documentId: 'doc-1' }, ok: true }) + await owner.waitFor((e) => e.event === CollabWsEvents.peerJoined) + + draftStub.importCrdt.mockClear() + await request(reader, CollabWsEvents.update, { + draftId: 'd1', + update: b64([3]), + }) + expect(draftStub.importCrdt).toHaveBeenCalledWith( + 'd1', + new Uint8Array([3]), + expect.any(String), + 'human:r1', + ) + const relayed = await owner.waitFor( + (e) => e.event === CollabWsEvents.update, + ) + expect(relayed.payload).toEqual({ documentId: 'doc-1', update: b64([3]) }) + + await request(owner, CollabWsEvents.update, { + draftId: 'd1', + update: b64([4]), + }) + const back = await reader.waitFor( + (e) => e.event === CollabWsEvents.update, + ) + expect(back.payload).toEqual({ documentId: 'doc-1', update: b64([4]) }) + + await request(reader, CollabWsEvents.presence, { + caret: null, + draftId: 'd1', + }) + const presence = await owner.waitFor( + (e) => e.event === CollabWsEvents.presence, + ) + expect(presence.payload).toMatchObject({ + user: { id: 'r1', name: 'Reader r1' }, + }) + + testApp.moduleRef.get(CollabRoomService).revoke('doc-1', 'r1') + const revoked = await reader.waitFor( + (e) => e.event === CollabWsEvents.revoked, + ) + expect(revoked.payload).toEqual({ documentId: 'doc-1' }) + await owner.waitFor((e) => e.event === CollabWsEvents.peerLeft) + expect( + await request(reader, CollabWsEvents.update, { + draftId: 'd1', + update: b64([5]), + }), + ).toEqual({ code: 'NOT_JOINED', ok: false }) + } finally { + await owner.closeAndWait() + await reader.closeAndWait() + } + }) + + it('refuses web sockets that are anonymous or not collaborators', async () => { + const anonymous = await connectReader(null) + const stranger = await connectReader('r2') + try { + expect( + await request(anonymous, CollabWsEvents.join, { draftId: 'd1' }), + ).toEqual({ code: 'NOT_AUTHENTICATED', ok: false }) + expect( + await request(stranger, CollabWsEvents.join, { draftId: 'd1' }), + ).toEqual({ code: 'FORBIDDEN', ok: false }) + } finally { + await anonymous.closeAndWait() + await stranger.closeAndWait() + } + }) + + it('keeps collab rooms out of the public room join', async () => { + const reader = await connectReader(null) + try { + expect( + await request(reader, 'room.join', { room: 'collab:doc-1' }), + ).toEqual({ code: 'ROOM_INVALID', ok: false }) + } finally { + await reader.closeAndWait() + } + }) + + it('waits for the session lookup before answering an early join', async () => { + testApp.authService.getSessionUserFromHeaders.mockImplementationOnce( + () => + new Promise((resolve) => + setTimeout(() => resolve({ user: { id: 'r1' } } as never), 300), + ), + ) + const reader = await WsTestClient.connect(testApp.webUrl(), { + headers: { cookie: 'better-auth.session_token=x' }, + }) + try { + expect( + await request(reader, CollabWsEvents.join, { draftId: 'd1' }), + ).toMatchObject({ ok: true }) + } finally { + await reader.closeAndWait() + } + }) +}) diff --git a/apps/core/test/src/processors/gateway/e2e/gateway-e2e.harness.ts b/apps/core/test/src/processors/gateway/e2e/gateway-e2e.harness.ts index df5e86bd964..77a46b4e61b 100644 --- a/apps/core/test/src/processors/gateway/e2e/gateway-e2e.harness.ts +++ b/apps/core/test/src/processors/gateway/e2e/gateway-e2e.harness.ts @@ -28,6 +28,7 @@ import { AdminWsHandlerRegistry } from '~/processors/gateway/admin/admin-ws-hand import { AdminEventsGateway } from '~/processors/gateway/admin/events.gateway' import { GatewayService } from '~/processors/gateway/gateway.service' import { WebEventsGateway } from '~/processors/gateway/web/events.gateway' +import { WebWsHandlerRegistry } from '~/processors/gateway/web/web-ws-handler.registry' import { createWsAdapter } from '~/processors/gateway/ws/ws-adapter.factory' import { WsBusService } from '~/processors/gateway/ws/ws-bus.service' import { WsInboundEvents } from '~/processors/gateway/ws/ws-events' @@ -115,6 +116,7 @@ export async function createGatewayApp( { provide: APP_INTERCEPTOR, useClass: ResponseInterceptor }, { provide: APP_INTERCEPTOR, useClass: IdempotenceInterceptor }, AdminWsHandlerRegistry, + WebWsHandlerRegistry, ...extraProviders, ], }) diff --git a/apps/core/test/src/processors/gateway/web/events.gateway.spec.ts b/apps/core/test/src/processors/gateway/web/events.gateway.spec.ts index d62a963d5ec..32cdeb07717 100644 --- a/apps/core/test/src/processors/gateway/web/events.gateway.spec.ts +++ b/apps/core/test/src/processors/gateway/web/events.gateway.spec.ts @@ -1,6 +1,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import { WebEventsGateway } from '~/processors/gateway/web/events.gateway' +import { WebWsHandlerRegistry } from '~/processors/gateway/web/web-ws-handler.registry' const flush = () => new Promise((resolve) => setTimeout(resolve, 0)) @@ -101,6 +102,7 @@ describe('WebEventsGateway connection tracking', () => { fakes.redisService as any, fakes.gatewayService as any, authService as any, + new WebWsHandlerRegistry(), fakes.bus as any, fakes.presence as any, ) diff --git a/apps/core/vite.config.ts b/apps/core/vite.config.ts index 92cf8c152f7..16f08ee320b 100644 --- a/apps/core/vite.config.ts +++ b/apps/core/vite.config.ts @@ -89,7 +89,7 @@ export default defineConfig(({ command }) => { entryFileNames: '[name].mjs', chunkFileNames: 'chunks/[name]-[hash].mjs', }, - external: ['sharp', 'ws'], + external: ['sharp', 'ws', 'loro-crdt'], }, }, }), diff --git a/dockerfile b/dockerfile index b05cb6a5b38..a37dc8e185f 100644 --- a/dockerfile +++ b/dockerfile @@ -47,9 +47,9 @@ COPY --from=builder /app/out . RUN npm i sharp -g RUN npm i sharp -# WsAdapter and sandbox workers load these packages outside the Vite bundle. +# WsAdapter, sandbox workers and loro-crdt (reads its wasm next to itself) load outside the Vite bundle. # The production image only copies the Vite bundle, so install it next to sharp. -RUN npm i ws@8.21.3 undici@8.11.2 +RUN npm i ws@8.21.3 undici@8.11.2 loro-crdt@1.16.4 COPY --chmod=755 docker-entrypoint.sh . COPY docker-browser-watchdog.mjs . diff --git a/docs/superpowers/specs/2026-10-25-collab-ai-peer-design.md b/docs/superpowers/specs/2026-10-25-collab-ai-peer-design.md new file mode 100644 index 00000000000..0b507dd22dc --- /dev/null +++ b/docs/superpowers/specs/2026-10-25-collab-ai-peer-design.md @@ -0,0 +1,100 @@ +# AI peer in the collab doc + +Date: 2026-10-25 +Status: approved 2026-10-25 (client-side variant, as the author asked: the +bash environment stays in the browser) +Phase 3 of 4 in the collaborative editing track +(`2026-10-25-loro-content-storage-design.md`, "Track"). Builds on phase 2 +(`2026-10-25-collab-realtime-room-design.md`). + +## Intent + +The write agent's loop runs in core, but its document tool (`bash` over a +virtual `/doc.xml`) executes in the browser. Today the tab turns each write +into review operations; on the Loro path phase 1 auto-accepts them, so the +editor briefly holds `agent-diff` decorator nodes, then materializes them as a +normal edit by the tab's own Loro peer. The agent's edits are therefore +indistinguishable from the human's peer, they apply on top of the editor as it +is now (not as the agent saw it), and nobody else can tell the agent is +working. + +This phase makes the agent a member of the document while keeping its tool in +the browser: each write becomes a Loro commit by an agent peer, merged against +the version the agent read, relayed to the room like any edit, and the agent +shows its own caret. + +## Goals + +- On the Loro path, an agent write never goes through review operations or + `agent-diff` nodes. It is applied to the Loro doc directly. +- Each write is a commit `agent:` by a PeerID that belongs to the + agent run, three-way merged against the version the agent read, so human + typing made meanwhile survives. +- Everyone in the room sees the write as it lands (phase 2 relay) and an "AI" + caret where the agent last wrote. +- One agent turn stays one undoable timeline row. + +## Non-goals + +- Moving the bash environment or any tool execution to the server. +- Agent runs that outlive the tab (unchanged: disconnect aborts the run). +- Character-by-character streaming of writes. +- Markdown documents (the agent is already disabled for them). + +## Decisions + +1. **Shared Loro helpers.** The headless-editor helpers in + `apps/core/src/modules/draft/draft-crdt.doc.ts` (node set with passthrough + decorators, `readDocContent`, `editDocAt`, `revertChange`, `vetUpdate`) move + to a new `@mx-space/editor/loro` entry so core and admin run the same merge + code. `loro-crdt` and `@haklex/rich-collab-loro` become peer dependencies of + that entry only. `editDocAt` gains an optional `peer`. +2. **Workspace hook.** The browser workspace (`createDocumentBashWorkspace`) + gains an optional `applyDocument(target)`. When set, a write that changes + `/doc.xml` calls it with the parsed target state (ids minted as today) and + returns no operations, so nothing enters the review store. `/review.xml` is + always empty and the base state is the live document. +3. **Agent view and merge.** At the start of each run the CRDT session opens an + agent turn and records the doc frontiers the workspace is seeded from. Each + write is `editDocAt(doc, viewFrontiers, target, message, { peer })` on a + fork; the update is imported into the doc through the binding (so the + editor updates), then synced by the normal push. The view moves to the + fork's frontiers after the write, so the next write diffs against the + agent's own last write. Human edits made mid-turn are not shown to the agent + until its next turn (unchanged rule). +4. **PeerID per run.** A random Loro peer per run, not per session: two tabs + may continue one session in turn, and a per-session peer could reuse op + counters a tab has not seen yet. The timeline already groups a run's writes + by peer and message into one row; undo reverts the row (phase 1's known + limitation for interleaved human edits applies). +5. **Commit message** `agent:`, summary = first 60 characters of the + turn's user message (unchanged format). +6. **Agent presence.** The tab running the agent publishes a second presence + entry keyed `agent:` with user `{ id: 'agent', name: 'AI' }`, a fixed + violet and a caret at the end of the block the last write changed. It is + removed when the run ends. Other members render it like any peer. +7. **Prompt.** On the Loro path the document system prompt drops the + proposal/review wording: writes apply to the shared document immediately, + other people may be editing, and each turn can be undone from history. +8. **Unsaved documents.** A new post with no draft has no Loro doc: the agent + panel runs the first save, then starts the run. If the save fails, the run + does not start and the panel shows the error. +9. **Removed:** `useCrdtAgentTurns` (auto-accept and commit relabelling) and + routing agent operations into review batches on the Loro path. + +## Testing + +- `@mx-space/editor/loro`: the existing core doc specs move with the code; + new: `editDocAt` with `peer` commits under that peer. +- Admin: workspace with `applyDocument` returns no operations and passes the + minted target; CRDT session agent turn merges a write with a concurrent + edit made after the view, commits under the run peer with the agent + message, and advances the view; agent presence entry appears and is + removed; first-save-then-run. + +## Verification (end to end) + +Two browsers on one document. A asks the agent to rewrite a paragraph while B +keeps typing in another paragraph. Both see the "AI" caret and the rewrite +appear; B's typing survives; the timeline shows one agent row; undoing it +restores the paragraph in both browsers. diff --git a/docs/superpowers/specs/2026-10-25-collab-coauthors-design.md b/docs/superpowers/specs/2026-10-25-collab-coauthors-design.md new file mode 100644 index 00000000000..670484c1e74 --- /dev/null +++ b/docs/superpowers/specs/2026-10-25-collab-coauthors-design.md @@ -0,0 +1,127 @@ +# Co-authors: invites, reader editing, byline + +Date: 2026-10-25 +Status: approved +Phase 4 of 4 in the collaborative editing track +(`2026-10-25-loro-content-storage-design.md`, "Track"). Builds on phases 2 +(`2026-10-25-collab-realtime-room-design.md`) and 3 +(`2026-10-25-collab-ai-peer-design.md`). + +## Intent + +The owner can now co-write with their own tabs and an AI peer. This phase lets +the owner invite readers into one document: they sign in with an existing +reader OAuth provider, edit the body and title live in the same room, see and +are seen, and are credited in the published article's byline. The track's +original "per-document AI quota" item is dropped: co-authors get no AI (author's +decision), so nothing spends the owner's model budget on their behalf. + +## Goals + +- The owner creates an invite link for a Lexical document, revokes it, lists + collaborators and removes them. +- A reader opens the link, signs in with a configured OAuth provider, accepts, + and lands on a co-author page at `#/collab/:documentId` in the admin SPA. +- Co-authors edit the body and title through the same Loro room, with carets + and avatars, HTTP fallback included. They cannot read or change metadata + fields, checkpoints, history, sharing, or publish. +- Every presence entry and every commit message from a co-author is bound to + their server-side identity. +- Co-authors who actually edited, and whom the owner keeps listed, appear as + `coAuthors` on the post/note read APIs. + +## Non-goals + +- File and image uploads by co-authors (upload endpoints stay owner-only; the + co-author editor hides upload actions). +- Co-authors creating documents, or access to more than the invited document. +- Comments, suggestions mode, per-co-author undo or history UI. +- AI for co-authors: no agent panel on the co-author page, no `agent.*` + handlers on `/ws/web`, no quota or usage tracking. The owner's own agent + still edits the shared document as in phase 3. +- Rendering the byline in Yohaku (separate repo); this phase exposes the data + and updates `@mx-space/api-client` types. +- Email invitations (links only). + +## Decisions (designer's calls, for the author to confirm) + +1. **Data (additive migration 0044, two tables).** + - `content_collaborators`: `id`, `document_id` (FK cascade), `reader_id` + (FK cascade), `show_in_byline bool default true`, `contributed_at` (null + until their first accepted update), `created_at`; unique + `(document_id, reader_id)`. + - `content_collab_invites`: `id`, `document_id` (FK cascade), `token_hash` + (sha256 of a 32-byte random token; the token itself is shown once), + `expires_at` (default 7 days), `max_uses` (null = unlimited), `uses`, + `revoked_at`, `created_at`. +2. **Invite flow.** Link `…/#/collab/join/` (public route). Signed-out + readers see the enabled OAuth providers and sign in with `callbackURL` back + to the link. `POST /collab/invites/:token/accept` (reader session) checks + expiry, revocation and use count, upserts the collaborator, and returns the + document id. The owner opening the link is sent to the write page. +3. **Co-authors connect on `/ws/web`.** That gateway already resolves the + reader from the session cookie, so `collab.*` handlers are registered there + too, with `canJoin` = owner, or a reader who is a collaborator of the + document. `/ws/admin` stays owner-only. The room relay publishes to both + namespaces so owner and co-authors share one room. +4. **HTTP fallback for co-authors:** `GET/POST /collab/documents/:id/crdt` + (reader session + collaborator guard), same vetting as the owner route. + `GET /collab/documents/:id` returns title, kind, my identity, and the + collaborator list (name, avatar). Nothing else of the document is exposed. +5. **Bound attribution.** A co-author's update is rejected unless every change + in it carries `human:` (checked in `vetUpdate` with the + expected reader); co-authors have no agent, so `agent:` is refused too. + The owner keeps `human:owner` and `agent:` for their own agent. +6. **Presence becomes server-stamped JSON** (replacing Loro `EphemeralStore`, + which also fixes the clock-skew minor from phase 2). A client sends + `{ caret, entry? }`; the server relays `{ connectionId, entry, user, caret }` + with `user` taken from the session (agent entries get `{ name: 'AI' }`). + Clients key entries by `connectionId` (+ `entry`), take `user` only from the + server, and expire entries on local receive time. Avatars therefore only + come from reader records. +7. **Rate limit.** Non-owner members get a per-connection token bucket on + `collab.update` / `collab.presence` (60 frames/s, burst 120); excess frames + are acked `RATE_LIMITED` and dropped. +8. **Co-author page.** A public route with its own reader gate renders the + title, the rich editor bound to the room, avatars and carets. No agent + panel, metadata form, timeline, share, publish or uploads. +9. **Byline.** `coAuthors` = collaborators with `contributed_at` set and + `show_in_byline`, ordered by first contribution: `{ name, avatar, url }` + from the reader record. Returned on post and note detail responses + (snake_case on the wire) and typed in `@mx-space/api-client`. +10. **Owner UI.** A "Collaborators" panel next to the timeline on Lexical + documents: create link (copy), active links with revoke, collaborators + with byline toggle and remove. The timeline labels + co-author steps with the co-author's name. +11. **Removal is immediate.** Removing a collaborator or revoking access + closes their open room membership (server sends `collab.revoked` and drops + them from the room); their past edits stay. + +## Known limitation + +Co-authors receive the full Loro document, oplog included, so text deleted +before they joined can be recovered with developer tools. A shallow snapshot +would hide it but rejects concurrent owner ops whose dependencies predate the +shallow root, which breaks live editing. The author accepted this; the +co-authors panel states it next to the invite button. + +## Testing + +- Core: repository specs for the two tables; invite accept (expired, + revoked, exhausted, idempotent re-accept); `canJoin` for owner, collaborator, + stranger; `vetUpdate` attribution rule; presence stamping; rate limiter; + `coAuthors` on post/note detail; contract + specs for the new endpoints and envelope. +- Gateway e2e: a reader socket on `/ws/web` joins a document it collaborates + on and exchanges updates with an owner socket on `/ws/admin`; a stranger is + refused; removal kicks the member out. +- Admin: join page (signed out / accepted / invalid link), co-author page + wiring, collaborators panel, presence rendering from stamped frames. + +## Verification (end to end) + +Local stack, owner in browser A, a reader (temporary local session for an +existing reader row) in browser B: A creates a link; B opens it, accepts, and +lands on the co-author page; both edit live with named carets; the +co-author page shows no agent panel; A removes B and B is dropped from +the room; publishing the post returns B in `coAuthors`. diff --git a/docs/superpowers/specs/2026-10-25-collab-realtime-room-design.md b/docs/superpowers/specs/2026-10-25-collab-realtime-room-design.md new file mode 100644 index 00000000000..3568d12326c --- /dev/null +++ b/docs/superpowers/specs/2026-10-25-collab-realtime-room-design.md @@ -0,0 +1,177 @@ +# Realtime collab room over WebSocket + +Date: 2026-10-25 +Status: approved 2026-10-25 +Phase 2 of 4 in the collaborative editing track +(`2026-10-25-loro-content-storage-design.md`, "Track"). + +## Intent + +Phase 1 made one Loro doc per Lexical document the source of truth and synced +it over HTTP: ~100 ms POST batches up, a `draft.update` signal, then a +`GET ?since=` pull down. That works, but every remote keystroke costs a signal +plus a round trip, and nobody can see where anyone else is. + +This phase moves the doc sync onto the admin WebSocket as `collab.*` frames +and adds presence: who has the document open, and a named caret for each of +them. It is the transport phase 3 (server AI peer) and phase 4 (co-authors) +plug into, so the room must not assume that every member is the owner. + +## Goals + +- Updates reach the server and every other open editor in one WS frame each + way, without a pull. +- Each member sees the others: an avatar stack in the write-page header, and a + colored caret with a name label (plus the selected range) inside the editor. +- Rooms span replicas (two pods behind Dokploy) through the existing Redis WS + bus. +- The HTTP endpoints from phase 1 keep working and stay the fallback while the + socket is down, for the CLI and for old admin builds during cutover. +- Server-side doc changes (restore, undo, legacy write, cutover reconcile) + reach open editors immediately. + +## Non-goals + +- Server-side AI peer and its caret (phase 3). This phase leaves room for a + non-connection member, but ships none. +- Readers, invites, reader auth (phase 4). Phase 2 rooms only see the owner's + connections, because `/ws/admin` only admits the owner. +- Client-side persistence of unsent updates (still none, as in phase 1). +- Live-syncing metadata fields between tabs (still not synced, phase 1 ruling). +- Comments, follow-me scrolling, per-user undo stacks. + +## Decisions (designer's calls, for the author to confirm) + +1. **Transport is the existing `/ws/admin` socket**, events registered through + `AdminWsHandlerRegistry` like `agent.*`. No second socket. Phase 4 mounts + the same room service on a reader-authenticated gateway. +2. **Binary in JSON.** The envelope is JSON (`{v, event, payload, id}`), so + Loro bytes travel base64 in the payload. Updates larger than 512 KiB after + encoding go through the phase 1 `POST /drafts/:id/crdt` instead (the socket's + `maxPayload` is 1 MiB; the HTTP route allows 4 MiB). +3. **The server stays authoritative and stateless per frame.** A `collab.update` + goes through the same `importUpdate` path as the HTTP POST (vet on a fork, + append, compaction). Only after the append succeeds is the update relayed to + the room. A rejected update is acked `ok:false` with the same code, and the + client runs the phase 1 rejection recovery. +4. **Presence is ephemeral and serverless.** Clients keep a Loro + `EphemeralStore` (awareness) per document and exchange its encoded deltas as + `collab.presence` frames that the server relays without storing. A joining + member triggers `collab.peer.joined`, and every member answers by sending its + full presence once, so late joiners catch up without server state. Entries + expire after 30 s without a refresh; clients refresh every 10 s. +5. **Carets are anchored to Loro tree ids, not Lexical keys.** A position is + `{ id: TreeID, offset }` (text node id from `binding.idOf`), so it means the + same thing in every editor; the receiver maps it back with `binding.keyOf`. +6. **Identity.** Phase 2 presence shows the owner's name and avatar, a + per-connection color, and a "this device / other tab" distinction by + connection id. Each tab is its own member; two tabs of the owner show two + carets. +7. **`draft.update` keeps firing** after every append (HTTP or WS), unchanged, + for old admin builds and the drafts list. New clients in a room ignore it for + content sync. +8. **Access check is one replaceable function** `canJoin(member, documentId)`; + phase 2's implementation is "the connection is on `/ws/admin`". + +## Protocol + +All events ride the `@mx-space/ws-client` envelope. Requests carry `id` and get +an `ack` (`{ ok, code?, data? }`). Server pushes have no `id`. + +Client → server: + +| Event | Payload | Ack data | +| --- | --- | --- | +| `collab.join` | `{ draftId, since?: base64 VV }` | `{ documentId, update: base64, presence: [] }` — snapshot when `since` is absent, else the updates the client lacks | +| `collab.update` | `{ draftId, update: base64 }` | — (`ok:false, code: DRAFT_CRDT_UPDATE_REJECTED` on rejection) | +| `collab.presence` | `{ draftId, state: base64 }` (encoded `EphemeralStore` delta) | — | +| `collab.leave` | `{ draftId }` | — | + +Server → room members: + +| Event | Payload | Sent to | +| --- | --- | --- | +| `collab.update` | `{ documentId, update: base64 }` | room, excluding the sender's connection | +| `collab.presence` | `{ documentId, state: base64 }` | room, excluding the sender | +| `collab.peer.joined` | `{ documentId, connectionId }` | room, excluding the joiner | +| `collab.peer.left` | `{ documentId, connectionId }` | room | + +Room name: `collab:`. `draftId` is accepted on the wire because +that is the id the write page holds; the server resolves it to the document +and its working copy exactly as the HTTP routes do (any branch id of a Lexical +document reaches its working copy). + +## Server + +- `CollabRoomService` (draft module): `join`, `update`, `presence`, `leave`, + `disconnect(connectionId)`. Holds no document state. It calls + `DraftCrdtService` for snapshots/updates, `canJoin` for access, and a sink + interface for "send to room except X" so the gateway owns sockets. +- The admin gateway registers the four events through + `AdminWsHandlerRegistry` and a disconnect listener that turns into + `collab.peer.left` for each room the connection was in. +- Room membership uses the gateway's existing `WsRoomManager` + `WsPresenceService` + (per-pod membership, Redis presence index). Fan-out uses `bus.publish` with + `rooms` and `exclude`, so members on the other pod receive the frame. +- **Every doc change is relayed.** `DraftCrdtService` gains one hook, + `onDocumentChanged(documentId, update, origin)`, called after any append: + client imports (HTTP or WS) and server-side edits (restore, undo, legacy + write, reconcile, import). `CollabRoomService` subscribes and relays + `collab.update` to the room, excluding `origin` when the change came from a + connection. Server-side edits therefore show up live in open editors. +- Frame size: the gateway rejects a `collab.update` whose decoded update + exceeds the HTTP limit (4 MiB) the same way the POST does. +- Rate: no extra limiting in phase 2 (owner only). Phase 4 adds per-member + limits. + +## Client (admin write page) + +- `crdt-sync` gets a transport interface with two implementations: the phase 1 + HTTP one, and a WS one built on `getAdminSocket()`. The WS transport is used + while the socket is `open`; on `reconnecting`/`closed` the session falls back + to HTTP push and pull, and on reopen it re-joins with `since` = local oplog + version and pushes anything unsent (`sent` vector logic is unchanged). +- Incoming `collab.update` frames are imported through the binding (as pulls + are today). The `draft.update` → pull path stays only for the HTTP fallback. +- Presence: `crdt-draft-session` owns an `EphemeralStore`. On every editor + selection change (debounced 50 ms) it writes + `{ user: { name, avatar, color }, anchor, focus }` with tree-id anchors, and + clears the caret on blur. Remote states render as: + - a caret + name label at `focus`, and a translucent highlight over the + `anchor..focus` range, drawn in an overlay like `apps/admin/src/author/AgentCursor.tsx`; + - an avatar stack (max 4 + "+N") in the write-page header, tooltip with name + and "other tab" for the owner's own other connections. +- Colors: a fixed palette of 8, picked by hashing the connection id. + +## Failure modes + +- **Socket drops mid-push.** The update stays `dirty` in `crdt-sync`; after + fallback or reconnect it is re-sent. Loro import is idempotent, so a frame + that reached the server before the drop costs nothing when re-sent. +- **Pod restart.** Members reconnect, re-join with `since`, and miss nothing, + because the doc lives in PostgreSQL and the join ack carries what they lack. +- **Relay ordering across pods.** Loro tolerates out-of-order updates; an + update whose deps have not arrived is held as pending by the client doc and + applied when they do. The join ack's catch-up closes any gap. +- **Presence for a gone member.** Expires after 30 s; `collab.peer.left` clears + it immediately when the disconnect is seen. + +## Testing + +- Core: `CollabRoomService` unit tests with a fake sink (join returns snapshot / + catch-up, update relays to others only after a successful append, rejected + update acks `ok:false` and relays nothing, presence relays without storage, + disconnect emits `peer.left`). PG e2e: a restore relays an update that brings + a second doc to the same state. +- Gateway: handler wiring through the registry, using the existing gateway + spec patterns. +- Admin: WS transport against a fake socket (join catch-up, push, fallback to + HTTP on close, rejoin with `since`); presence encode/decode and tree-id + anchoring through a real binding; caret overlay component test. + +## Verification (end to end) + +Two browsers on the local stack: both see each other's avatar and caret, +typing in one shows up in the other without any `GET /crdt` request in the +network log, a restore from the timeline appears in the other tab live, and +killing/restarting the core process loses no edit after reconnect. diff --git a/docs/superpowers/specs/2026-10-25-loro-content-storage-design.md b/docs/superpowers/specs/2026-10-25-loro-content-storage-design.md new file mode 100644 index 00000000000..8d97cbe96c1 --- /dev/null +++ b/docs/superpowers/specs/2026-10-25-loro-content-storage-design.md @@ -0,0 +1,464 @@ +# Loro as the content store for drafts + +Date: 2026-10-25 +Status: draft, pending approval +Phase 1 of 4 in the collaborative editing track (see "Track"). + +## Intent + +The author wants articles to be co-written live: the author, invited readers +(co-authors) and AI agents all editing one document, each with a visible +cursor, every change attributed, and any single step revertible. That needs a +CRDT underneath the editor. haklex already ships one binding, +`@haklex/rich-collab-loro` (Lexical ↔ LoroTree), and `mxs author` already runs +a human + agent session on it. + +Today's draft system cannot carry that. It is a git-like model built to +survive conflicts: `content_revisions` is a snapshot chain, `drafts` are +branches with base/head pointers, saves use an optimistic lock +(`expectedHeadRevisionId` / `DRAFT_HEAD_CONFLICT`), and the admin write page +carries a three-way merge, a conflict dialog and a version tree to cope. In +the author's words, draft branches only exist because of conflicts. A CRDT +makes conflicts disappear, and with them the reason for branches. Two +histories (Loro's oplog and our revision chain) both claiming to be the truth +is the clash this phase removes. + +This phase swaps the storage under the editor and nothing else: one user (the +owner), one admin write page, HTTP sync. Realtime transport, the server-side +AI peer, and the co-author page come in later phases and build on this one. + +## Track + +1. **This document.** Loro doc per content document is the source of truth; + history becomes a linear timeline of checkpoints; branches and the + optimistic lock go away. +2. Realtime room over WebSocket (`collab.*` events on the existing + `@mx-space/ws-client` envelope), presence and cursors. +3. Server-side AI peer that writes straight into the doc. +4. Co-author page (`#/collab/:documentId` inside admin), invite links, + reader OAuth login, per-document AI quota, byline. + +Each phase gets its own design. Phase 2 and 3 protocol work is being aligned +with the Write Agent WebSocket design in a parallel session; nothing here may +assume the transport. + +## Decisions made by the author + +- Loro's history replaces our own. The existing draft history conflicts with + it and should not be maintained alongside it. +- Draft branches are not useful and go away. The one thing branches did that + matters, "go back to an old version, keep editing, and the newer history + still exists", is covered by Loro's revert-as-new-ops (`revertTo`, or + restoring a checkpoint as a new commit). History stays linear and nothing + is lost. +- Reverting works at two grains: the whole document back to a point, and a + single step (one AI turn, one human edit run) undone on its own while later + edits survive. +- AI edits land directly in the document like a human's and are reverted after + the fact if unwanted. (Matters from phase 3; this phase must not block it.) +- Markdown is deprecated. Nothing new is built for Markdown content. +- A CLI save-and-publish publishes only what the CLI changed on top of the + published version, never the owner's unfinished work in the admin. The CLI + change is also merged into the working copy, so a later publish from the + admin keeps it. +- The author asked for a prototype first. It was built and confirmed the + interaction (see "Prototype findings"). + +## Goals + +- Every Lexical content document (post, note, page) has exactly one Loro doc, + stored in PostgreSQL, which is the source of truth for its working content. +- The admin write page edits through the Loro binding. Saving never conflicts. + Two admin tabs (or devices) editing the same document converge without a + dialog. +- History is one linear timeline per document: checkpoints (saves, publishes, + restores, undos, imports) plus the Loro change log, attributed by peer. +- Restore a checkpoint, and undo a single step, both as new commits. +- Everything outside the editor keeps working on materialized snapshots: + publish, share links, Yohaku `/preview/:token`, the dashboard desk, AI cover, + file references, the CLI. +- No existing revision is lost. + +## Non-goals + +- Realtime transport, presence, cursors (phase 2). Phase 1 syncs over HTTP. +- Server-side AI peer, co-authors, invites, reader auth (phases 3 and 4). +- Shallow snapshots / history trimming. +- Replaying pre-existing revisions into the Loro oplog. Old revisions stay as + checkpoints with materialized content only. +- Any change to Markdown documents. +- Changing the public API consumed by Yohaku (`GET /drafts/shared/:token`). + +## Anti-goals + +- Publishing content that differs from what the owner saw when they pressed + publish. +- A document whose Loro doc and visible checkpoints disagree with no way back + (every import or legacy write must leave a checkpoint). +- Losing an edit during the rolling-deploy window, when old pods still accept + legacy `PUT /drafts/:id` saves. +- A second history system creeping back (e.g. the client keeping its own + revision chain or merge logic). + +## Prototype findings + +Throwaway branch `spike/collab-room` (worktree `../mx-core-spike-collab`) +extended the `mxs author` local server into a multi-client room. Verified with +two independent browsers plus an agent editing the file on disk: + +- Two people typing in different paragraphs converge; an agent rewrite merged + while a human kept typing, both kept. +- Single-step undo of the agent turn: on a fork, + `fork.applyDiff(doc.diff([lastIdOfStep], depsOfStep), { fullState: true })`, + commit, export the update and import it into the main doc. The agent's + paragraph came back; human edits made before and after it survived; both + clients synced; the file persisted. This is the undo used below. +- Not covered: undoing a grouped row when another peer's changes interleave + inside the group (they would be reverted too). +- The spike's agent writes through `streamAtVersion`, which forks and + therefore gets a fresh PeerID per turn. Phase 3 must give each agent session + a fixed PeerID. Not a phase 1 concern, but phase 1 must store commit + messages and peers so phase 3 can attribute. + +## Design + +### What lives in the Loro doc + +One `LoroDoc` per `content_documents` row: + +- `lexical` tree — the body, via `createLoroBinding` (`TREE_NAME`). +- `title` — `LoroText`. + +Nothing else. `meta` and `typeSpecificData` (slug, category, tags, note +`password`, paywall settings in post `meta`, …) stay **out** of the CRDT and +live on the working copy (see "Metadata fields"). (Designer's call.) Reason: +Loro history cannot be redacted, and from phase 4 co-authors sync the whole +doc, so every value ever stored in it would reach them; co-authors also do not +edit metadata. Deciding this now avoids rewriting history later. + +`text` (Markdown rendering) and `images` are not stored; they are derived when +materializing. `contentFormat` is always `lexical`. + +### Metadata fields + +- New nullable column `content_crdt_docs.fields jsonb`: the working copy's + current `{ meta, typeSpecificData }`, normalized with `canonicalSnapshot` + rules. +- Written by the owner through a JSON endpoint that takes only the keys that + changed and merges them per top-level key (last writer wins per key), so + two tabs editing different fields do not clobber each other. +- Every checkpoint copies `fields` into its `meta` / `typeSpecificData` + columns, so restore, compare, share and publish see them as today. +- Metadata changes appear in the timeline as checkpoints, not as Loro steps, + and are not undoable step by step. Restoring a checkpoint restores its + fields too. +- **Edits made outside drafts flow in.** `PATCH /posts/:id` (`categoryId`, + `pinAt`), `PATCH /notes/:id` (`mood`, `weather`, `topicId`, `bookmark`) and + the AI slug backfill change the live article directly. When the document + has a working copy, the same service call also merges those keys into + `fields`. Otherwise the long-lived working copy would undo them on the next + publish. (Designer's call.) + +Commit messages carry attribution: `human:`, `agent:`, +`restore `, `undo @`, `import `, +`legacy-save `. Phase 1 only produces the owner's `human:` plus +the server-side kinds; the format must not need changing for phases 3 and 4. + +### Storage + +All additive (single release, expand only): + +- `content_crdt_docs`: `document_id` (PK, FK → `content_documents`, cascade), + `snapshot bytea`, `snapshot_version bytea` (encoded version vector covered + by the snapshot), `synced_revision_id` (FK → `content_revisions`, the + checkpoint whose content the doc last matched or produced), `fields jsonb` + (see "Metadata fields"), `updated_at`. +- `content_crdt_updates`: `id` (snowflake), `document_id` (FK, cascade), + `bytes bytea`, `created_at`. Append-only. Loro merges are commutative, so + two replicas appending concurrently need no lock. +- Compaction: when a document's pending updates pass a threshold, a replica + takes a PostgreSQL advisory lock on the document, imports snapshot + updates, + writes a new snapshot and deletes exactly the update rows it folded in. + Loading is snapshot + every remaining update row. Do not filter by + `id >` some maximum: snowflake ids from two replicas do not commit in id + order, so a lower id can commit after compaction. Re-importing an update + Loro already has is a no-op. +- `content_revisions`: add nullable `frontiers jsonb` and `kind text` + (`save | publish | restore | undo | import | legacy-save`). Existing rows keep + `frontiers = null`, `kind = null` (read as legacy). `parent_revision_id` + links each new checkpoint to the previous one, so the chain is linear from + here on. + +### The working copy + +Most consumers key on a branch id: publish jobs (`branchId`), file references +(`FileReferenceType.Draft`), share links in follow mode (`draft_id`), the +`?draftId=` URL, AI cover, content migration, the desk, the CLI. To keep this +phase's blast radius small, the `drafts` table stays, with a new invariant: + +- **At most one active `drafts` row per document: the working copy.** Its id + is the stable handle for the Loro doc. Its `head_revision_id` is the latest + checkpoint. `base_revision_id` is kept as-is and no longer means anything. +- Existing documents with several active branches: on first load through the + new path, the most recently updated active branch becomes the working copy; + the others are archived. Their revisions remain and are restorable from the + timeline. A follow-mode share pointing at an archived branch is converted to + a pinned share at that branch's head, so its readers keep seeing the same + content. (Designer's call.) +- A document with no active branch (all archived, or published-only) gets a + new working copy seeded from the published revision on first load. + +**Discarding the draft.** `DELETE /drafts/:id` (write page, drafts list bulk +delete, `mxs draft delete`) on a working copy restores the published +checkpoint as a new commit (`restore`, as below) instead of archiving: the +working copy goes back to what is live, and the discarded work stays in the +timeline. For a document that was never published, it archives the working +copy and the document drops out of draft lists, as today. (Designer's call, +following "revert is a new commit, nothing is lost".) + +Renaming or retiring the `drafts` table is a later cleanup, not this phase. + +### Checkpoints + +A checkpoint is a `content_revisions` row with `frontiers` set and the full +materialized snapshot (title, text, content, images, meta, typeSpecificData). +Everything that reads revisions today keeps reading these rows. + +Created by: + +- The write page: on ⌘S, and on the existing 10-second idle timer (which today + does `PUT /drafts/:id`). Skipped when the materialized content equals the + previous checkpoint (`sameRevisionContent`). +- The server: before a publish job, on restore, on undo, on import, on a + legacy write. + +File references for the working copy are refreshed when a checkpoint is +created, not on every update. + +### Materializing on the server + +`apps/core` gains `loro-crdt` (Node build) and `@haklex/rich-collab-loro`, and +binds a headless Lexical editor (from `@haklex/rich-headless`) to the doc to +read the body as Lexical JSON, then derives `text` with the existing +`helper.lexical.service` `$toMarkdown` path. + +Risk the builder must cover: the headless node set must register every node +type the admin editor can produce (including admin-only ones such as +`CatalogDynamicEditNode` (`dynamic`) and `AgentDiffEditNode`), or binding and +import fail. `mxs author`'s `createAuthorHeadlessEditor` already solves this +for the CLI and is the reference. + +### Sync API (phase 1, HTTP) + +Owner-only (`@Auth()`), binary bodies, `@HTTPDecorators.RawResponse`: + +- `GET /drafts/:id/crdt` → full snapshot. +- `GET /drafts/:id/crdt?since=` → updates the caller + lacks. +- `POST /drafts/:id/crdt` (octet-stream) → import. The server validates on a + fork before appending: size limit, the update does not add a second root to + the `lexical` tree (the `mxs author` lineage check), and the result still + materializes. Rejected updates return an `AppException`. + +A rejected update is already in the client's local doc, and every later local +update depends on it, so that tab can never sync again. On rejection the +client reloads the doc from the server and re-applies its own materialized +content as one legacy write (below), then continues on the fresh doc. The +user's text survives; their step history for that session collapses into one +`legacy-save`. (Designer's call.) + +No client-side persistence of unsent updates (IndexedDB) in phase 1. Updates +leave the browser in ~100 ms batches, which is more durable than today's +10-second `PUT`. A failed send stays queued in memory and is retried, and the +page warns on unload while anything is unsent, as `mxs author` does. + +After a successful append the server emits the existing `draft.update` admin +broadcast (payload unchanged in shape; `headRevisionId` is the latest +checkpoint). Other tabs react by pulling `?since=`. Phase 2 replaces both the +POST/GET pair and this signal with `collab.*` frames. + +Endpoint names are the builder's call; the semantics are not. + +### Legacy whole-document writes + +`POST /drafts` and `PUT /drafts/:id` keep their request shapes, because the CLI +(and old admin builds during cutover) use them. They become "apply this whole +snapshot to the working copy at a known version": + +- `PUT /drafts/:id {data, expectedHeadRevisionId}`: take the frontiers of the + `expectedHeadRevisionId` checkpoint and apply the body there with + `editAtVersion` (a three-way merge against everything since). `editAtVersion` + only touches the `lexical` tree, so the title is applied at the same version + separately. For metadata, merge only the keys whose value differs from that + checkpoint into `fields`. Commit as `legacy-save`, create a checkpoint, + return the working copy view. `DRAFT_HEAD_CONFLICT` is no longer thrown. +- A legacy write carrying `contentFormat: markdown` to a document on the Loro + path is rejected (400). (Designer's call.) +- If the base checkpoint has no frontiers (a legacy revision), apply at the + doc's current frontiers instead (last writer wins for that write). +- `POST /drafts` with a `refId` or `baseRevisionId` on a document that already + has a working copy writes into it the same way, with the base checkpoint as + the version. Without a document it creates the document, the working copy + and the Loro doc. + +**A legacy write produces two checkpoints, and returns the first.** The CLI +saves `published + patch` with `baseRevisionId = published` +(`cli/draft/_shared.ts:124`), then publishes the returned `headRevisionId` +(`publishSavedDraft`, `:198`; used by `note edit --publish` and friends). The +author decided that this must publish only what the CLI changed, never the +owner's unfinished work in the working copy, while the working copy must also +keep the CLI's change. So every legacy write: + +1. Creates **the write's own checkpoint**: the base plus this write, nothing + else. When the base checkpoint has frontiers, it is the fork state right + after `editAtVersion` at those frontiers. The ops are real, so the + checkpoint stores those frontiers and is a genuine version of the doc. + When the base has no frontiers (a legacy revision), the server builds the + content from the base's materialized content plus the write without Loro, + and the checkpoint stores no frontiers. Either way it carries the full + materialized snapshot, so publish can use it. +2. Imports the same ops into the doc, which merges them with everything since, + and creates a `legacy-save` checkpoint at the new head. That checkpoint + becomes the working copy's `head_revision_id`. +3. Returns the working copy view with `headRevision` / `headRevisionId` set to + checkpoint 1, the revision this write produced. This differs from the + working copy's stored head, and the response documents it as such. + +The CLI therefore keeps its request shapes and publishes exactly +`published + patch`. The owner's unpublished work stays unpublished and now +includes the CLI change. The existing `newerDraftChanges` report then tells +the publish caller that the working copy holds more than what went live. + +### Lazy import and the cutover window + +No batch migration. A document gets its Loro doc the first time the new path +touches it (open in the write page, a legacy write, a publish): + +1. Pick the working copy (above). +2. Create the doc from its head revision's content in one commit, + `import `, and set `synced_revision_id`. + +During the rolling deploy, old pods still accept `PUT /drafts/:id` and move the +branch head without touching the Loro doc. So on every load, if the working +copy's `head_revision_id` is a revision with no frontiers and differs from +`synced_revision_id`, the server applies that revision's content as a legacy +write at the synced checkpoint's frontiers, then updates `synced_revision_id`. +This closes the window without coordination. + +### History, restore and undo + +- `GET /drafts/:id/history` returns the timeline: checkpoints (all kinds, + including legacy revisions without frontiers) merged with the Loro change + log (peer, counter, lamport, timestamp, commit message), newest first. Human + changes group by author and idle gap, agent changes by turn, as in the + `mxs author` history panel. +- **Restore a checkpoint**: `editAtVersion(doc, doc.frontiers(), …, checkpoint + content)`, commit `restore `, new checkpoint of kind `restore`. Works for + legacy checkpoints too, because it uses content, not frontiers. +- **Undo one step**: the fork + `applyDiff` method from the prototype, commit + `undo …`, new checkpoint of kind `undo`. Offered for timeline entries that + come from the change log. +- `GET /drafts/compare/:leftId/:rightId` keeps working on checkpoint content. + `relation` is now `same`, `ancestor` or `descendant` along the linear chain; + `diverged` can only come from archived legacy branches. + +### Publish + +`POST /publish-jobs` keeps its shape. The write page creates a `publish` +checkpoint at the current head and passes it as `revisionId`, with +`branchId` = the working copy. Changes in `publish.service.ts`: + +- Drop the `headRevisionId === revisionId` check (`:82`). Any checkpoint of the + document may be published; edits made after it are reported through the + existing `hasNewerBranchHead` / `newerDraftChanges` path. +- `confirmDiverged` is accepted and ignored for working copies. +- `PUBLISHED_REVISION_CHANGED` stays. + +The snapshot frozen into the job is the checkpoint's materialized content, so +what is published is exactly what was checkpointed. + +### Share links + +No schema change. Pinned shares point at a checkpoint. Follow shares point at +the working copy row and resolve to its `head_revision_id`, the latest +checkpoint, which the idle timer keeps fresh. Yohaku and `api-client` are +untouched. + +### Admin write page + +For Lexical documents: + +- The editor binds to the Loro doc (a `LoroSyncPlugin` like `apps/admin/src/author/`, + using the HTTP endpoints above). The title binds to the `title` text. The + metadata form saves changed keys through the fields endpoint. +- The save layer stops sending snapshots. The 10-second timer and ⌘S create + checkpoints instead. +- Removed: `DRAFT_HEAD_CONFLICT` handling and `reconcileWithRemote`, + `merge-draft-conflict.ts`, `DraftConflictDialog`, the `remoteUpdateStale` + banner, `DraftRecoveryReview`, branch selection (`availableDraft`, + `continueDraft` / `applyDraft`), the `?baseRevisionId` fork-on-save flow. +- `VersionTreePanel` and `version-tree/*` are replaced by a timeline panel with + restore and undo-this-step, plus the existing share bar. +- `features/drafts/*`: the list shows one row per document; "Restore" restores + in place instead of opening a fork. +- The dashboard desk keeps reading `relationToPublished` and `headRevision`. + +The write-page AI agent's accept/reject review goes away in this phase. Its +pending `agent-diff` nodes are stripped before every save today +(`stripPendingAgentDiffs`, WRVC:5109) because they must never be persisted; +with the editor bound to Loro, every editor update is persisted and synced, so +those nodes cannot exist. Agent operations (`insert` / `replace` / `delete`) +apply straight to the editor, inside one Loro commit whose message is +`agent:` (set the next commit message before applying), so the turn +shows as one undoable step. This is the author's "AI edits land directly" +decision brought forward to phase 1 for the client-side agent. (Designer's +call.) The agent loop moving to the server (parallel Write Agent WS design) +does not change this: whoever applies the operation commits it as one +`agent:` step. + +**Markdown boundary.** (Designer's call throughout; Markdown gets no new work.) + +- The signal is the working copy head's `contentFormat`. Lexical → Loro path. + Markdown → the current flow, unchanged, including several branches and + `DRAFT_HEAD_CONFLICT`. The legacy endpoints therefore behave by format; the + one-working-copy rule applies to Lexical documents. +- The staged conversion (published article is Markdown, draft is Lexical) is + on the Loro path, because the working copy is Lexical. The migration commit + currently requires the branch head and its hash to be unchanged + (`content-migration-commit.service.ts:331`). Idle checkpoints move the head, + so that precondition must check the checkpoint being published instead of + the live head. +- Markdown code paths are removed when Markdown is removed, not here. + +### CLI and other server consumers + +- CLI: no request-shape change. Legacy writes return the write's own + checkpoint, so CLI save-and-publish publishes only the CLI's change (above). + `post/apply.ts`'s "more than one active branch" failure can no longer trigger + after import. +- `ai-image` cover, aggregate desk, content migration, delete cascades + (`deleteByRef`): unchanged, they read the working copy row and checkpoints. +- `file-reference-usage.repository.ts` keeps scanning `content_revisions`. + +## Constraints + +- Rolling deploy with two replicas: everything in this phase is additive. No + drop, rename or type change. The `drafts` table and its columns stay. +- Schema migrations follow the `mx-migration-author` skill and pass + `lint:migrations`. +- API envelope rules from `AGENTS.md`: binary endpoints use `RawResponse`; JSON + endpoints return bare values or `withMeta`. +- `loro-crdt` must load in the core runtime (Node, wasm) and in the Docker + image. + +## Left to the builder + +- Endpoint paths and the exact checkpoint skip rule. +- Compaction thresholds and whether to cache loaded docs in memory per pod + (if cached, reload updates `id >` last seen before each read). +- Timeline panel visuals. +- How the admin decides "this document uses the Loro path" (presence of a + `content_crdt_docs` row after import is the obvious signal). +- Tests: repository specs for the new tables; e2e for sync, legacy writes at a + version, lazy import, the cutover reconciliation, restore, undo, publish of a + non-head checkpoint; admin component tests replacing the conflict-merge ones. diff --git a/packages/api-client/models/base.ts b/packages/api-client/models/base.ts index fdb38b20711..39e1dce5e3f 100644 --- a/packages/api-client/models/base.ts +++ b/packages/api-client/models/base.ts @@ -159,3 +159,8 @@ export interface ResponseMeta extends BaseResponseMeta { summary?: SummaryMeta skills?: SkillBundleView[] } + +export interface CoAuthor { + avatar: string | null + name: string +} diff --git a/packages/api-client/models/note.ts b/packages/api-client/models/note.ts index cbab590aad4..91f40d4cfc5 100644 --- a/packages/api-client/models/note.ts +++ b/packages/api-client/models/note.ts @@ -1,4 +1,4 @@ -import type { Image } from './base' +import type { CoAuthor, Image } from './base' import type { EnrichmentResult } from './enrichment' import type { TopicModel } from './topic' @@ -31,6 +31,7 @@ export interface NoteModel { topicId?: string | null topic?: TopicModel | null + coAuthors?: CoAuthor[] createdAt: string modifiedAt: string | null diff --git a/packages/api-client/models/post.ts b/packages/api-client/models/post.ts index 460ba09d48f..a8aab0871ee 100644 --- a/packages/api-client/models/post.ts +++ b/packages/api-client/models/post.ts @@ -1,4 +1,4 @@ -import type { Image } from './base' +import type { CoAuthor, Image } from './base' import type { CategoryModel } from './category' import type { EnrichmentResult } from './enrichment' @@ -36,6 +36,7 @@ export interface PostModelMarkdown { related?: PostRelatedSummary[] enrichments?: EnrichmentMap isPremium: boolean + coAuthors?: CoAuthor[] } export interface PostModelLexical { @@ -62,6 +63,7 @@ export interface PostModelLexical { related?: PostRelatedSummary[] enrichments?: EnrichmentMap isPremium: boolean + coAuthors?: CoAuthor[] } export type PostModel = PostModelMarkdown | PostModelLexical diff --git a/packages/db-schema/src/schema/content.ts b/packages/db-schema/src/schema/content.ts index 26f40f5daf0..ac0e557792c 100644 --- a/packages/db-schema/src/schema/content.ts +++ b/packages/db-schema/src/schema/content.ts @@ -3,6 +3,7 @@ import type { AnyPgColumn } from 'drizzle-orm/pg-core' import { boolean, check, + customType, index, integer, jsonb, @@ -15,6 +16,12 @@ import { import { readers } from './auth' import { createdAt, pkText, refText, tsCol, updatedAt } from './columns' +const bytea = customType<{ data: Uint8Array; driverData: Buffer }>({ + dataType: () => 'bytea', + toDriver: (value) => Buffer.from(value), + fromDriver: (value) => new Uint8Array(value), +}) + export const categories = pgTable( 'categories', { @@ -259,6 +266,10 @@ export const contentRevisions = pgTable( string, unknown > | null>(), + frontiers: jsonb('frontiers').$type< + { counter: number; peer: `${number}` }[] | null + >(), + kind: text('kind'), }, (table) => [ index('content_revisions_document_idx').on(table.documentId), @@ -346,6 +357,81 @@ export const contentPublicationEvents = pgTable( ], ) +export const contentCrdtDocs = pgTable('content_crdt_docs', { + documentId: refText('document_id') + .primaryKey() + .notNull() + .references(() => contentDocuments.id, { onDelete: 'cascade' }), + snapshot: bytea('snapshot').notNull(), + snapshotVersion: bytea('snapshot_version').notNull(), + syncedRevisionId: refText('synced_revision_id').references( + () => contentRevisions.id, + { onDelete: 'set null' }, + ), + fields: jsonb('fields').$type<{ + meta: Record | null + typeSpecificData: Record | null + } | null>(), + createdAt: createdAt(), + updatedAt: updatedAt(), +}) + +export const contentCrdtUpdates = pgTable( + 'content_crdt_updates', + { + id: pkText(), + documentId: refText('document_id') + .notNull() + .references(() => contentDocuments.id, { onDelete: 'cascade' }), + bytes: bytea('bytes').notNull(), + createdAt: createdAt(), + }, + (table) => [index('content_crdt_updates_document_idx').on(table.documentId)], +) + +export const contentCollaborators = pgTable( + 'content_collaborators', + { + id: pkText(), + documentId: refText('document_id') + .notNull() + .references(() => contentDocuments.id, { onDelete: 'cascade' }), + readerId: text('reader_id') + .notNull() + .references(() => readers.id, { onDelete: 'cascade' }), + showInByline: boolean('show_in_byline').notNull().default(true), + contributedAt: tsCol('contributed_at'), + removedAt: tsCol('removed_at'), + createdAt: createdAt(), + }, + (table) => [ + unique('content_collaborators_document_reader_uniq').on( + table.documentId, + table.readerId, + ), + ], +) + +export const contentCollabInvites = pgTable( + 'content_collab_invites', + { + id: pkText(), + documentId: refText('document_id') + .notNull() + .references(() => contentDocuments.id, { onDelete: 'cascade' }), + tokenHash: text('token_hash').notNull(), + expiresAt: tsCol('expires_at').notNull(), + maxUses: integer('max_uses'), + uses: integer('uses').notNull().default(0), + revokedAt: tsCol('revoked_at'), + createdAt: createdAt(), + }, + (table) => [ + unique('content_collab_invites_token_hash_uniq').on(table.tokenHash), + index('content_collab_invites_document_idx').on(table.documentId), + ], +) + /** * Self-referential thread structure plus polymorphic ref to content (Post/Note/Page/Recently). */ diff --git a/packages/editor/package.json b/packages/editor/package.json index aa2a9288310..60ece8b53d1 100644 --- a/packages/editor/package.json +++ b/packages/editor/package.json @@ -21,6 +21,10 @@ "types": "./src/core/index.ts", "import": "./src/core/index.ts" }, + "./loro": { + "types": "./src/loro/index.ts", + "import": "./src/loro/index.ts" + }, "./package.json": "./package.json" }, "publishConfig": { @@ -37,6 +41,10 @@ "types": "./dist/core/index.d.ts", "import": "./dist/core/index.js" }, + "./loro": { + "types": "./dist/loro/index.d.ts", + "import": "./dist/loro/index.js" + }, "./package.json": "./package.json" } }, @@ -68,8 +76,22 @@ "lexical": "^0.52.0" }, "devDependencies": { + "@haklex/rich-collab-loro": "0.55.0", + "loro-crdt": "^1.16.4", "tsdown": "0.23.0", "typescript": "6.0.3", "vitest": "5.0.5" + }, + "peerDependencies": { + "@haklex/rich-collab-loro": "0.55.0", + "loro-crdt": "^1.16.4" + }, + "peerDependenciesMeta": { + "@haklex/rich-collab-loro": { + "optional": true + }, + "loro-crdt": { + "optional": true + } } } diff --git a/packages/editor/src/loro/index.ts b/packages/editor/src/loro/index.ts new file mode 100644 index 00000000000..34aee108121 --- /dev/null +++ b/packages/editor/src/loro/index.ts @@ -0,0 +1,243 @@ +import { + $reconcileRoot, + createLoroBinding, + TREE_NAME, +} from '@haklex/rich-collab-loro' +import { allHeadlessNodes } from '@haklex/rich-headless' +import { createHeadlessEditor } from '@lexical/headless' +import { + DecoratorNode, + type Klass, + type LexicalEditor, + type LexicalNode, + type NodeKey, + type SerializedEditorState, + type SerializedLexicalNode, +} from 'lexical' +import { LoroDoc, type OpId } from 'loro-crdt' + +export type Frontiers = ReturnType +export type PeerId = Parameters[0] + +export interface DocContent { + content: string + title: string +} + +const TITLE = 'title' + +const BUILTIN_TYPES = new Set([ + 'root', + 'paragraph', + 'text', + 'linebreak', + 'tab', + ...allHeadlessNodes.map((node) => + ('replace' in node ? (node.replace as Klass) : node).getType(), + ), +]) + +const passthroughCache = new Map>() + +// Blocks only the admin editor renders (map, stock, …); the server keeps their JSON verbatim. +const passthroughNode = (type: string): Klass => { + const cached = passthroughCache.get(type) + if (cached) return cached + class PassthroughNode extends DecoratorNode { + __json: SerializedLexicalNode + + static getType(): string { + return type + } + + static clone(node: PassthroughNode): PassthroughNode { + return new PassthroughNode(node.__json, node.__key) + } + + static importJSON(json: SerializedLexicalNode): PassthroughNode { + // The Loro binding rebuilds every node with `children: []`, which a leaf block never had. + const { children: _children, ...rest } = json as SerializedLexicalNode & { + children?: unknown + } + return new PassthroughNode(rest) + } + + constructor( + json: SerializedLexicalNode = { type, version: 1 }, + key?: NodeKey, + ) { + super(key) + this.__json = json + } + + exportJSON(): SerializedLexicalNode { + return this.__json + } + + createDOM(): never { + throw new Error(`${type} has no DOM on the server`) + } + + updateDOM(): boolean { + return false + } + + isInline(): boolean { + return false + } + + decorate(): null { + return null + } + } + passthroughCache.set(type, PassthroughNode) + return PassthroughNode +} + +const collectJsonTypes = (node: unknown, out: Set) => { + if (!node || typeof node !== 'object') return + const { children, type } = node as { children?: unknown; type?: unknown } + if (typeof type === 'string') out.add(type) + if (Array.isArray(children)) { + for (const child of children) collectJsonTypes(child, out) + } +} + +const collectDocTypes = (doc: LoroDoc, out: Set) => { + for (const node of doc.getTree(TREE_NAME).nodes()) { + const type = node.data.get('type') + if (typeof type === 'string') out.add(type) + } +} + +const createEditor = (types: Set): LexicalEditor => + createHeadlessEditor({ + nodes: [ + ...allHeadlessNodes, + ...[...types] + .filter((type) => !BUILTIN_TYPES.has(type)) + .map(passthroughNode), + ], + onError: (error) => { + throw error + }, + }) + +const parseState = (content: string): SerializedEditorState => + JSON.parse(content) as SerializedEditorState + +export const createDocument = (): LoroDoc => { + const doc = new LoroDoc() + doc.setRecordTimestamp(true) + return doc +} + +export function createContentDoc( + content: DocContent, + message: string, +): LoroDoc { + const doc = createDocument() + const state = parseState(content.content) + const types = new Set() + collectJsonTypes(state.root, types) + const editor = createEditor(types) + editor.setEditorState(editor.parseEditorState(state)) + doc.getText(TITLE).insert(0, content.title) + createLoroBinding(editor, doc, { commitMessage: message }).dispose() + doc.commit({ message }) + return doc +} + +export function readDocContent(doc: LoroDoc): DocContent { + const types = new Set() + collectDocTypes(doc, types) + const editor = createEditor(types) + if (doc.getTree(TREE_NAME).roots().length > 0) { + createLoroBinding(editor, doc).dispose() + } + return { + content: JSON.stringify(editor.getEditorState().toJSON()), + title: doc.getText(TITLE).toString(), + } +} + +export function readDocContentAt( + doc: LoroDoc, + frontiers: Frontiers, +): DocContent { + return readDocContent(doc.forkAt(frontiers)) +} + +export function editDocAt( + doc: LoroDoc, + frontiers: Frontiers, + target: DocContent, + message: string, + options: { peer?: PeerId } = {}, +): Uint8Array { + const fork = doc.forkAt(frontiers) + if (options.peer !== undefined) fork.setPeerId(options.peer) + fork.setRecordTimestamp(true) + const state = parseState(target.content) + const types = new Set() + collectDocTypes(fork, types) + collectJsonTypes(state.root, types) + const editor = createEditor(types) + const binding = createLoroBinding(editor, fork, { commitMessage: message }) + const from = fork.oplogVersion() + const title = fork.getText(TITLE) + if (title.toString() !== target.title) title.update(target.title) + editor.update(() => $reconcileRoot(state.root), { discrete: true }) + fork.commit({ message }) + binding.dispose() + return fork.export({ from, mode: 'update' }) +} + +export function revertChange( + doc: LoroDoc, + id: OpId, + before: OpId[], + message: string, +): Uint8Array { + const fork = doc.fork() + fork.setRecordTimestamp(true) + const from = fork.oplogVersion() + fork.applyDiff(doc.diff([id], before, false), { fullState: true }) + fork.commit({ message }) + return fork.export({ from, mode: 'update' }) +} + +export function vetUpdate( + doc: LoroDoc, + bytes: Uint8Array, + options: { author?: string } = {}, +): LoroDoc { + const probe = doc.fork() + if (probe.import(bytes).pending) { + throw new Error('update depends on changes missing from the document') + } + if (probe.getTree(TREE_NAME).roots().length > 1) { + throw new Error('update belongs to a different document lineage') + } + if (options.author !== undefined) { + const known = doc.oplogVersion() + for (const [peer, changes] of probe.getAllChanges()) { + const end = known.get(peer) ?? 0 + // A peer id stays bound to one author, or a forged op could claim ids another member + // is about to send and make the server drop theirs as duplicates. + const sharedPeer = + end > 0 && + doc.getChangeAt({ counter: end - 1, peer }).message !== options.author + for (const change of changes) { + if (change.counter + change.length <= end) continue + if (sharedPeer || change.message !== options.author) { + throw new Error( + `update carries changes not made by ${options.author} (author mismatch)`, + ) + } + } + } + } + readDocContent(probe) + return probe +} diff --git a/packages/editor/tsdown.config.ts b/packages/editor/tsdown.config.ts index e74b849fed4..d27a8701b10 100644 --- a/packages/editor/tsdown.config.ts +++ b/packages/editor/tsdown.config.ts @@ -3,7 +3,7 @@ import { defineConfig } from 'tsdown' export default defineConfig({ clean: true, target: 'es2022', - entry: ['src/index.ts', 'src/core/index.ts'], + entry: ['src/index.ts', 'src/core/index.ts', 'src/loro/index.ts'], outDir: 'dist', dts: true, format: ['esm'], diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 913db6b34c5..b5f6cc887a9 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -718,6 +718,9 @@ importers: '@fastify/static': specifier: 10.1.5 version: 10.1.5 + '@haklex/rich-collab-loro': + specifier: 0.55.0 + version: 0.55.0(lexical@0.52.0(typescript@6.0.3))(loro-crdt@1.16.4) '@haklex/rich-headless': specifier: 0.55.0 version: 0.55.0(lexical@0.52.0(typescript@6.0.3))(typescript@6.0.3) @@ -886,6 +889,9 @@ importers: linkedom: specifier: 0.18.13 version: 0.18.13 + loro-crdt: + specifier: ^1.16.4 + version: 1.16.4 lru-cache: specifier: 11.5.3 version: 11.5.3 @@ -1376,6 +1382,12 @@ importers: specifier: 0.52.0 version: 0.52.0(typescript@6.0.3) devDependencies: + '@haklex/rich-collab-loro': + specifier: 0.55.0 + version: 0.55.0(lexical@0.52.0(typescript@6.0.3))(loro-crdt@1.16.4) + loro-crdt: + specifier: ^1.16.4 + version: 1.16.4 tsdown: specifier: 0.23.0 version: 0.23.0(tsx@4.23.15)(typescript@6.0.3)