diff --git a/.changeset/grab-as-attachment.md b/.changeset/grab-as-attachment.md new file mode 100644 index 000000000..4cb046bae --- /dev/null +++ b/.changeset/grab-as-attachment.md @@ -0,0 +1,5 @@ +--- +'@conciv/embed': patch +--- + +Grabs are attachments: the snapshot survives a panel reload, sent grabs render as a card in the transcript instead of raw HTML text, and composer attachments now persist in the session draft. diff --git a/apps/conciv/src/app/pane-context.ts b/apps/conciv/src/app/pane-context.ts index bfa8db8ef..fee08a602 100644 --- a/apps/conciv/src/app/pane-context.ts +++ b/apps/conciv/src/app/pane-context.ts @@ -1,30 +1,6 @@ -import {createContext, createSignal, useContext, type Accessor} from 'solid-js' -import type {Grab, GrabProvider} from '@conciv/grab' - -export type StagedGrab = Grab | {text: string} - -export type PaneGrabStore = { - grabs: Accessor - stage: (grab: Grab) => void - stageTexts: (texts: string[]) => void - stageAll: (grabs: StagedGrab[]) => void - replace: (grab: StagedGrab, next: StagedGrab) => void - remove: (grab: StagedGrab) => void - clear: () => void -} - -export function makeGrabStore(): PaneGrabStore { - const [grabs, setGrabs] = createSignal([]) - return { - grabs, - stage: (grab) => setGrabs((prev) => [...prev, grab]), - stageTexts: (texts) => setGrabs(texts.map((text) => ({text}))), - stageAll: (staged) => setGrabs((prev) => [...prev, ...staged.filter((grab) => !prev.includes(grab))]), - replace: (grab, next) => setGrabs((prev) => prev.map((entry) => (entry === grab ? next : entry))), - remove: (grab) => setGrabs((prev) => prev.filter((entry) => entry !== grab)), - clear: () => setGrabs([]), - } -} +import {createContext, useContext, type Accessor} from 'solid-js' +import type {GrabProvider} from '@conciv/grab' +import type {GrabStaging} from '../pane/grab-staging.js' export type PendingAttachmentQueue = { enqueue: (file: File) => void @@ -38,7 +14,7 @@ export type PaneContextValue = { setLockedFor: (id: string) => (locked: boolean) => void slideClass: Accessor resetSlide: () => void - grabStore: PaneGrabStore + grabStaging: GrabStaging grabProvider: GrabProvider | undefined attachments: PendingAttachmentQueue newSession: () => void diff --git a/apps/conciv/src/app/pane-provider.tsx b/apps/conciv/src/app/pane-provider.tsx index 936640180..ba5760b98 100644 --- a/apps/conciv/src/app/pane-provider.tsx +++ b/apps/conciv/src/app/pane-provider.tsx @@ -2,7 +2,9 @@ import {type JSX} from 'solid-js' import {useQuery} from '@tanstack/solid-query' import {useRouter} from '@tanstack/solid-router' import {useAppData, useGrabProvider, useRpc} from './context.js' -import {PaneContext, makeGrabStore, makePendingAttachmentQueue, type PaneContextValue} from './pane-context.js' +import {PaneContext, makePendingAttachmentQueue, type PaneContextValue} from './pane-context.js' +import {makeGrabStaging} from '../pane/grab-staging.js' +import {resolveGrabSource} from '../pane/grab-source-resolve.js' export function PaneProvider(props: { sessionId: string @@ -30,7 +32,7 @@ export function PaneProvider(props: { setLockedFor: () => () => {}, slideClass: () => '', resetSlide: () => {}, - grabStore: makeGrabStore(), + grabStaging: makeGrabStaging({ground: (grab) => resolveGrabSource(grab, (input) => rpc.page.symbolicate(input))}), grabProvider, attachments: makePendingAttachmentQueue(), newSession: () => { diff --git a/apps/conciv/src/entry-standalone.tsx b/apps/conciv/src/entry-standalone.tsx index 45984986c..b3f1cc768 100644 --- a/apps/conciv/src/entry-standalone.tsx +++ b/apps/conciv/src/entry-standalone.tsx @@ -1,6 +1,7 @@ import {render} from 'solid-js/web' import {RouterProvider, createBrowserHistory} from '@tanstack/solid-router' import {makeBrowserRpcClient} from '@conciv/contract' +import pageExtension from '@conciv/extension-page/client' import {parseConcivSettings} from './data/settings.js' import {createConcivRouter} from './router.js' @@ -14,6 +15,7 @@ function start(): void { history: createBrowserHistory(), environment: {rootNode: document, document}, settings, + extensions: [pageExtension], apiBase: () => apiBase, }) const root = document.getElementById('app') diff --git a/apps/conciv/src/extension/create-instances.ts b/apps/conciv/src/extension/create-instances.ts new file mode 100644 index 000000000..8e9b8e09b --- /dev/null +++ b/apps/conciv/src/extension/create-instances.ts @@ -0,0 +1,21 @@ +import type {AnyExtension} from '@conciv/extension' +import type {ExtensionInstance} from './extension-slots.js' + +export function createInstances(extensions: AnyExtension[]): ExtensionInstance[] { + return extensions.flatMap((extension) => { + try { + const result = extension.__client?.() + return [ + { + extension, + clientValue: result?.value ?? {}, + effects: result?.effects ?? [], + dispose: result?.dispose ?? (() => {}), + }, + ] + } catch (error) { + console.error(`[conciv] extension "${extension.name}" failed to mount`, error) + return [] + } + }) +} diff --git a/apps/conciv/src/extension/pane-grab.ts b/apps/conciv/src/extension/pane-grab.ts index 9a0295ad8..0c13f8ba8 100644 --- a/apps/conciv/src/extension/pane-grab.ts +++ b/apps/conciv/src/extension/pane-grab.ts @@ -1,8 +1,8 @@ import type {GrabApi, GrabProvider} from '@conciv/grab' import {grabApi as pageGrabApi} from '@conciv/page' -import type {PaneGrabStore} from '../app/pane-context.js' +import type {GrabStaging} from '../pane/grab-staging.js' -export function makePaneGrabApi(store: PaneGrabStore, provider?: GrabProvider): GrabApi { +export function makePaneGrabApi(staging: GrabStaging, provider?: GrabProvider): GrabApi { const actions = provider?.() ?? pageGrabApi return { pick: actions.pick, @@ -10,8 +10,8 @@ export function makePaneGrabApi(store: PaneGrabStore, provider?: GrabProvider): cancel: actions.cancel, isActive: actions.isActive, grabbable: actions.grabbable, - stage: store.stage, - staged: () => store.grabs().flatMap((entry) => ('preview' in entry ? [entry] : [])), - clear: store.clear, + stage: staging.stage, + staged: staging.staged, + clear: staging.clear, } } diff --git a/apps/conciv/src/pane/chat-pane.tsx b/apps/conciv/src/pane/chat-pane.tsx index 07ecc10f4..08e0ebb95 100644 --- a/apps/conciv/src/pane/chat-pane.tsx +++ b/apps/conciv/src/pane/chat-pane.tsx @@ -37,15 +37,12 @@ import {collectToolRenderers} from '@conciv/extension' import {HostApiProvider} from '@conciv/extension/host' import type {Grab} from '@conciv/grab' import {paneAttachments} from './pane-attachments.js' -import {resolveGrabSource} from './grab-source-resolve.js' import {useAnnounce, useAppData, useConnected, useInstances, useRpc} from '../app/context.js' import {usePanelComposerFocus} from '../app/panel-focus.js' -import {usePane, type StagedGrab} from '../app/pane-context.js' +import {usePane} from '../app/pane-context.js' import {foldToolDurations} from './tool-durations.js' import {ToolFallbackCard} from './tool-fallback-card.js' import {useComposerTriggerSources} from './trigger-sources.js' -import {GrabReference} from './grab-reference.js' -import {GrabStrip} from './grab-strip.js' import {CompactSpinner, ConversationSkeleton, Divider, ThinkingBubble} from './indicators.js' import {ComposerActionsPending} from '../shell/pending.js' import {EmptyStateSlot} from '../shell/empty-state.js' @@ -96,24 +93,26 @@ function activeCallTitle( return title } -function grabTexts(grabs: ReadonlyArray): string[] { - return grabs.map((grab) => grab.text) -} - type ComposerApi = { - addAttachment: (file: File) => Promise + addAttachment: (file: File) => Promise } function ComposerWiring(props: {onReady: (api: ComposerApi) => void}): JSX.Element { const pane = usePane() const context = useComposerContext() onMount(() => { - const restored = context.grabs() - if (restored.length > 0) pane.grabStore.stageTexts(restored) + pane.grabStaging.connect({ + attachments: context.attachments, + addAttachment: context.addAttachment, + replaceAttachment: context.replaceAttachment, + removeAttachment: context.removeAttachment, + hasAttachment: context.hasAttachment, + }) props.onReady({addAttachment: context.addAttachment}) for (const file of pane.attachments.drain()) void context.addAttachment(file) }) - createEffect(() => context.setGrabs(grabTexts(pane.grabStore.grabs()))) + onCleanup(() => pane.grabStaging.disconnect()) + createEffect(() => pane.grabStaging.reconcile(context.attachments())) return <> } @@ -167,7 +166,6 @@ export function ChatPane(props: {sessionId: string}): JSX.Element { appData, reachability, notices, - grabStore: pane.grabStore, refetchMarkers: () => markers.refetch(), }) @@ -240,17 +238,11 @@ export function ChatPane(props: {sessionId: string}): JSX.Element { const PaneAttachment = (slotProps: {removable?: boolean}): JSX.Element => ( ) - const groundGrab = async (grab: Grab): Promise => { - const grounded = await resolveGrabSource(grab, (input) => rpc.page.symbolicate(input)) - if (!grounded) return - pane.grabStore.replace(grab, grounded) - } const stageGrab = (grab: Grab) => { - pane.grabStore.stage(grab) + pane.grabStaging.stage(grab) focusInput() - void groundGrab(grab) } - const paneGrab = makePaneGrabApi(pane.grabStore, pane.grabProvider) + const paneGrab = makePaneGrabApi(pane.grabStaging, pane.grabProvider) const dividersAt = (count: number): MarkerRow[] => (markers.data ?? []).filter((row) => row.afterTurn === count) const dividersInRange = (start: number, end: number): MarkerRow[] => @@ -333,13 +325,6 @@ export function ChatPane(props: {sessionId: string}): JSX.Element { - 0}> - - - {(grab) => pane.grabStore.remove(grab)} />} - - - {(storage) => ( diff --git a/apps/conciv/src/pane/draft-storage.ts b/apps/conciv/src/pane/draft-storage.ts index 89e2af826..70e49c975 100644 --- a/apps/conciv/src/pane/draft-storage.ts +++ b/apps/conciv/src/pane/draft-storage.ts @@ -6,9 +6,17 @@ import type {SelectionOffsets} from './composer-input-adapter.js' const WRITE_DELAY_MS = 300 +const PersistedAttachmentSchema = z.object({ + id: z.string(), + type: z.string(), + name: z.string(), + contentType: z.string(), + data: z.string(), +}) + const PersistedDraftSchema = z.object({ text: z.string().catch(''), - grabs: z.array(z.string()).catch([]), + attachments: z.array(PersistedAttachmentSchema).catch([]), }) type PersistedDraft = z.infer @@ -41,13 +49,13 @@ export async function appendDraft(rpc: RpcClient, sessionId: string, text: strin text: nextText, selectionStart: nextText.length, selectionEnd: nextText.length, - grabs: row?.grabs ?? [], + attachments: row?.attachments ?? [], }) } export async function makeDraftStorage(rpc: RpcClient, sessionId: string): Promise { const row = await rpc.drafts.get({sessionId}).catch(() => null) - let cache = row ? JSON.stringify({text: row.text, quote: null, grabs: row.grabs, attachments: []}) : null + let cache = row ? JSON.stringify({text: row.text, quote: null, attachments: row.attachments}) : null let selection: SelectionOffsets | null = null const write = debounce( (draft: PersistedDraft) => { @@ -58,7 +66,7 @@ export async function makeDraftStorage(rpc: RpcClient, sessionId: string): Promi text: draft.text, selectionStart: offsets.start, selectionEnd: offsets.end, - grabs: draft.grabs, + attachments: draft.attachments, }) .catch(() => {}) }, diff --git a/apps/conciv/src/pane/grab-reference.tsx b/apps/conciv/src/pane/grab-reference.tsx deleted file mode 100644 index 0536b1687..000000000 --- a/apps/conciv/src/pane/grab-reference.tsx +++ /dev/null @@ -1,97 +0,0 @@ -import {Show, type JSX} from 'solid-js' -import X from 'lucide-solid/icons/x' -import {TooltipIconButton} from '@conciv/ui-kit-system' -import type {GrabPreview, Grab} from '@conciv/grab' -import {sourceLabel} from './grab-source-label.js' - -function domPreview(preview: GrabPreview): Extract | null { - return preview.kind === 'dom' ? preview : null -} - -function imagePreview(preview: GrabPreview): Extract | null { - return preview.kind === 'image' ? preview : null -} - -function ScaledSnapshot(props: {preview: GrabPreview}): JSX.Element { - return ( - - {(preview) => ( - - )} - - } - > - {(preview) => ( - - -
el.appendChild(preview().node.cloneNode(true))} - /> - - - )} - - ) -} - -function stagedGrab(grab: Grab | {text: string}): Grab | null { - return 'preview' in grab ? grab : null -} - -export function GrabReference(props: {grab: Grab | {text: string}; onRemove: () => void}): JSX.Element { - return ( -
- props.onRemove()} - > - - {props.grab.text}} - > - {(grab) => ( - <> - - - {(source) => ( - - {(label) => ( - - {' '} - in {label()} - - )} - - )} - - - )} - -
- ) -} diff --git a/apps/conciv/src/pane/grab-source-label.ts b/apps/conciv/src/pane/grab-source-label.ts deleted file mode 100644 index e1d14ce47..000000000 --- a/apps/conciv/src/pane/grab-source-label.ts +++ /dev/null @@ -1,14 +0,0 @@ -import type {ElementSource} from '@conciv/grab' - -function whereLabel(source: ElementSource): string { - if (source.filePath === '') return '' - if (source.lineNumber === null) return source.filePath - return `${source.filePath}:${source.lineNumber}` -} - -export function sourceLabel(source: ElementSource): string { - const where = whereLabel(source) - if (!source.componentName) return where - if (where === '') return source.componentName - return `${source.componentName} at ${where}` -} diff --git a/apps/conciv/src/pane/grab-staging.ts b/apps/conciv/src/pane/grab-staging.ts new file mode 100644 index 000000000..d81baee04 --- /dev/null +++ b/apps/conciv/src/pane/grab-staging.ts @@ -0,0 +1,185 @@ +import {createSignal} from 'solid-js' +import { + GRAB_MIME, + grabToFile, + grabToPayload, + imagePreviewBudget, + parseGrabPayload, + type GrabPayload, +} from '@conciv/grab/grab-attachment' +import type {Grab} from '@conciv/grab' +import {fitImagePreview} from '@conciv/page' +import type {AttachmentState} from '@conciv/ui-kit-chat' + +export type ComposerGrabPort = { + attachments: () => readonly AttachmentState[] + addAttachment: (file: File) => Promise + replaceAttachment: (id: string, file: File) => Promise + removeAttachment: (id: string) => Promise + hasAttachment: (id: string) => boolean +} + +export type GrabStaging = { + stage: (grab: Grab) => void + staged: () => readonly Grab[] + clear: () => void + reconcile: (attachments: readonly AttachmentState[]) => void + connect: (port: ComposerGrabPort) => void + disconnect: () => void +} + +type Deps = { + ground: (grab: Grab) => Promise +} + +type StagedEntry = {grab: Grab; id: string | null; grounded: Grab | null; placing: Promise | null} + +function toGrab(payload: GrabPayload): Grab { + return { + text: payload.text, + ...(payload.snippet === undefined ? {} : {snippet: payload.snippet}), + preview: payload.preview ?? {kind: 'dom', html: '', width: 0, height: 0}, + source: payload.source, + rect: payload.rect, + } +} + +function isGrabAttachment(attachment: AttachmentState): boolean { + return attachment.contentType === GRAB_MIME +} + +async function fitted(grab: Grab): Promise { + const preview = grab.preview + if (preview.kind !== 'image') return grab + return {...grab, preview: await fitImagePreview(preview, imagePreviewBudget(grab, preview))} +} + +export function makeGrabStaging(deps: Deps): GrabStaging { + const [payloads, setPayloads] = createSignal>(new Map()) + const [pending, setPending] = createSignal([]) + const [port, setPort] = createSignal(null) + + const remember = (id: string, payload: GrabPayload): void => { + setPayloads((current) => new Map(current).set(id, payload)) + } + + const forget = (id: string): void => { + setPayloads((current) => new Map([...current].filter(([known]) => known !== id))) + } + + const drop = (entry: StagedEntry): void => { + setPending((current) => current.filter((candidate) => candidate !== entry)) + } + + const isLive = (entry: StagedEntry, composer: ComposerGrabPort): boolean => + port() === composer && pending().includes(entry) + + const putOnComposer = async (entry: StagedEntry, composer: ComposerGrabPort): Promise => { + try { + const prepared = await fitted(entry.grounded ?? entry.grab) + if (!isLive(entry, composer)) return + const id = await composer.addAttachment(grabToFile(prepared)) + if (!isLive(entry, composer)) { + if (id) void composer.removeAttachment(id) + return + } + entry.id = id + if (id) remember(id, grabToPayload(prepared)) + if (entry.grounded === null) return + drop(entry) + } finally { + if (entry.id === null) entry.placing = null + } + } + + const ensureOnComposer = (entry: StagedEntry): Promise => { + const composer = port() + if (!composer || entry.id !== null) return Promise.resolve() + if (entry.placing) return entry.placing + entry.placing = putOnComposer(entry, composer) + return entry.placing + } + + const settle = async (entry: StagedEntry): Promise => { + const composer = port() + if (!composer || entry.id === null || entry.grounded === null) return + const prepared = await fitted(entry.grounded) + if (!isLive(entry, composer)) return + const replaced = await composer.replaceAttachment(entry.id, grabToFile(prepared)) + if (!isLive(entry, composer)) { + if (replaced) void composer.removeAttachment(replaced) + return + } + forget(entry.id) + if (replaced) remember(replaced, grabToPayload(prepared)) + drop(entry) + } + + const ground = async (entry: StagedEntry): Promise => { + entry.grounded = await deps.ground(entry.grab) + if (entry.grounded === null) { + if (entry.id !== null) drop(entry) + return + } + await settle(entry) + } + + const placeThenGround = async (entry: StagedEntry): Promise => { + await ensureOnComposer(entry).catch(() => {}) + await ground(entry) + } + + const stage = (grab: Grab): void => { + const entry: StagedEntry = {grab, id: null, grounded: null, placing: null} + setPending((current) => [...current, entry]) + void placeThenGround(entry).catch(() => {}) + } + + const staged = (): readonly Grab[] => { + const composer = port() + const known = payloads() + const placed = (composer?.attachments() ?? []).filter(isGrabAttachment).flatMap((attachment) => { + const payload = known.get(attachment.id) + return payload ? [toGrab(payload)] : [] + }) + const waiting = pending().filter((entry) => entry.id === null) + return [...placed, ...waiting.map((entry) => entry.grounded ?? entry.grab)] + } + + const clear = (): void => { + setPending([]) + const composer = port() + if (!composer) return + for (const attachment of composer.attachments().filter(isGrabAttachment)) + void composer.removeAttachment(attachment.id) + } + + const hydrate = async (attachment: AttachmentState, file: File): Promise => { + const payload = parseGrabPayload(await file.text()) + const stillLive = (port()?.attachments() ?? []).some((entry) => entry.id === attachment.id) + if (payload && stillLive) remember(attachment.id, payload) + } + + const reconcile = (attachments: readonly AttachmentState[]): void => { + const grabs = attachments.filter(isGrabAttachment) + const live = new Set(grabs.map((attachment) => attachment.id)) + setPayloads((current) => new Map([...current].filter(([id]) => live.has(id)))) + for (const attachment of grabs) { + const file = attachment.file + if (payloads().has(attachment.id) || !file) continue + void hydrate(attachment, file) + } + } + + const connect = (next: ComposerGrabPort): void => { + setPort(() => next) + for (const entry of pending().filter((candidate) => candidate.id === null)) + void ensureOnComposer(entry).catch(() => {}) + } + + const disconnect = (): void => { + setPort(null) + } + + return {stage, staged, clear, reconcile, connect, disconnect} +} diff --git a/apps/conciv/src/pane/grab-strip.tsx b/apps/conciv/src/pane/grab-strip.tsx deleted file mode 100644 index 3944b5955..000000000 --- a/apps/conciv/src/pane/grab-strip.tsx +++ /dev/null @@ -1,64 +0,0 @@ -import {createResizable} from '@conciv/ui-kit-system' -import {createSignal, type JSX, type ParentProps} from 'solid-js' - -const GRAB_STRIP_MIN = 48 - -const GRABBER_HIT = - 'flex items-center justify-center h-4 w-24 cursor-ns-resize self-center shrink-0 touch-none focus-visible:outline-none [&:hover>div]:opacity-100 [&:hover>div]:bg-pw-text-3 [&:focus-visible>div]:opacity-100 [&:focus-visible>div]:bg-pw-accent' -const GRABBER_PILL = - 'rounded-full bg-pw-line-2 h-1.5 w-9 opacity-0 transition-opacity duration-150 group-hover:opacity-100' - -export function GrabStrip(props: ParentProps<{class: string}>): JSX.Element { - let scroller: HTMLDivElement | undefined - let content: HTMLDivElement | undefined - const resize = createResizable({ - initial: 288, - min: GRAB_STRIP_MIN, - max: () => content?.offsetHeight ?? Number.POSITIVE_INFINITY, - grow: () => 'up', - }) - const [sized, setSized] = createSignal(false) - const beginResize = () => { - if (scroller) resize.set(scroller.getBoundingClientRect().height) - setSized(true) - } - return ( -
- - ) -} diff --git a/apps/conciv/src/pane/pane-composer.tsx b/apps/conciv/src/pane/pane-composer.tsx index 224929e54..c35eee69d 100644 --- a/apps/conciv/src/pane/pane-composer.tsx +++ b/apps/conciv/src/pane/pane-composer.tsx @@ -104,7 +104,7 @@ export function PaneComposer(props: PaneComposerProps): JSX.Element { )}
-
+
( }> diff --git a/apps/conciv/src/pane/use-pane-messaging.ts b/apps/conciv/src/pane/use-pane-messaging.ts index 94e7d526c..778a5d710 100644 --- a/apps/conciv/src/pane/use-pane-messaging.ts +++ b/apps/conciv/src/pane/use-pane-messaging.ts @@ -4,7 +4,6 @@ import type {RpcClient} from '@conciv/contract' import type {MultimodalContent} from '@tanstack/ai-client' import type {UiAnswerValue} from '@conciv/protocol/ui-types' import type {AppData} from '../data/app-data.js' -import type {StagedGrab} from '../app/pane-context.js' import type {EngineNotices} from '../shell/notice-context.js' import {checkSend, type SendVerdict} from './send-checks.js' @@ -35,11 +34,6 @@ type PaneMessagingDeps = { appData: AppData reachability: EngineNotices['reachability'] notices: EngineNotices['notices'] - grabStore: { - grabs: () => StagedGrab[] - clear: () => void - stageAll: (grabs: StagedGrab[]) => void - } refetchMarkers: () => Promise } @@ -87,16 +81,7 @@ export function usePaneMessaging(deps: PaneMessagingDeps): PaneMessaging { reachable: deps.reachability.online(), }) if (!verdict.ok) throw sendRejection(verdict) - const staged = deps.grabStore.grabs() - deps.grabStore.clear() - const before = deps.chat.error() - try { - await deps.chat.sendMessage(content) - } catch (failure) { - deps.grabStore.stageAll(staged) - throw failure - } - if (deps.chat.error() !== before) deps.grabStore.stageAll(staged) + await deps.chat.sendMessage(content) } const onSendError = (failure: unknown): void => { diff --git a/apps/conciv/src/router.tsx b/apps/conciv/src/router.tsx index c2939e936..c027ed57c 100644 --- a/apps/conciv/src/router.tsx +++ b/apps/conciv/src/router.tsx @@ -10,6 +10,7 @@ import {routeTree} from './routeTree.gen' import {makeAppData, type AppData} from './data/app-data.js' import type {ConcivSettings} from './data/settings.js' import type {ExtensionInstance} from './extension/extension-slots.js' +import {createInstances} from './extension/create-instances.js' import highlight from './extensions/highlight.js' import {PendingPane} from './shell/pending.js' import {defaultErrorComponent} from './shell/default-error-component.js' @@ -49,25 +50,6 @@ export type ConcivRouterConfig = { connectionGeneration?: () => number } -function createInstances(extensions: AnyExtension[]): ExtensionInstance[] { - return extensions.flatMap((extension) => { - try { - const result = extension.__client?.() - return [ - { - extension, - clientValue: result?.value ?? {}, - effects: result?.effects ?? [], - dispose: result?.dispose ?? (() => {}), - }, - ] - } catch (error) { - console.error(`[conciv] extension "${extension.name}" failed to mount`, error) - return [] - } - }) -} - function disposeExtensionInstances(instances: ExtensionInstance[]): void { for (const instance of instances) { try { diff --git a/apps/conciv/src/routes/panel.$sessionId.$view.tsx b/apps/conciv/src/routes/panel.$sessionId.$view.tsx index 41294cfce..1dfadb24a 100644 --- a/apps/conciv/src/routes/panel.$sessionId.$view.tsx +++ b/apps/conciv/src/routes/panel.$sessionId.$view.tsx @@ -1,5 +1,5 @@ import {createFileRoute, useBlocker, useRouter} from '@tanstack/solid-router' -import {For, Show, Suspense, createMemo, type JSX} from 'solid-js' +import {Show, Suspense, createMemo, type JSX} from 'solid-js' import {HostApiProvider} from '@conciv/extension/host' import {MountedView} from '@conciv/extension/client' import {useAppData, useConnectionGeneration, useInstances, useRpc} from '../app/context.js' @@ -7,8 +7,6 @@ import {usePane} from '../app/pane-context.js' import {collectViews} from '../extension/extension-views.js' import {makePaneGrabApi} from '../extension/pane-grab.js' import {appendDraft} from '../pane/draft-storage.js' -import {GrabReference} from '../pane/grab-reference.js' -import {GrabStrip} from '../pane/grab-strip.js' export const Route = createFileRoute('/panel/$sessionId/$view')({component: PanelView}) @@ -45,7 +43,7 @@ function PanelView(): JSX.Element { {(mount) => ( params().sessionId} - grab={makePaneGrabApi(pane.grabStore, pane.grabProvider)} + grab={makePaneGrabApi(pane.grabStaging, pane.grabProvider)} insert={(text) => void appendDraft(rpc, params().sessionId, text).catch(() => {})} attach={(file) => pane.attachments.enqueue(file)} newSession={newSession} @@ -60,13 +58,6 @@ function PanelView(): JSX.Element { }} class={`flex flex-1 flex-col min-h-0 ${pane.slideClass()}`} > - 0}> - - - {(grab) => pane.grabStore.remove(grab)} />} - - - diff --git a/apps/conciv/src/routes/panel.$sessionId.tsx b/apps/conciv/src/routes/panel.$sessionId.tsx index 02d21800c..4fa56b4bc 100644 --- a/apps/conciv/src/routes/panel.$sessionId.tsx +++ b/apps/conciv/src/routes/panel.$sessionId.tsx @@ -8,7 +8,9 @@ import {For, Show, Suspense, createMemo, createSignal, type JSX} from 'solid-js' import {Dynamic} from 'solid-js/web' import {isSessionId} from '@conciv/protocol/chat-types' import {useAnnounce, useAppData, useDisconnect, useGrabProvider, useInstances, useRpc} from '../app/context.js' -import {PaneContext, makeGrabStore, makePendingAttachmentQueue, type PaneContextValue} from '../app/pane-context.js' +import {PaneContext, makePendingAttachmentQueue, type PaneContextValue} from '../app/pane-context.js' +import {makeGrabStaging} from '../pane/grab-staging.js' +import {resolveGrabSource} from '../pane/grab-source-resolve.js' import {SessionSelector} from '../composer/session-selector.js' import {usePanelChrome} from '../app/panel-chrome.js' import {ContextTracker} from '../pane/context-tracker.js' @@ -83,7 +85,9 @@ function PanelSession(): JSX.Element { announce('Started a new session') } - const grabStore = makeGrabStore() + const grabStaging = makeGrabStaging({ + ground: (grab) => resolveGrabSource(grab, (input) => rpc.page.symbolicate(input)), + }) useBlocker({ shouldBlockFn: ({current, next}) => @@ -97,7 +101,7 @@ function PanelSession(): JSX.Element { setLockedFor, slideClass, resetSlide: () => setSlideDir(null), - grabStore, + grabStaging, grabProvider, attachments: makePendingAttachmentQueue(), newSession: () => void newSession(), diff --git a/apps/conciv/test/chat-pane.browser.test.tsx b/apps/conciv/test/chat-pane.browser.test.tsx index bb9ee5b32..5bdbf66a5 100644 --- a/apps/conciv/test/chat-pane.browser.test.tsx +++ b/apps/conciv/test/chat-pane.browser.test.tsx @@ -2,14 +2,26 @@ import './helpers/utilities.css' import {afterAll, afterEach, beforeAll, expect, test} from 'vitest' import {page, userEvent} from 'vitest/browser' import type {RpcClient} from '@conciv/contract' +import pageExtension from '@conciv/extension-page/client' +import {GRAB_FILE_NAME, GRAB_MIME} from '@conciv/grab/grab-attachment' +import type {Grab} from '@conciv/grab' import {ChatPane} from '../src/pane/chat-pane.js' import {coreControl} from './helpers/core-control.js' import {coreRpc, createSession, openTranscriptStream, runTurn, seedDraft, sendTurn} from './helpers/core-session.js' -import {mountPane, type PaneMount} from './helpers/pane-harness.js' +import { + grabProviderFor, + HERO_GRAB, + HERO_LABEL, + persistedGrab, + PRICING_GRAB, + PRICING_LABEL, +} from './helpers/grab-fixtures.js' +import {mountPane, type PaneMount, type PaneMountOptions} from './helpers/pane-harness.js' import {trackedFaults} from './helpers/tracked-faults.js' const SEND_PATH = ['chat', 'send'] const SUBSCRIBE_PATH = ['chat', 'subscribe'] +const DRAFTS_SET_PATH = ['drafts', 'set'] const core = {base: ''} const active: {pane: PaneMount | null} = {pane: null} @@ -35,23 +47,38 @@ async function newSession(): Promise<{rpc: RpcClient; sessionId: string}> { return {rpc, sessionId: await createSession(rpc)} } -function mountChatPane(sessionId: string): PaneMount { - const mount = mountPane({base: core.base, sessionId}, () => ) +function grabOptions(...grabs: Grab[]): Pick { + return {grabProvider: grabProviderFor(...grabs), extensions: [pageExtension]} +} + +function mountChatPane( + sessionId: string, + options: Pick = {}, +): PaneMount { + const mount = mountPane({base: core.base, sessionId, ...options}, () => ) active.pane = mount return mount } const input = () => page.getByRole('textbox', {name: 'Message the conciv agent'}) -const removeGrab = () => page.getByRole('button', {name: 'Remove grabbed element'}) +const removeGrab = () => page.getByRole('button', {name: `Remove ${GRAB_FILE_NAME}`}) +const grabButton = () => page.getByRole('button', {name: 'Select an element from the page'}) +const snapshot = () => page.getByTitle('Grabbed element snapshot') const notifications = () => page.getByRole('region', {name: /Notifications/}) const stopButton = () => page.getByRole('button', {name: 'Stop generating'}) const skeleton = () => page.getByRole('status', {name: 'Loading conversation'}) +async function stageGrabThroughComposer(): Promise { + await expect.element(input()).toBeVisible() + await userEvent.click(grabButton()) + await expect.element(snapshot()).toBeVisible() + await expect.element(page.getByText(HERO_LABEL)).toBeVisible() +} + async function sendWithStagedGrab(): Promise { - const {rpc, sessionId} = await newSession() - await seedDraft(rpc, sessionId, {grabs: ['the grabbed hero section']}) - mountChatPane(sessionId) - await expect.element(removeGrab()).toBeVisible() + const {sessionId} = await newSession() + mountChatPane(sessionId, grabOptions(HERO_GRAB)) + await stageGrabThroughComposer() await input().fill('explain the section I grabbed') await userEvent.keyboard('{Enter}') } @@ -67,12 +94,33 @@ async function startStreamingRun(): Promise { test('restores the server-side draft text and staged grabs when the pane mounts', async () => { const {rpc, sessionId} = await newSession() - await seedDraft(rpc, sessionId, {text: 'kept across the reload', grabs: ['a grabbed heading']}) + await seedDraft(rpc, sessionId, { + text: 'kept across the reload', + attachments: [persistedGrab('grab-1', HERO_GRAB)], + }) - mountChatPane(sessionId) + mountChatPane(sessionId, grabOptions(HERO_GRAB)) await expect.element(input()).toHaveTextContent('kept across the reload') - await expect.element(page.getByText('a grabbed heading')).toBeVisible() + await expect.element(snapshot()).toBeVisible() + await expect.element(page.getByText(HERO_LABEL)).toBeVisible() +}) + +test('a staged grab keeps its snapshot and source label across a panel reload', async () => { + const {sessionId} = await newSession() + const since = await coreControl.rpcMark() + const first = mountChatPane(sessionId, grabOptions(HERO_GRAB)) + await stageGrabThroughComposer() + + expect(await coreControl.awaitRpcCall(DRAFTS_SET_PATH, since, {sessionId})).toBe(200) + const persisted = await coreRpc(core.base).drafts.get({sessionId}) + expect(persisted?.attachments.map((attachment) => attachment.contentType)).toEqual([GRAB_MIME]) + first.dispose() + + mountChatPane(sessionId, {extensions: [pageExtension]}) + + await expect.element(snapshot()).toBeVisible() + await expect.element(page.getByText(HERO_LABEL)).toBeVisible() }) test('a rejected send keeps the draft in the composer and tells the user why', async () => { @@ -102,7 +150,7 @@ test('a send the server refuses puts the staged grab card back', async () => { await expect.element(notifications()).toHaveTextContent(/Internal Server Error|could not be sent/) await expect.element(removeGrab()).toBeVisible() - await expect.element(page.getByText('the grabbed hero section', {exact: true})).toBeVisible() + await expect.element(page.getByText(HERO_LABEL).last()).toBeVisible() }) test('a send that throws at the transport puts the staged grab card back', async () => { @@ -111,32 +159,33 @@ test('a send that throws at the transport puts the staged grab card back', async await expect.element(notifications()).toHaveTextContent(/could not be sent|fetch/) await expect.element(removeGrab()).toBeVisible() - await expect.element(page.getByText('the grabbed hero section', {exact: true})).toBeVisible() + await expect.element(page.getByText(HERO_LABEL).last()).toBeVisible() }) test('a queued second send cannot cross-restore the grabs of the turn that failed', async () => { const {rpc, sessionId} = await newSession() - await seedDraft(rpc, sessionId, {grabs: ['the grabbed hero section']}) + await seedDraft(rpc, sessionId, {attachments: [persistedGrab('grab-1', HERO_GRAB)]}) await coreControl.holdTurn() - const mount = mountChatPane(sessionId) + mountChatPane(sessionId, grabOptions(PRICING_GRAB)) - await expect.element(page.getByText('the grabbed hero section', {exact: true})).toBeVisible() + await expect.element(page.getByText(HERO_LABEL)).toBeVisible() await input().fill('turn A') await userEvent.keyboard('{Enter}') await expect.element(stopButton()).toBeVisible() - mount.pane.grabStore.stageAll([{text: 'the grabbed pricing table'}]) - await expect.element(page.getByText('the grabbed pricing table', {exact: true})).toBeVisible() + await userEvent.click(grabButton()) + await expect.element(page.getByText(PRICING_LABEL)).toBeVisible() await input().fill('turn B') await userEvent.keyboard('{Enter}') await expect.element(page.getByRole('button', {name: 'Remove from queue'})).toBeVisible() - await expect.element(page.getByText('the grabbed pricing table', {exact: true})).not.toBeInTheDocument() + await expect.element(page.getByText(PRICING_LABEL)).not.toBeInTheDocument() await coreControl.scriptError('turn A failed') await coreControl.releaseTurn() - await expect.element(page.getByText('the grabbed hero section', {exact: true})).toBeVisible() - await expect.element(page.getByText('the grabbed pricing table', {exact: true})).not.toBeInTheDocument() + await expect.element(removeGrab()).toBeVisible() + await expect.element(page.getByText(HERO_LABEL).last()).toBeVisible() + await expect.element(page.getByText(PRICING_LABEL)).not.toBeInTheDocument() }) test('sending announces thinking and then the reply through the live region', async () => { diff --git a/apps/conciv/test/commands/core-control.ts b/apps/conciv/test/commands/core-control.ts index 677a1b716..518152707 100644 --- a/apps/conciv/test/commands/core-control.ts +++ b/apps/conciv/test/commands/core-control.ts @@ -174,8 +174,18 @@ const rpcCallCount: BrowserCommand<[string[]]> = (ctx, path): number => observer const rpcMark: BrowserCommand<[]> = (ctx): number => observerOf(ctx).mark() -const awaitRpcCall: BrowserCommand<[string[], number]> = async (ctx, path, since): Promise => { - const record = await observerOf(ctx).completed({path, since, timeout: AWAIT_RPC_TIMEOUT_MS}) +const awaitRpcCall: BrowserCommand<[string[], number, unknown?]> = async ( + ctx, + path, + since, + input, +): Promise => { + const record = await observerOf(ctx).completed({ + path, + since, + timeout: AWAIT_RPC_TIMEOUT_MS, + ...(input === undefined ? {} : {input}), + }) return record.status } diff --git a/apps/conciv/test/draft-storage.test.ts b/apps/conciv/test/draft-storage.test.ts index 64a3d3711..c09631805 100644 --- a/apps/conciv/test/draft-storage.test.ts +++ b/apps/conciv/test/draft-storage.test.ts @@ -8,7 +8,7 @@ import {proxyTo, type ProxyCore} from './helpers/proxy.js' const DRAFTS_GET_PATH = '/rpc/drafts/get' const DRAFTS_SET_PATH = '/rpc/drafts/set' -type StoredDraft = {sessionId: string; text: string; selectionStart: number; selectionEnd: number; grabs: string[]} +type StoredDraft = {sessionId: string; text: string; selectionStart: number; selectionEnd: number} const test = baseTest.extend<{kit: CoreKit; core: ProxyCore; sessionId: string}>({ kit: [ @@ -30,8 +30,8 @@ const test = baseTest.extend<{kit: CoreKit; core: ProxyCore; sessionId: string}> }, }) -function composerDraft(text: string, grabs: string[] = []): string { - return JSON.stringify({text, quote: null, grabs, attachments: []}) +function composerDraft(text: string): string { + return JSON.stringify({text, quote: null, attachments: []}) } async function storedDraft(rpc: RpcClient, sessionId: string): Promise { @@ -42,19 +42,17 @@ async function storedDraft(rpc: RpcClient, sessionId: string): Promise { - await seedDraft(kit.rpc, sessionId, {text: 'kept across the reload', grabs: ['a grabbed heading']}) + await seedDraft(kit.rpc, sessionId, {text: 'kept across the reload'}) const draftStorage = await makeDraftStorage(makeRpcClient(core.base), sessionId) expect(JSON.parse(draftStorage.storage.getItem('any') ?? '')).toEqual({ text: 'kept across the reload', quote: null, - grabs: ['a grabbed heading'], attachments: [], }) }) @@ -68,7 +66,7 @@ test('starts empty when the server has no draft', async ({core, sessionId}) => { test('writes the composer draft back to the server with the caret at the end', async ({kit, core, sessionId}) => { const draftStorage = await makeDraftStorage(makeRpcClient(core.base), sessionId) - draftStorage.storage.setItem('any', composerDraft('a fresh draft', ['a heading'])) + draftStorage.storage.setItem('any', composerDraft('a fresh draft')) await core.awaitRequest(DRAFTS_SET_PATH) @@ -77,7 +75,6 @@ test('writes the composer draft back to the server with the caret at the end', a text: 'a fresh draft', selectionStart: 13, selectionEnd: 13, - grabs: ['a heading'], }) }) @@ -95,7 +92,6 @@ test('collapses a burst of writes into the last draft', async ({kit, core, sessi text: 'abc', selectionStart: 3, selectionEnd: 3, - grabs: [], }) expect(core.requestCount(DRAFTS_SET_PATH)).toBe(1) }) @@ -115,7 +111,6 @@ test('keeps the latest value readable even when the payload cannot be persisted' text: 'a draft that parses', selectionStart: 19, selectionEnd: 19, - grabs: [], }) expect(core.requestCount(DRAFTS_SET_PATH)).toBe(1) }) @@ -133,7 +128,6 @@ test('persists the noted caret offsets with the draft text', async ({kit, core, text: 'say hello!', selectionStart: 4, selectionEnd: 4, - grabs: [], }) }) @@ -150,12 +144,11 @@ test('clamps noted offsets that fall beyond the persisted text', async ({kit, co text: 'short', selectionStart: 5, selectionEnd: 5, - grabs: [], }) }) test('appends to the stored draft on a new line with the caret at the end', async ({kit, core, sessionId}) => { - await seedDraft(kit.rpc, sessionId, {text: 'a first line', grabs: ['a heading']}) + await seedDraft(kit.rpc, sessionId, {text: 'a first line'}) await appendDraft(makeRpcClient(core.base), sessionId, 'a second line') @@ -164,10 +157,30 @@ test('appends to the stored draft on a new line with the caret at the end', asyn text: 'a first line\na second line', selectionStart: 26, selectionEnd: 26, - grabs: ['a heading'], }) }) +const PERSISTED = { + id: 'a1', + type: 'document', + name: 'Grabbed element', + contentType: 'application/vnd.conciv.grab+json', + data: 'eyJ4IjoxfQ==', +} + +test('an attachment written to the draft comes back on the next mount', async ({kit, core, sessionId}) => { + const first = await makeDraftStorage(makeRpcClient(core.base), sessionId) + + first.storage.setItem(sessionId, JSON.stringify({text: 'look at this', quote: null, attachments: [PERSISTED]})) + await core.awaitRequest(DRAFTS_SET_PATH) + + expect((await kit.rpc.drafts.get({sessionId}))?.attachments).toEqual([PERSISTED]) + + const second = await makeDraftStorage(makeRpcClient(core.base), sessionId) + + expect(JSON.parse(second.storage.getItem(sessionId) ?? '{}')).toMatchObject({attachments: [{id: 'a1'}]}) +}) + test('survives a failed initial read and still accepts writes', async ({kit, core, sessionId}) => { await seedDraft(kit.rpc, sessionId, {text: 'unreachable at boot'}) core.fail(DRAFTS_GET_PATH) @@ -187,6 +200,5 @@ test('survives a failed initial read and still accepts writes', async ({kit, cor text: 'after the outage', selectionStart: 16, selectionEnd: 16, - grabs: [], }) }) diff --git a/apps/conciv/test/grab-reference.browser.test.tsx b/apps/conciv/test/grab-reference.browser.test.tsx deleted file mode 100644 index 515ee0e55..000000000 --- a/apps/conciv/test/grab-reference.browser.test.tsx +++ /dev/null @@ -1,47 +0,0 @@ -import './helpers/utilities.css' -import {render} from '@solidjs/testing-library' -import {expect, test} from 'vitest' -import {page} from 'vitest/browser' -import type {Grab} from '@conciv/grab' -import {makeImageHostGrab} from '@conciv/extension-testkit/host/grab' -import {GrabReference} from '../src/pane/grab-reference.js' - -const IMAGE_DATA_URL = - 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAQAAAAECAIAAAAmkwkpAAAAEElEQVR4nGP4z8AARwzEcQCukw/x0F8jngAAAABJRU5ErkJggg==' - -function mount(grab: Grab | {text: string}): HTMLElement { - return render(() => {}} />).container -} - -function domGrab(size: {width: number; height: number} = {width: 200, height: 40}): Grab { - const node = document.createElement('div') - node.textContent = 'Payroll Deposit clone' - return { - text: 'Payroll Deposit', - preview: {kind: 'dom', node, width: size.width, height: size.height}, - source: null, - rect: {x: 0, y: 0, width: size.width, height: size.height}, - } -} - -function previewBox(host: HTMLElement): HTMLElement | null | undefined { - return host.querySelector('[data-pw-grab-scale]')?.parentElement -} - -test('grab reference renders the dom preview arm by appending the cloned node', () => { - const host = mount(domGrab()) - expect(page.getByText('Payroll Deposit clone').elements()).toHaveLength(1) - expect(host.querySelector('img')).toBeNull() -}) - -test('grab reference renders the image preview arm as an img element', () => { - const host = mount(makeImageHostGrab(IMAGE_DATA_URL)) - expect(host.querySelector('img')?.getAttribute('src')).toBe(IMAGE_DATA_URL) -}) - -test('a text-only grab has no preview box at all', () => { - const host = mount({text: 'just the copied text'}) - - expect(page.getByText('just the copied text').elements()).toHaveLength(1) - expect(previewBox(host)).toBeUndefined() -}) diff --git a/apps/conciv/test/grab-staging-module.browser.test.ts b/apps/conciv/test/grab-staging-module.browser.test.ts new file mode 100644 index 000000000..c226a6520 --- /dev/null +++ b/apps/conciv/test/grab-staging-module.browser.test.ts @@ -0,0 +1,370 @@ +import {createEffect, createRoot} from 'solid-js' +import {expect, it} from 'vitest' +import {GRAB_MIME, grabToFile, MAX_PAYLOAD_BYTES, parseGrabPayload} from '@conciv/grab/grab-attachment' +import type {Grab} from '@conciv/grab' +import type {AttachmentState} from '@conciv/ui-kit-chat' +import {makeGrabStaging, type ComposerGrabPort, type GrabStaging} from '../src/pane/grab-staging.js' + +const GRAB: Grab = { + text: '

Payroll Deposit

at src/routes/index.tsx:12:9', + snippet: '

Payroll Deposit

', + preview: {kind: 'dom', html: '

Payroll Deposit clone

', width: 200, height: 40}, + source: {componentName: 'Hero', filePath: 'src/routes/index.tsx', lineNumber: 12}, + rect: {x: 0, y: 0, width: 200, height: 40}, +} + +const GROUNDED: Grab = {...GRAB, text: '

Payroll Deposit

at src/routes/hero.tsx:4:1'} + +type FakeComposer = ComposerGrabPort & { + list: () => AttachmentState[] + nextAdd: () => Promise + nextRemove: () => Promise + nextAddStart: () => Promise + nextReplace: () => Promise + holdAdds: () => () => void + failNextAdd: () => void +} + +function fakeComposer(): FakeComposer { + const entries: AttachmentState[] = [] + const addWaiters: ((attachment: AttachmentState) => void)[] = [] + const removeWaiters: ((id: string) => void)[] = [] + const startWaiters: (() => void)[] = [] + const replaceWaiters: ((attachment: AttachmentState) => void)[] = [] + const gate: {open: Promise | null; release: () => void} = {open: null, release: () => {}} + const failure = {pending: 0} + let nextId = 0 + const entryFor = (id: string, file: File): AttachmentState => ({ + id, + type: 'document', + name: file.name, + contentType: file.type, + file, + status: {type: 'requires-action', reason: 'composer-send'}, + }) + return { + list: () => entries, + nextAdd: () => new Promise((resolve) => addWaiters.push(resolve)), + nextRemove: () => new Promise((resolve) => removeWaiters.push(resolve)), + nextAddStart: () => new Promise((resolve) => startWaiters.push(resolve)), + nextReplace: () => new Promise((resolve) => replaceWaiters.push(resolve)), + holdAdds: () => { + gate.open = new Promise((resolve) => { + gate.release = resolve + }) + return () => { + gate.open = null + gate.release() + } + }, + failNextAdd: () => { + failure.pending += 1 + }, + attachments: () => entries, + hasAttachment: (id) => entries.some((entry) => entry.id === id), + addAttachment: async (file) => { + startWaiters.splice(0).forEach((resolve) => resolve()) + if (failure.pending > 0) { + failure.pending -= 1 + throw new Error('the adapter refused the attachment') + } + if (gate.open) await gate.open + nextId += 1 + const id = `attachment-${nextId}` + const added = entryFor(id, file) + entries.push(added) + addWaiters.splice(0).forEach((resolve) => resolve(added)) + return id + }, + replaceAttachment: async (id, file) => { + const position = entries.findIndex((entry) => entry.id === id) + if (position < 0) return null + nextId += 1 + const next = `attachment-${nextId}` + const swapped = entryFor(next, file) + entries.splice(position, 1, swapped) + replaceWaiters.splice(0).forEach((resolve) => resolve(swapped)) + return next + }, + removeAttachment: async (id) => { + const position = entries.findIndex((entry) => entry.id === id) + if (position >= 0) entries.splice(position, 1) + removeWaiters.splice(0).forEach((resolve) => resolve(id)) + }, + } +} + +async function bodyOf(attachment: AttachmentState): Promise { + if (!attachment.file) throw new Error('expected the attachment to carry a file') + return attachment.file.text() +} + +async function grabTextOf(attachment: AttachmentState): Promise { + return parseGrabPayload(await bodyOf(attachment))?.text +} + +function imageGrab(width: number, height: number): Grab { + const canvas = document.createElement('canvas') + canvas.width = width + canvas.height = height + const context = canvas.getContext('2d') + if (!context) throw new Error('expected a canvas context') + const pixels = context.createImageData(width, height) + let noise = 1 + for (let index = 0; index < pixels.data.length; index += 1) { + noise = (noise * 1_103_515_245 + 12_345) % 2_147_483_648 + pixels.data[index] = index % 4 === 3 ? 255 : noise % 256 + } + context.putImageData(pixels, 0, 0) + return {...GRAB, preview: {kind: 'image', dataUrl: canvas.toDataURL('image/png'), width, height}} +} + +function stagedWhen(staging: GrabStaging, count: number): Promise { + let release = () => {} + const settledStaged = new Promise((resolve) => { + createRoot((disposeRoot) => { + release = disposeRoot + createEffect(() => { + const value = staging.staged() + if (value.length === count) resolve(value) + }) + }) + }) + return settledStaged.finally(() => release()) +} + +function neverGrounds(): Promise { + return Promise.resolve(null) +} + +it('attaches optimistically and replaces the payload once grounding resolves', async () => { + const composer = fakeComposer() + const staging = makeGrabStaging({ground: async () => GROUNDED}) + staging.connect(composer) + const replaced = composer.nextReplace() + + staging.stage(GRAB) + await replaced + + expect(composer.list()).toHaveLength(1) + expect(await grabTextOf(composer.list()[0] ?? never())).toBe(GROUNDED.text) +}) + +it('keeps the optimistic attachment when grounding yields nothing', async () => { + const composer = fakeComposer() + const staging = makeGrabStaging({ground: neverGrounds}) + staging.connect(composer) + const added = composer.nextAdd() + + staging.stage(GRAB) + await added + + expect(composer.list()).toHaveLength(1) + expect(await grabTextOf(composer.list()[0] ?? never())).toBe(GRAB.text) +}) + +it('attaches a grab staged before the composer connected, carrying its grounded payload', async () => { + const composer = fakeComposer() + const staging = makeGrabStaging({ground: async () => GROUNDED}) + + staging.stage(GRAB) + expect(await stagedWhen(staging, 1)).toHaveLength(1) + expect(composer.list()).toHaveLength(0) + + const added = composer.nextAdd() + staging.connect(composer) + await added + + expect(composer.list()).toHaveLength(1) + expect(await grabTextOf(composer.list()[0] ?? never())).toBe(GROUNDED.text) +}) + +it('attaches a grab staged before the composer connected even when grounding yields nothing', async () => { + const composer = fakeComposer() + const staging = makeGrabStaging({ground: neverGrounds}) + + staging.stage(GRAB) + expect(await stagedWhen(staging, 1)).toHaveLength(1) + + const added = composer.nextAdd() + staging.connect(composer) + await added + + expect(composer.list()).toHaveLength(1) + expect(await grabTextOf(composer.list()[0] ?? never())).toBe(GRAB.text) +}) + +it('reports a staged grab reactively once its payload is known', async () => { + const composer = fakeComposer() + const staging = makeGrabStaging({ground: neverGrounds}) + staging.connect(composer) + const seen: number[] = [] + const dispose = createRoot((disposeRoot) => { + createEffect(() => seen.push(staging.staged().length)) + return disposeRoot + }) + + staging.stage(GRAB) + await stagedWhen(staging, 1) + + expect(seen.at(-1)).toBe(1) + dispose() +}) + +it('clears only the grab attachments', async () => { + const composer = fakeComposer() + const staging = makeGrabStaging({ground: neverGrounds}) + staging.connect(composer) + await composer.addAttachment(new File(['hello'], 'notes.txt', {type: 'text/plain'})) + const added = composer.nextAdd() + staging.stage(GRAB) + await added + + const removed = composer.nextRemove() + staging.clear() + await removed + + expect(composer.list().map((entry) => entry.name)).toEqual(['notes.txt']) + expect(staging.staged()).toHaveLength(0) +}) + +it('hydrates a restored grab attachment and prunes one that is gone', async () => { + const composer = fakeComposer() + const staging = makeGrabStaging({ground: neverGrounds}) + staging.connect(composer) + const restored = await composer.addAttachment(grabToFile(GRAB)) + if (!restored) throw new Error('expected the restored attachment to be added') + + staging.reconcile(composer.attachments()) + + expect((await stagedWhen(staging, 1)).map((grab) => grab.text)).toEqual([GRAB.text]) + + await composer.removeAttachment(restored) + staging.reconcile(composer.attachments()) + + expect(await stagedWhen(staging, 0)).toHaveLength(0) +}) + +it('does not hydrate a payload whose attachment was removed while it decoded', async () => { + const composer = fakeComposer() + const staging = makeGrabStaging({ground: neverGrounds}) + staging.connect(composer) + const body = grabToFile(GRAB) + const restored = await composer.addAttachment(body) + if (!restored) throw new Error('expected the restored attachment to be added') + + staging.reconcile(composer.attachments()) + await composer.removeAttachment(restored) + await body.text() + + expect(staging.staged()).toHaveLength(0) +}) + +it('fits an oversized image preview into the payload budget instead of dropping it', async () => { + const composer = fakeComposer() + const oversized = imageGrab(1600, 1600) + const preview = oversized.preview + if (preview.kind !== 'image') throw new Error('expected an image preview') + expect(new TextEncoder().encode(preview.dataUrl).length).toBeGreaterThan(MAX_PAYLOAD_BYTES) + const staging = makeGrabStaging({ground: neverGrounds}) + staging.connect(composer) + const added = composer.nextAdd() + + staging.stage(oversized) + + const payload = parseGrabPayload(await bodyOf(await added)) + expect(payload?.preview?.kind).toBe('image') + expect( + payload?.preview?.kind === 'image' ? new TextEncoder().encode(payload.preview.dataUrl).length : 0, + ).toBeLessThanOrEqual(MAX_PAYLOAD_BYTES) +}) + +it('stages through a port that arrives and leaves again', async () => { + const composer = fakeComposer() + const staging: GrabStaging = makeGrabStaging({ground: async () => GROUNDED}) + staging.connect(composer) + const replaced = composer.nextReplace() + staging.stage(GRAB) + await replaced + await stagedWhen(staging, 1) + + staging.disconnect() + + expect(staging.staged()).toHaveLength(0) + expect(composer.list()).toHaveLength(1) + expect(composer.list()[0]?.contentType).toBe(GRAB_MIME) +}) + +function never(): never { + throw new Error('expected an attachment') +} + +it('a clear while a placement is in flight does not put the grab back on the composer', async () => { + const composer = fakeComposer() + const release = composer.holdAdds() + const staging = makeGrabStaging({ground: neverGrounds}) + staging.connect(composer) + const started = composer.nextAddStart() + + staging.stage(GRAB) + await started + const added = composer.nextAdd() + const removed = composer.nextRemove() + staging.clear() + release() + await added + await removed + + expect(composer.list()).toHaveLength(0) + expect(staging.staged()).toHaveLength(0) +}) + +it('a placement that lands after disconnect is redirected to the composer connected next', async () => { + const first = fakeComposer() + const second = fakeComposer() + const release = first.holdAdds() + const staging = makeGrabStaging({ground: neverGrounds}) + staging.connect(first) + const started = first.nextAddStart() + + staging.stage(GRAB) + await started + const firstAdded = first.nextAdd() + const firstRemoved = first.nextRemove() + staging.disconnect() + release() + await firstAdded + await firstRemoved + + const secondAdded = second.nextAdd() + staging.connect(second) + await secondAdded + + expect(first.list()).toHaveLength(0) + expect(second.list()).toHaveLength(1) +}) + +it('a grab whose placement fails can still be placed on the next connection', async () => { + const composer = fakeComposer() + composer.failNextAdd() + const grounding: {reached: () => void} = {reached: () => {}} + const groundingReached = new Promise((resolve) => { + grounding.reached = resolve + }) + const staging = makeGrabStaging({ + ground: async () => { + grounding.reached() + return null + }, + }) + staging.connect(composer) + + staging.stage(GRAB) + await groundingReached + + const added = composer.nextAdd() + staging.connect(composer) + await added + + expect(composer.list()).toHaveLength(1) +}) diff --git a/apps/conciv/test/grab-staging.browser.test.tsx b/apps/conciv/test/grab-staging.browser.test.tsx new file mode 100644 index 000000000..b691f1a1f --- /dev/null +++ b/apps/conciv/test/grab-staging.browser.test.tsx @@ -0,0 +1,53 @@ +import './helpers/utilities.css' +import {afterAll, afterEach, beforeAll, expect, test} from 'vitest' +import {page} from 'vitest/browser' +import pageExtension from '@conciv/extension-page/client' +import {GRAB_FILE_NAME} from '@conciv/grab/grab-attachment' +import {ChatPane} from '../src/pane/chat-pane.js' +import {makePaneGrabApi} from '../src/extension/pane-grab.js' +import {coreControl} from './helpers/core-control.js' +import {coreRpc, createSession} from './helpers/core-session.js' +import {HERO_GRAB, HERO_LABEL} from './helpers/grab-fixtures.js' +import {mountPane, type PaneMount} from './helpers/pane-harness.js' + +const core = {base: ''} +const active: {pane: PaneMount | null} = {pane: null} + +beforeAll(async () => { + const booted = await coreControl.bootCore({id: 'grab-staging', allowedOrigins: [window.location.origin]}) + core.base = booted.base +}, 60_000) + +afterAll(async () => { + await coreControl.closeCore() +}, 30_000) + +afterEach(() => { + active.pane?.dispose() + active.pane = null +}) + +const input = () => page.getByRole('textbox', {name: 'Message the conciv agent'}) +const snapshot = () => page.getByTitle('Grabbed element snapshot') +const removeGrab = () => page.getByRole('button', {name: `Remove ${GRAB_FILE_NAME}`}) + +test('the grab api an extension receives can stage, read and clear, as the terminal does', async () => { + const sessionId = await createSession(coreRpc(core.base)) + const mount = mountPane({base: core.base, sessionId, extensions: [pageExtension]}, () => ( + + )) + active.pane = mount + await expect.element(input()).toBeVisible() + const api = makePaneGrabApi(mount.pane.grabStaging, mount.pane.grabProvider) + + api.stage(HERO_GRAB) + + await expect.element(snapshot()).toBeVisible() + await expect.element(page.getByText(HERO_LABEL)).toBeVisible() + expect(api.staged().map((grab) => grab.text)).toEqual([HERO_GRAB.text]) + + api.clear() + + await expect.element(removeGrab()).not.toBeInTheDocument() + expect(api.staged()).toHaveLength(0) +}) diff --git a/apps/conciv/test/helpers/core-control.ts b/apps/conciv/test/helpers/core-control.ts index 0f805a36f..b58704d14 100644 --- a/apps/conciv/test/helpers/core-control.ts +++ b/apps/conciv/test/helpers/core-control.ts @@ -16,7 +16,7 @@ declare module 'vitest/internal/browser' { terminalLaunches: () => Promise rpcCallCount: (path: string[]) => Promise rpcMark: () => Promise - awaitRpcCall: (path: string[], since: number) => Promise + awaitRpcCall: (path: string[], since: number, input?: unknown) => Promise installFault: (spec: FaultSpec) => Promise releaseFault: (handle: string) => Promise faultPending: (handle: string) => Promise diff --git a/apps/conciv/test/helpers/core-session.ts b/apps/conciv/test/helpers/core-session.ts index 4df9d492b..0e7c8563a 100644 --- a/apps/conciv/test/helpers/core-session.ts +++ b/apps/conciv/test/helpers/core-session.ts @@ -1,7 +1,7 @@ import {EventType} from '@tanstack/ai' -import {makeRpcClient, type RpcClient} from '@conciv/contract' +import {makeRpcClient, type PersistedAttachment, type RpcClient} from '@conciv/contract' -export type SeededDraft = {text?: string; grabs?: string[]} +export type SeededDraft = {text?: string; attachments?: PersistedAttachment[]} export type TranscriptStream = {awaitTurnEnd: () => Promise; close: () => void} @@ -21,7 +21,7 @@ export async function seedDraft(rpc: RpcClient, sessionId: string, draft: Seeded text, selectionStart: text.length, selectionEnd: text.length, - grabs: draft.grabs ?? [], + attachments: draft.attachments ?? [], }) } diff --git a/apps/conciv/test/helpers/grab-fixtures.ts b/apps/conciv/test/helpers/grab-fixtures.ts new file mode 100644 index 000000000..0f50ddace --- /dev/null +++ b/apps/conciv/test/helpers/grab-fixtures.ts @@ -0,0 +1,46 @@ +import {GRAB_FILE_NAME, GRAB_MIME, grabToPayload} from '@conciv/grab/grab-attachment' +import type {Grab, GrabProvider} from '@conciv/grab' +import type {PersistedAttachment} from '@conciv/contract' + +export const HERO_GRAB: Grab = { + text: '

Payroll Deposit

at src/routes/index.tsx:12:9', + snippet: '

Payroll Deposit

', + preview: {kind: 'dom', html: '

Payroll Deposit clone

', width: 200, height: 40}, + source: {componentName: 'Hero', filePath: 'src/routes/index.tsx', lineNumber: 12}, + rect: {x: 0, y: 0, width: 200, height: 40}, +} + +export const PRICING_GRAB: Grab = { + text: 'Pricing
at src/routes/pricing.tsx:4:1', + snippet: 'Pricing
', + preview: {kind: 'dom', html: '

Pricing table clone

', width: 200, height: 40}, + source: {componentName: 'Pricing', filePath: 'src/routes/pricing.tsx', lineNumber: 4}, + rect: {x: 0, y: 0, width: 200, height: 40}, +} + +export const HERO_LABEL = 'Hero at src/routes/index.tsx:12' + +export const PRICING_LABEL = 'Pricing at src/routes/pricing.tsx:4' + +export function tallGrab(componentName: string, height: number): Grab { + return { + text: `
${componentName}
at src/routes/${componentName}.tsx:1:1`, + snippet: `
${componentName}
`, + preview: {kind: 'dom', html: `

${componentName} clone

`, width: 200, height}, + source: {componentName, filePath: `src/routes/${componentName}.tsx`, lineNumber: 1}, + rect: {x: 0, y: 0, width: 200, height}, + } +} + +export function persistedGrab(id: string, grab: Grab): PersistedAttachment { + const bytes = new TextEncoder().encode(JSON.stringify(grabToPayload(grab))) + let binary = '' + for (const byte of bytes) binary += String.fromCharCode(byte) + return {id, type: 'document', name: GRAB_FILE_NAME, contentType: GRAB_MIME, data: btoa(binary)} +} + +export function grabProviderFor(...grabs: Grab[]): GrabProvider { + const queue = [...grabs] + const take = async (): Promise => (queue.length > 1 ? (queue.shift() ?? HERO_GRAB) : (queue[0] ?? HERO_GRAB)) + return () => ({pick: take, comment: take, cancel: () => {}, isActive: () => false}) +} diff --git a/apps/conciv/test/helpers/pane-harness.tsx b/apps/conciv/test/helpers/pane-harness.tsx index bc8ffd8a1..f74384488 100644 --- a/apps/conciv/test/helpers/pane-harness.tsx +++ b/apps/conciv/test/helpers/pane-harness.tsx @@ -6,18 +6,23 @@ import '../../src/lib/api-base.js' import {HostApiProvider} from '@conciv/extension/host' import {AppContext, type AppContextValue} from '../../src/app/context.js' import {EngineReachabilityContext, makeEngineReachability} from '../../src/app/reachability.js' -import { - PaneContext, - makeGrabStore, - makePendingAttachmentQueue, - type PaneContextValue, -} from '../../src/app/pane-context.js' +import type {AnyExtension} from '@conciv/extension' +import type {Grab, GrabProvider} from '@conciv/grab' +import {PaneContext, makePendingAttachmentQueue, type PaneContextValue} from '../../src/app/pane-context.js' +import {createInstances} from '../../src/extension/create-instances.js' +import {makeGrabStaging} from '../../src/pane/grab-staging.js' import {makeAppData, type AppData} from '../../src/data/app-data.js' import {parseConcivSettings} from '../../src/data/settings.js' import {makeLayerStack} from '../../src/shell/dialogs.js' import {NoticeContextProvider, NoticeSurface} from '../../src/shell/notice-context.js' -export type PaneMountOptions = {base: string; sessionId: string} +export type PaneMountOptions = { + base: string + sessionId: string + grabProvider?: GrabProvider + extensions?: AnyExtension[] + ground?: (grab: Grab) => Promise +} export type PaneMount = { dispose: () => void @@ -41,6 +46,7 @@ export function mountPane(options: PaneMountOptions, view: (pane: PaneContextVal window.__CONCIV_API_BASE__ = options.base browserRpcConnection(options.base, 'fetch') const queryClient = new QueryClient() + const instances = createInstances(options.extensions ?? []) const data = makeAppData(rpc, queryClient) const [announced, setAnnounced] = createSignal([]) const app: AppContextValue = { @@ -53,7 +59,7 @@ export function mountPane(options: PaneMountOptions, view: (pane: PaneContextVal layers: makeLayerStack(), suppressed: () => undefined, fabPosition: () => 'bottom-right', - instances: [], + instances, connected: () => true, arrivedFromConnect: () => false, connectBind: async () => '', @@ -68,8 +74,8 @@ export function mountPane(options: PaneMountOptions, view: (pane: PaneContextVal setLockedFor: () => () => {}, slideClass: () => '', resetSlide: () => {}, - grabStore: makeGrabStore(), - grabProvider: undefined, + grabStaging: makeGrabStaging({ground: options.ground ?? (async () => null)}), + grabProvider: options.grabProvider, attachments: makePendingAttachmentQueue(), newSession: () => {}, } @@ -99,6 +105,7 @@ export function mountPane(options: PaneMountOptions, view: (pane: PaneContextVal return { dispose: () => { mounted.unmount() + for (const instance of instances) instance.dispose() reachabilityRoot.dispose() closeBrowserRpcConnection(options.base) delete window.__CONCIV_API_BASE__ diff --git a/apps/conciv/test/pane-attachments.browser.test.ts b/apps/conciv/test/pane-attachments.browser.test.ts index 9c24435d1..8f6d06c01 100644 --- a/apps/conciv/test/pane-attachments.browser.test.ts +++ b/apps/conciv/test/pane-attachments.browser.test.ts @@ -1,5 +1,6 @@ import {describe, expect, it} from 'vitest' import {defineAttachment, defineExtension} from '@conciv/extension' +import pageExtension from '@conciv/extension-page/client' import {paneAttachments} from '../src/pane/pane-attachments.js' function fixtureExtension() { @@ -18,6 +19,20 @@ describe('paneAttachments', () => { expect(pending.type).toBe('document') }) + it('accepts grab attachments when the page extension is installed', () => { + const {cards, adapter} = paneAttachments([pageExtension], false) + + expect(cards.some((entry) => entry.mime === 'application/vnd.conciv.grab+json')).toBe(true) + expect(adapter.accept).toContain('application/vnd.conciv.grab+json') + }) + + it('does not accept grab attachments with no extensions installed', () => { + const {cards, adapter} = paneAttachments([], false) + + expect(cards.some((entry) => entry.mime === 'application/vnd.conciv.grab+json')).toBe(false) + expect(adapter.accept).not.toContain('application/vnd.conciv.grab+json') + }) + it('still accepts text files and gates images on harness support', () => { expect(paneAttachments([], false).adapter.accept).not.toContain('image/*') expect(paneAttachments([], 'native').adapter.accept).toContain('image/*') diff --git a/apps/conciv/test/pane-harness-disposal.browser.test.tsx b/apps/conciv/test/pane-harness-disposal.browser.test.tsx new file mode 100644 index 000000000..5ee5c29c9 --- /dev/null +++ b/apps/conciv/test/pane-harness-disposal.browser.test.tsx @@ -0,0 +1,25 @@ +import './helpers/utilities.css' +import {expect, test} from 'vitest' +import {defineExtension} from '@conciv/extension' +import {mountPane} from './helpers/pane-harness.js' + +test('disposing a mounted pane disposes every extension instance it created', () => { + const order: string[] = [] + const first = defineExtension({name: 'pane-harness-disposal-a'}).client(() => ({ + value: {}, + dispose: () => order.push('a'), + })) + const second = defineExtension({name: 'pane-harness-disposal-b'}).client(() => ({ + value: {}, + dispose: () => order.push('b'), + })) + const mount = mountPane({base: 'http://127.0.0.1:9', sessionId: 'conciv_1', extensions: [first, second]}, () => ( +
pane
+ )) + + expect(order).toEqual([]) + + mount.dispose() + + expect(order).toEqual(['a', 'b']) +}) diff --git a/apps/conciv/test/panel-min-height.browser.test.tsx b/apps/conciv/test/panel-min-height.browser.test.tsx index 605975e01..64a866704 100644 --- a/apps/conciv/test/panel-min-height.browser.test.tsx +++ b/apps/conciv/test/panel-min-height.browser.test.tsx @@ -1,11 +1,18 @@ import './helpers/utilities.css' import {afterAll, afterEach, beforeAll, expect, test} from 'vitest' import {page} from 'vitest/browser' +import pageExtension from '@conciv/extension-page/client' import {coreControl} from './helpers/core-control.js' import {coreRpc, createSession, seedDraft} from './helpers/core-session.js' +import {persistedGrab, tallGrab} from './helpers/grab-fixtures.js' import {createShellHarness} from './helpers/shell-harness.js' -const STAGED = ['the grabbed hero section', 'the grabbed price row', 'the grabbed footer', 'the grabbed nav'] +const STAGED = [ + persistedGrab('grab-1', tallGrab('Hero', 900)), + persistedGrab('grab-2', tallGrab('Pricing', 900)), + persistedGrab('grab-3', tallGrab('Footer', 900)), + persistedGrab('grab-4', tallGrab('Nav', 900)), +] const core = {base: ''} const harness = createShellHarness(() => core.base) @@ -24,8 +31,8 @@ afterEach(harness.dispose) async function openPanel(): Promise { const rpc = coreRpc(core.base) const sessionId = await createSession(rpc) - await seedDraft(rpc, sessionId, {grabs: STAGED}) - harness.mountShell(`/panel/${sessionId}?open=true`) + await seedDraft(rpc, sessionId, {attachments: STAGED}) + harness.mountShell(`/panel/${sessionId}?open=true`, [pageExtension]) } const input = () => page.getByRole('textbox', {name: 'Message the conciv agent'}) @@ -34,7 +41,8 @@ test('the composer stays reachable when the viewport clamps the panel below its await page.viewport(1000, 400) await openPanel() - await expect.element(page.getByText('the grabbed hero section')).toBeVisible() + await expect.element(page.getByText('Hero at src/routes/Hero.tsx:1')).toBeVisible() + await expect.element(input()).toBeInViewport({ratio: 1}) await input().click() await input().fill('still typeable at the smallest panel') @@ -43,31 +51,13 @@ test('the composer stays reachable when the viewport clamps the panel below its await expect.element(page.getByText('still typeable at the smallest panel')).toBeVisible() }, 30_000) -test('the grabs strip resizes through the shared separator handle', async () => { +test('staged grab cards never push the composer out of the viewport as they pile up', async () => { await page.viewport(1000, 900) await openPanel() - const handle = page.getByRole('separator', {name: 'Resize grabs height'}) - await expect.element(handle).toBeVisible() - await expect.element(handle).toHaveAttribute('aria-valuenow', '288') - await expect.element(page.getByText('the grabbed hero section')).toBeVisible() - await expect.element(page.getByText('the grabbed nav')).toBeVisible() -}, 30_000) - -test('the staged grabs come back into the flow once the panel has room again', async () => { - await page.viewport(1000, 400) - await openPanel() - - await expect.element(input()).toBeVisible() + await expect.element(page.getByText('Hero at src/routes/Hero.tsx:1')).toBeVisible() + await expect.element(page.getByText('Nav at src/routes/Nav.tsx:1')).toBeVisible() - await page.viewport(1000, 900) - - await expect.element(page.getByText('the grabbed hero section')).toBeVisible() - await expect.element(page.getByText('the grabbed nav')).toBeVisible() - await page.getByRole('button', {name: 'Remove grabbed element'}).last().click() - await expect.element(page.getByText('the grabbed nav')).not.toBeInTheDocument() - await input().click() - await input().fill('room to breathe') - await page.getByRole('button', {name: 'Send message'}).click() - await expect.element(page.getByText('room to breathe')).toBeVisible() + await expect.element(input()).toBeInViewport({ratio: 1}) + await expect.element(page.getByRole('button', {name: 'Send message'})).toBeInViewport({ratio: 1}) }, 30_000) diff --git a/packages/contract/src/rows.ts b/packages/contract/src/rows.ts index ad778f4b5..ae5fec623 100644 --- a/packages/contract/src/rows.ts +++ b/packages/contract/src/rows.ts @@ -19,12 +19,25 @@ export const SessionMetaSchema = z.object({ }) export type SessionMeta = z.infer +export const PersistedAttachmentSchema = z.object({ + id: z.string(), + type: z.string(), + name: z.string(), + contentType: z.string(), + data: z.string(), +}) +export type PersistedAttachment = z.infer + export const DraftRowSchema = z.object({ sessionId: z.string(), text: z.string(), selectionStart: z.number().int().min(0), selectionEnd: z.number().int().min(0), - grabs: z.array(z.string()), + attachments: z + .array(PersistedAttachmentSchema) + .nullable() + .default([]) + .transform((value) => value ?? []), updatedAt: z.number(), }) export type DraftRow = z.infer diff --git a/packages/contract/test/rows.test.ts b/packages/contract/test/rows.test.ts index bc37338be..2b6bc97af 100644 --- a/packages/contract/test/rows.test.ts +++ b/packages/contract/test/rows.test.ts @@ -22,15 +22,15 @@ describe('row schemas', () => { expect(() => MarkerRowSchema.parse({id: 'm1', sessionId: 'conciv_1', afterTurn: 2, kind: 'weird'})).toThrow() }) - it('drafts are explicit about selection and grabs', () => { + it('drafts are explicit about selection and default their attachments', () => { const draft = DraftRowSchema.parse({ sessionId: 'conciv_1', text: 'hi', selectionStart: 2, selectionEnd: 2, - grabs: ['
'], updatedAt: 5, }) - expect(draft.grabs).toEqual(['
']) + expect(draft.selectionEnd).toBe(2) + expect(draft.attachments).toEqual([]) }) }) diff --git a/packages/core/src/chat/run.ts b/packages/core/src/chat/run.ts index d77fdb2c7..74602c972 100644 --- a/packages/core/src/chat/run.ts +++ b/packages/core/src/chat/run.ts @@ -451,15 +451,6 @@ export async function expandUserParts(content: UserContent, expanders: Attachmen return expanded } -async function composeUserContent(db: ConcivDb, sessionId: string, content: UserContent): Promise { - const rows = await db.select({grabs: drafts.grabs}).from(drafts).where(eq(drafts.sessionId, sessionId)) - const grabs = rows[0]?.grabs ?? [] - if (grabs.length === 0) return content - const prefix = grabs.join('\n') - if (typeof content === 'string') return content ? `${prefix}\n${content}` : prefix - return [{type: 'text', content: `${prefix}\n`}, ...content] -} - export type Send = (sessionId: string, runId: string, content: UserContent) => Promise const RUN_ID_TAKEN_ERROR_NAME = 'RunIdTakenError' @@ -477,8 +468,7 @@ export function isRunIdTakenError(error: unknown): error is Error { async function prepareLaunchContent(deps: ChatDeps, sessionId: string, content: UserContent): Promise { deps.onRunStart?.(sessionId) await ensureRow(deps.db, sessionId, deps.harness.id, deps.cwd) - const userContent = await composeUserContent(deps.db, sessionId, content) - return expandUserParts(userContent, deps.attachmentExpanders) + return expandUserParts(content, deps.attachmentExpanders) } async function settleLiveRuns(deps: ChatDeps, sessionId: string): Promise { diff --git a/packages/core/test/rpc/wire.it.test.ts b/packages/core/test/rpc/wire.it.test.ts index 69c735db5..aef4ee6ac 100644 --- a/packages/core/test/rpc/wire.it.test.ts +++ b/packages/core/test/rpc/wire.it.test.ts @@ -71,7 +71,7 @@ describe('rpc over the wire (real app, real http, typed client)', () => { expect(types).toContain(EventType.RUN_STARTED) }) - it('send consumes the server-side draft: grabs prefix the turn, row cleared', async () => { + it('send consumes the server-side draft: the turn is the user text alone and the row is cleared', async () => { const {kit} = await bootWire() const sessionId = await kit.session() const stream = await kit.attach(sessionId) @@ -80,7 +80,6 @@ describe('rpc over the wire (real app, real http, typed client)', () => { text: 'draft-text', selectionStart: 0, selectionEnd: 0, - grabs: ['
'], }) await kit.rpc.chat.send({runId: 'wire-5', sessionId, text: 'about the grabbed element'}) await stream.done({hangGuardMs: 10_000}) @@ -89,12 +88,11 @@ describe('rpc over the wire (real app, real http, typed client)', () => { ) const firstPart = partsOf(visibleUser)[0] const text = isRecord(firstPart) && typeof firstPart.content === 'string' ? firstPart.content : '' - expect(text.startsWith('
\n')).toBe(true) - expect(text).toContain('about the grabbed element') + expect(text).toBe('about the grabbed element') expect(await kit.rpc.drafts.get({sessionId})).toBeNull() }) - it('send forwards multimodal content and keeps grab references as a text prefix', async () => { + it('send forwards multimodal content untouched by the draft row', async () => { const {kit} = await bootWire() const sessionId = await kit.session() const stream = await kit.attach(sessionId) @@ -103,7 +101,6 @@ describe('rpc over the wire (real app, real http, typed client)', () => { text: 'draft-text', selectionStart: 0, selectionEnd: 0, - grabs: [''], }) await kit.rpc.chat.send({ runId: 'wire-12', @@ -120,7 +117,6 @@ describe('rpc over the wire (real app, real http, typed client)', () => { if (!isRecord(visibleUser) || !('parts' in visibleUser)) throw new Error('stream did not include the user message parts') expect(visibleUser.parts).toEqual([ - {type: 'text', content: '\n'}, {type: 'text', content: 'what color is this? '}, {type: 'image', source: {type: 'data', mimeType: 'image/png', value: 'iVBORw0KGgo='}}, ]) diff --git a/packages/db/drizzle/20260814231022_draft_attachments/migration.sql b/packages/db/drizzle/20260814231022_draft_attachments/migration.sql new file mode 100644 index 000000000..cec3713e8 --- /dev/null +++ b/packages/db/drizzle/20260814231022_draft_attachments/migration.sql @@ -0,0 +1 @@ +ALTER TABLE `drafts` ADD `attachments` text; \ No newline at end of file diff --git a/packages/db/drizzle/20260814231022_draft_attachments/snapshot.json b/packages/db/drizzle/20260814231022_draft_attachments/snapshot.json new file mode 100644 index 000000000..3b4bc06e8 --- /dev/null +++ b/packages/db/drizzle/20260814231022_draft_attachments/snapshot.json @@ -0,0 +1,695 @@ +{ + "version": "7", + "dialect": "sqlite", + "id": "bd003028-f108-43e2-a0eb-585be16e9020", + "prevIds": ["5250e755-c403-442d-adbb-eabdd6688bcf"], + "ddl": [ + { + "name": "css_bundles", + "entityType": "tables" + }, + { + "name": "drafts", + "entityType": "tables" + }, + { + "name": "markers", + "entityType": "tables" + }, + { + "name": "navigation", + "entityType": "tables" + }, + { + "name": "sessions", + "entityType": "tables" + }, + { + "name": "tool_captures", + "entityType": "tables" + }, + { + "name": "replies", + "entityType": "tables" + }, + { + "name": "run_messages", + "entityType": "tables" + }, + { + "name": "runs", + "entityType": "tables" + }, + { + "name": "image_history", + "entityType": "tables" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "hash", + "entityType": "columns", + "table": "css_bundles" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "css", + "entityType": "columns", + "table": "css_bundles" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "created_at", + "entityType": "columns", + "table": "css_bundles" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "session_id", + "entityType": "columns", + "table": "drafts" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "text", + "entityType": "columns", + "table": "drafts" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "selection_start", + "entityType": "columns", + "table": "drafts" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "selection_end", + "entityType": "columns", + "table": "drafts" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "grabs", + "entityType": "columns", + "table": "drafts" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "attachments", + "entityType": "columns", + "table": "drafts" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "updated_at", + "entityType": "columns", + "table": "drafts" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "markers" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "session_id", + "entityType": "columns", + "table": "markers" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "after_turn", + "entityType": "columns", + "table": "markers" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "kind", + "entityType": "columns", + "table": "markers" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": "'navigation'", + "generated": null, + "name": "id", + "entityType": "columns", + "table": "navigation" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "entries", + "entityType": "columns", + "table": "navigation" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "index", + "entityType": "columns", + "table": "navigation" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "updated_at", + "entityType": "columns", + "table": "navigation" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "sessions" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "harness_session_id", + "entityType": "columns", + "table": "sessions" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "harness_kind", + "entityType": "columns", + "table": "sessions" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "origin", + "entityType": "columns", + "table": "sessions" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "title", + "entityType": "columns", + "table": "sessions" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "model", + "entityType": "columns", + "table": "sessions" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "usage", + "entityType": "columns", + "table": "sessions" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "cwd", + "entityType": "columns", + "table": "sessions" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "transcript_cwd", + "entityType": "columns", + "table": "sessions" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "attached_pid", + "entityType": "columns", + "table": "sessions" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "attached_at", + "entityType": "columns", + "table": "sessions" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "deleted_at", + "entityType": "columns", + "table": "sessions" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "created_at", + "entityType": "columns", + "table": "sessions" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "updated_at", + "entityType": "columns", + "table": "sessions" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "tool_call_id", + "entityType": "columns", + "table": "tool_captures" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "kind", + "entityType": "columns", + "table": "tool_captures" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "session_id", + "entityType": "columns", + "table": "tool_captures" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "css_bundle_id", + "entityType": "columns", + "table": "tool_captures" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "payload", + "entityType": "columns", + "table": "tool_captures" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "created_at", + "entityType": "columns", + "table": "tool_captures" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "session_id", + "entityType": "columns", + "table": "replies" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "key", + "entityType": "columns", + "table": "replies" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "value", + "entityType": "columns", + "table": "replies" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "created_at", + "entityType": "columns", + "table": "replies" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "session_id", + "entityType": "columns", + "table": "run_messages" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "messages", + "entityType": "columns", + "table": "run_messages" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "updated_at", + "entityType": "columns", + "table": "run_messages" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "session_id", + "entityType": "columns", + "table": "runs" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": "'idle'", + "generated": null, + "name": "status", + "entityType": "columns", + "table": "runs" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "0", + "generated": null, + "name": "run_epoch", + "entityType": "columns", + "table": "runs" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "last_error", + "entityType": "columns", + "table": "runs" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "last_error_epoch", + "entityType": "columns", + "table": "runs" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "updated_at", + "entityType": "columns", + "table": "runs" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "session_id", + "entityType": "columns", + "table": "image_history" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "messages", + "entityType": "columns", + "table": "image_history" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "updated_at", + "entityType": "columns", + "table": "image_history" + }, + { + "columns": ["tool_call_id", "kind", "session_id"], + "nameExplicit": true, + "name": "tool_captures_pk", + "entityType": "pks", + "table": "tool_captures" + }, + { + "columns": ["session_id", "key"], + "nameExplicit": false, + "name": "replies_pk", + "entityType": "pks", + "table": "replies" + }, + { + "columns": ["hash"], + "nameExplicit": false, + "name": "css_bundles_pk", + "table": "css_bundles", + "entityType": "pks" + }, + { + "columns": ["session_id"], + "nameExplicit": false, + "name": "drafts_pk", + "table": "drafts", + "entityType": "pks" + }, + { + "columns": ["id"], + "nameExplicit": false, + "name": "markers_pk", + "table": "markers", + "entityType": "pks" + }, + { + "columns": ["id"], + "nameExplicit": false, + "name": "navigation_pk", + "table": "navigation", + "entityType": "pks" + }, + { + "columns": ["id"], + "nameExplicit": false, + "name": "sessions_pk", + "table": "sessions", + "entityType": "pks" + }, + { + "columns": ["session_id"], + "nameExplicit": false, + "name": "run_messages_pk", + "table": "run_messages", + "entityType": "pks" + }, + { + "columns": ["session_id"], + "nameExplicit": false, + "name": "runs_pk", + "table": "runs", + "entityType": "pks" + }, + { + "columns": ["session_id"], + "nameExplicit": false, + "name": "image_history_pk", + "table": "image_history", + "entityType": "pks" + }, + { + "columns": [ + { + "value": "harness_kind", + "isExpression": false + }, + { + "value": "cwd", + "isExpression": false + }, + { + "value": "harness_session_id", + "isExpression": false + } + ], + "isUnique": true, + "where": null, + "origin": "manual", + "name": "sessions_native_key_unique", + "entityType": "indexes", + "table": "sessions" + }, + { + "columns": [ + { + "value": "session_id", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "tool_captures_session_id_idx", + "entityType": "indexes", + "table": "tool_captures" + } + ], + "renames": [] +} diff --git a/packages/db/drizzle/20260815111148_drop_draft_grabs/migration.sql b/packages/db/drizzle/20260815111148_drop_draft_grabs/migration.sql new file mode 100644 index 000000000..1640a0151 --- /dev/null +++ b/packages/db/drizzle/20260815111148_drop_draft_grabs/migration.sql @@ -0,0 +1 @@ +ALTER TABLE `drafts` DROP COLUMN `grabs`; \ No newline at end of file diff --git a/packages/db/drizzle/20260815111148_drop_draft_grabs/snapshot.json b/packages/db/drizzle/20260815111148_drop_draft_grabs/snapshot.json new file mode 100644 index 000000000..fe65b6db2 --- /dev/null +++ b/packages/db/drizzle/20260815111148_drop_draft_grabs/snapshot.json @@ -0,0 +1,685 @@ +{ + "version": "7", + "dialect": "sqlite", + "id": "f20c4be1-5519-42e9-a8f3-d4727da4f28b", + "prevIds": ["bd003028-f108-43e2-a0eb-585be16e9020"], + "ddl": [ + { + "name": "css_bundles", + "entityType": "tables" + }, + { + "name": "drafts", + "entityType": "tables" + }, + { + "name": "markers", + "entityType": "tables" + }, + { + "name": "navigation", + "entityType": "tables" + }, + { + "name": "sessions", + "entityType": "tables" + }, + { + "name": "tool_captures", + "entityType": "tables" + }, + { + "name": "replies", + "entityType": "tables" + }, + { + "name": "run_messages", + "entityType": "tables" + }, + { + "name": "runs", + "entityType": "tables" + }, + { + "name": "image_history", + "entityType": "tables" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "hash", + "entityType": "columns", + "table": "css_bundles" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "css", + "entityType": "columns", + "table": "css_bundles" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "created_at", + "entityType": "columns", + "table": "css_bundles" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "session_id", + "entityType": "columns", + "table": "drafts" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "text", + "entityType": "columns", + "table": "drafts" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "selection_start", + "entityType": "columns", + "table": "drafts" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "selection_end", + "entityType": "columns", + "table": "drafts" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "attachments", + "entityType": "columns", + "table": "drafts" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "updated_at", + "entityType": "columns", + "table": "drafts" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "markers" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "session_id", + "entityType": "columns", + "table": "markers" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "after_turn", + "entityType": "columns", + "table": "markers" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "kind", + "entityType": "columns", + "table": "markers" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": "'navigation'", + "generated": null, + "name": "id", + "entityType": "columns", + "table": "navigation" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "entries", + "entityType": "columns", + "table": "navigation" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "index", + "entityType": "columns", + "table": "navigation" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "updated_at", + "entityType": "columns", + "table": "navigation" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "sessions" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "harness_session_id", + "entityType": "columns", + "table": "sessions" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "harness_kind", + "entityType": "columns", + "table": "sessions" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "origin", + "entityType": "columns", + "table": "sessions" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "title", + "entityType": "columns", + "table": "sessions" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "model", + "entityType": "columns", + "table": "sessions" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "usage", + "entityType": "columns", + "table": "sessions" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "cwd", + "entityType": "columns", + "table": "sessions" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "transcript_cwd", + "entityType": "columns", + "table": "sessions" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "attached_pid", + "entityType": "columns", + "table": "sessions" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "attached_at", + "entityType": "columns", + "table": "sessions" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "deleted_at", + "entityType": "columns", + "table": "sessions" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "created_at", + "entityType": "columns", + "table": "sessions" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "updated_at", + "entityType": "columns", + "table": "sessions" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "tool_call_id", + "entityType": "columns", + "table": "tool_captures" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "kind", + "entityType": "columns", + "table": "tool_captures" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "session_id", + "entityType": "columns", + "table": "tool_captures" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "css_bundle_id", + "entityType": "columns", + "table": "tool_captures" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "payload", + "entityType": "columns", + "table": "tool_captures" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "created_at", + "entityType": "columns", + "table": "tool_captures" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "session_id", + "entityType": "columns", + "table": "replies" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "key", + "entityType": "columns", + "table": "replies" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "value", + "entityType": "columns", + "table": "replies" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "created_at", + "entityType": "columns", + "table": "replies" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "session_id", + "entityType": "columns", + "table": "run_messages" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "messages", + "entityType": "columns", + "table": "run_messages" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "updated_at", + "entityType": "columns", + "table": "run_messages" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "session_id", + "entityType": "columns", + "table": "runs" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": "'idle'", + "generated": null, + "name": "status", + "entityType": "columns", + "table": "runs" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "0", + "generated": null, + "name": "run_epoch", + "entityType": "columns", + "table": "runs" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "last_error", + "entityType": "columns", + "table": "runs" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "last_error_epoch", + "entityType": "columns", + "table": "runs" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "updated_at", + "entityType": "columns", + "table": "runs" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "session_id", + "entityType": "columns", + "table": "image_history" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "messages", + "entityType": "columns", + "table": "image_history" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "updated_at", + "entityType": "columns", + "table": "image_history" + }, + { + "columns": ["tool_call_id", "kind", "session_id"], + "nameExplicit": true, + "name": "tool_captures_pk", + "entityType": "pks", + "table": "tool_captures" + }, + { + "columns": ["session_id", "key"], + "nameExplicit": false, + "name": "replies_pk", + "entityType": "pks", + "table": "replies" + }, + { + "columns": ["hash"], + "nameExplicit": false, + "name": "css_bundles_pk", + "table": "css_bundles", + "entityType": "pks" + }, + { + "columns": ["session_id"], + "nameExplicit": false, + "name": "drafts_pk", + "table": "drafts", + "entityType": "pks" + }, + { + "columns": ["id"], + "nameExplicit": false, + "name": "markers_pk", + "table": "markers", + "entityType": "pks" + }, + { + "columns": ["id"], + "nameExplicit": false, + "name": "navigation_pk", + "table": "navigation", + "entityType": "pks" + }, + { + "columns": ["id"], + "nameExplicit": false, + "name": "sessions_pk", + "table": "sessions", + "entityType": "pks" + }, + { + "columns": ["session_id"], + "nameExplicit": false, + "name": "run_messages_pk", + "table": "run_messages", + "entityType": "pks" + }, + { + "columns": ["session_id"], + "nameExplicit": false, + "name": "runs_pk", + "table": "runs", + "entityType": "pks" + }, + { + "columns": ["session_id"], + "nameExplicit": false, + "name": "image_history_pk", + "table": "image_history", + "entityType": "pks" + }, + { + "columns": [ + { + "value": "harness_kind", + "isExpression": false + }, + { + "value": "cwd", + "isExpression": false + }, + { + "value": "harness_session_id", + "isExpression": false + } + ], + "isUnique": true, + "where": null, + "origin": "manual", + "name": "sessions_native_key_unique", + "entityType": "indexes", + "table": "sessions" + }, + { + "columns": [ + { + "value": "session_id", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "tool_captures_session_id_idx", + "entityType": "indexes", + "table": "tool_captures" + } + ], + "renames": [] +} diff --git a/packages/db/src/schema.ts b/packages/db/src/schema.ts index 15d3fdddc..ae6796e9d 100644 --- a/packages/db/src/schema.ts +++ b/packages/db/src/schema.ts @@ -24,12 +24,14 @@ export const sessions = sqliteTable( (table) => [uniqueIndex('sessions_native_key_unique').on(table.harnessKind, table.cwd, table.harnessSessionId)], ) +type PersistedAttachment = {id: string; type: string; name: string; contentType: string; data: string} + export const drafts = sqliteTable('drafts', { sessionId: text('session_id').primaryKey(), text: text('text').notNull(), selectionStart: integer('selection_start').notNull(), selectionEnd: integer('selection_end').notNull(), - grabs: text('grabs', {mode: 'json'}).$type().notNull(), + attachments: text('attachments', {mode: 'json'}).$type(), updatedAt: integer('updated_at').notNull(), }) diff --git a/packages/embed/tests/e2e/native-widget.it.test.ts b/packages/embed/tests/e2e/native-widget.it.test.ts index 3a2d74728..1cb467f8f 100644 --- a/packages/embed/tests/e2e/native-widget.it.test.ts +++ b/packages/embed/tests/e2e/native-widget.it.test.ts @@ -126,9 +126,9 @@ test.describe('native widget bridge', () => { }) => { test.setTimeout(120_000) const {page, bridge, observer} = await openNative(fixturePage) - const stagedForModel = observer.completed({ + const persistedGrab = observer.completed({ path: ['drafts', 'set'], - input: /\[view\][\s\S]*PaymentCardCell/, + input: /application\/vnd\.conciv\.grab\+json/, timeout: 30_000, }) const picked = Promise.withResolvers() @@ -148,7 +148,7 @@ test.describe('native widget bridge', () => { await callNative(page, 'grabResult', {v: 1, seq: 3, requestId: pick?.requestId, grab: NEUTRAL_GRAB}) await expect(panel(page).getByText('PaymentCardCell')).toBeVisible({timeout: 30_000}) await expect(grabPreview(page)).toHaveAttribute('src', IMAGE_DATA_URL) - expect(JSON.stringify((await stagedForModel).input)).toContain('PaymentCardCell') + expect(JSON.stringify((await persistedGrab).input)).toContain('Grabbed element') }) test('ignores a grabResult whose requestId does not match the pending pick', async ({page: fixturePage}) => { diff --git a/packages/embed/tests/e2e/navigation-hold.it.test.ts b/packages/embed/tests/e2e/navigation-hold.it.test.ts index 415dcd4aa..61c5b3d75 100644 --- a/packages/embed/tests/e2e/navigation-hold.it.test.ts +++ b/packages/embed/tests/e2e/navigation-hold.it.test.ts @@ -43,7 +43,6 @@ test.describe('holdFirstNavigationWrite holds one websocket frame without stalli text: 'sent while the navigation frame is held', selectionStart: 0, selectionEnd: 0, - grabs: [], }) expect(overtaking).toEqual({ok: true}) expect(await currentHref(suite.kit())).toBe('') diff --git a/packages/embed/tests/e2e/rpc-observer.it.test.ts b/packages/embed/tests/e2e/rpc-observer.it.test.ts index f75e5c369..fe156d029 100644 --- a/packages/embed/tests/e2e/rpc-observer.it.test.ts +++ b/packages/embed/tests/e2e/rpc-observer.it.test.ts @@ -49,7 +49,7 @@ async function openProbePage(page: Page): Promise<{page: Page; observer: ReturnT } function draft(sessionId: string, text: string): Record { - return {sessionId, text, selectionStart: 0, selectionEnd: 0, grabs: []} + return {sessionId, text, selectionStart: 0, selectionEnd: 0} } test.describe('the shared rpc observer correlates calls on both transports', () => { diff --git a/packages/extension-testkit/src/host/grab.ts b/packages/extension-testkit/src/host/grab.ts index 3455e4ae0..7254cb31d 100644 --- a/packages/extension-testkit/src/host/grab.ts +++ b/packages/extension-testkit/src/host/grab.ts @@ -28,9 +28,11 @@ function toElementSource(parts: SourceParts): ElementSource { function toGrab(element: Element): Grab { const box = element.getBoundingClientRect() const parts = sourceOf(element) + const clone = element.cloneNode(true) + const html = clone instanceof Element ? clone.outerHTML : '' return { text: element.textContent ?? '', - preview: {kind: 'dom', node: element.cloneNode(true) as HTMLElement, width: box.width, height: box.height}, + preview: {kind: 'dom', html, width: box.width, height: box.height}, source: parts ? toElementSource(parts) : null, rect: {x: box.x, y: box.y, width: box.width, height: box.height}, } diff --git a/packages/extensions/page/package.json b/packages/extensions/page/package.json index b36d62d06..2b7ccc001 100644 --- a/packages/extensions/page/package.json +++ b/packages/extensions/page/package.json @@ -59,6 +59,7 @@ }, "dependencies": { "@conciv/extension": "workspace:^", + "@conciv/grab": "workspace:^", "@conciv/page": "workspace:^", "@conciv/protocol": "workspace:^", "@conciv/ui-kit-chat": "workspace:*", diff --git a/packages/extensions/page/src/client.tsx b/packages/extensions/page/src/client.tsx index 89022d689..11115e98c 100644 --- a/packages/extensions/page/src/client.tsx +++ b/packages/extensions/page/src/client.tsx @@ -3,8 +3,16 @@ import {PAGE_SESSION_GROUP_KEY, type GroupEntry} from '@conciv/ui-kit-chat' import {PAGE_EXTENSION_NAME} from './shared/defs.js' import {PAGE_CLIENT_TOOLS} from './client/bodies.js' import {SessionCard} from './client/cards/session-card.js' +import {GrabCard} from './client/cards/grab-card.js' +import {grabAttachment} from './shared/grab-attachment.js' -export const page = defineExtension({name: PAGE_EXTENSION_NAME, tools: PAGE_CLIENT_TOOLS}).client(() => ({value: {}})) +grabAttachment.card(GrabCard) + +export const page = defineExtension({ + name: PAGE_EXTENSION_NAME, + tools: PAGE_CLIENT_TOOLS, + attachments: [grabAttachment], +}).client(() => ({value: {}})) export default page diff --git a/packages/extensions/page/src/client/cards/grab-card.tsx b/packages/extensions/page/src/client/cards/grab-card.tsx new file mode 100644 index 000000000..2487e3752 --- /dev/null +++ b/packages/extensions/page/src/client/cards/grab-card.tsx @@ -0,0 +1,161 @@ +import {createResource, createSignal, Show, type JSX} from 'solid-js' +import {useAttachment} from '@conciv/ui-kit-chat' +import {Button, Dialog} from '@conciv/ui-kit-system' +import type {AttachmentCardProps} from '@conciv/extension' +import {parseGrabPayload, type GrabPayload} from '@conciv/grab/grab-attachment' +import {sourceLabel} from '@conciv/grab' +import {GrabSnapshotFrame} from './grab-snapshot-frame.js' + +type AttachmentState = ReturnType + +type GrabPreview = NonNullable + +const CARD = + 'text-[0.6875rem] font-pw-mono p-3 border border-pw-line rounded-pw-md bg-pw-fill flex flex-col gap-2.5 items-start relative' +const PREVIEW_SLOT = 'relative max-w-full' +const OPEN_BUTTON = 'absolute inset-0 cursor-pointer' +const SOURCE_LINE = 'text-pw-text-2 flex gap-1.5 [word-break:break-all] items-center' +const AGENT_TEXT = 'text-pw-text-2 font-pw-mono text-xs whitespace-pre-wrap [word-break:break-all] m-0' +const CARD_SNAPSHOT = 'block w-auto h-auto max-w-full max-h-[40cqh]' + +const DIALOG_SNAPSHOT = 'block w-auto h-auto max-w-full max-h-[70vh]' + +const SNAPSHOT_LABEL = 'Grabbed element snapshot' + +function decodeBase64Utf8(value: string): string | null { + try { + const binary = atob(value) + const bytes = new Uint8Array(binary.length) + for (let index = 0; index < binary.length; index += 1) bytes[index] = binary.charCodeAt(index) + return new TextDecoder().decode(bytes) + } catch { + return null + } +} + +async function resolvePayload(attachment: AttachmentState): Promise { + if ('content' in attachment) + for (const part of attachment.content) + if (part.type === 'document' && part.source.type === 'data') { + const body = decodeBase64Utf8(part.source.value) + return body === null ? null : parseGrabPayload(body) + } + if (attachment.file) return parseGrabPayload(await attachment.file.text()) + return null +} + +function domPreview(preview: GrabPreview): Extract | null { + return preview.kind === 'dom' ? preview : null +} + +function imagePreview(preview: GrabPreview): Extract | null { + return preview.kind === 'image' ? preview : null +} + +function Preview(props: {preview: GrabPreview; class: string}): JSX.Element { + return ( + + {(image) => ( + {SNAPSHOT_LABEL} + )} + + } + > + {(dom) => } + + ) +} + +function GrabBody(props: {payload: GrabPayload; remove?: JSX.Element}): JSX.Element { + const [open, setOpen] = createSignal(false) + const [showAgentText, setShowAgentText] = createSignal(false) + return ( +
+ {props.remove} + {props.payload.text}}> + {(preview) => ( +
+ +
+ )} + + + {(source) => ( + + {(label) => ( + + {' '} + in {label()} + + )} + + )} + + setShowAgentText((shown) => !shown)}> + {showAgentText() ? 'Show snapshot' : 'What the agent sees'} + + } + > + + {props.payload.text}} + > + {(preview) => } + + + +
+ ) +} + +export function GrabCard(props: AttachmentCardProps): JSX.Element { + const attachment = useAttachment() + const [payload] = createResource(() => resolvePayload(attachment)) + return ( + + Reading grabbed element… +
+ } + > + + Grabbed element could not be read +
+ } + > + {(value) => } + + + ) +} diff --git a/packages/extensions/page/src/client/cards/grab-snapshot-frame.tsx b/packages/extensions/page/src/client/cards/grab-snapshot-frame.tsx new file mode 100644 index 000000000..222f6ddfc --- /dev/null +++ b/packages/extensions/page/src/client/cards/grab-snapshot-frame.tsx @@ -0,0 +1,65 @@ +import {createSignal, onMount, type JSX} from 'solid-js' + +const CSP = "default-src 'none'; img-src data:; style-src 'unsafe-inline' data:; font-src data:" + +const FRAME = 'block border-none bg-transparent pointer-events-none' + +const BASE_FALLBACK = '#000' + +const OVERLAY_FALLBACK = 'transparent' + +const SAFE_COLOR = /^[\w\s%.,()/#-]+$/ + +type Surface = {base: string; overlay: string} + +const DEFAULT_SURFACE: Surface = {base: BASE_FALLBACK, overlay: OVERLAY_FALLBACK} + +function safeColor(value: string, fallback: string): string { + const trimmed = value.trim() + return trimmed !== '' && SAFE_COLOR.test(trimmed) ? trimmed : fallback +} + +function readSurface(node: Element): Surface { + const styles = getComputedStyle(node) + return { + base: safeColor(styles.getPropertyValue('--pw-panel'), BASE_FALLBACK), + overlay: safeColor(styles.getPropertyValue('--pw-fill'), OVERLAY_FALLBACK), + } +} + +function documentFor(html: string, surface: Surface): string { + return `${html}` +} + +export function GrabSnapshotFrame(props: {html: string; width: number; height: number; class: string}): JSX.Element { + const [surface, setSurface] = createSignal(DEFAULT_SURFACE) + let node: SVGSVGElement | undefined + onMount(() => { + if (node) setSurface(readSurface(node)) + }) + return ( + { + node = element + }} + class={props.class} + width={props.width} + height={props.height} + viewBox={`0 0 ${props.width} ${props.height}`} + > + +